diff --git a/bottlecap/src/lifecycle/invocation/processor.rs b/bottlecap/src/lifecycle/invocation/processor.rs index d6e44a98e..64a8582b1 100644 --- a/bottlecap/src/lifecycle/invocation/processor.rs +++ b/bottlecap/src/lifecycle/invocation/processor.rs @@ -23,7 +23,13 @@ use crate::{ telemetry::events::{ReportMetrics, Status}, traces::{ context::SpanContext, - propagation::{DatadogCompositePropagator, Propagator}, + propagation::{ + text_map_propagator::{ + DatadogHeaderPropagator, DATADOG_PARENT_ID_KEY, DATADOG_SPAN_ID_KEY, + DATADOG_TRACE_ID_KEY, + }, + DatadogCompositePropagator, Propagator, + }, trace_processor, }, }; @@ -278,6 +284,8 @@ impl Processor { self.span.trace_id = 0; self.span.parent_id = 0; self.span.span_id = 0; + self.span.error = 0; + self.span.meta.clear(); self.extracted_span_context = None; let payload_value = match serde_json::from_slice::(&payload) { @@ -288,6 +296,7 @@ impl Processor { self.inferrer.infer_span(&payload_value, &self.aws_config); self.extracted_span_context = self.extract_span_context(&headers, &payload_value); + // Set the extracted trace context to the spans if let Some(sc) = &self.extracted_span_context { self.span.trace_id = sc.trace_id; self.span.parent_id = sc.span_id; @@ -302,6 +311,8 @@ impl Processor { } } + // If we have an inferred span, set the invocation span parent id + // to be the inferred span id, even if we don't have an extracted trace context if let Some(inferred_span) = &self.inferrer.inferred_span { self.span.parent_id = inferred_span.span_id; } @@ -312,11 +323,8 @@ impl Processor { headers: &HashMap, payload_value: &Value, ) -> Option { - if let Some(carrier) = self.inferrer.get_carrier() { - if let Some(sc) = self.propagator.extract(&carrier) { - debug!("Extracted trace context from inferred span"); - return Some(sc); - } + if let Some(sc) = self.inferrer.get_span_context(&self.propagator) { + return Some(sc); } if let Some(payload_headers) = payload_value.get("headers") { @@ -363,36 +371,51 @@ impl Processor { } fn update_span_context_from_headers(&mut self, headers: &HashMap) { - // todo: fix this, code is a copy of the existing logic in Go, not accounting - // when a 128 bit trace id exist let mut trace_id = 0; - let mut span_id = 0; let mut parent_id = 0; + let mut tags: HashMap = HashMap::new(); - // If we have a trace context, update the span context + // If we have a trace context, this means we got it from + // distributed tracing if let Some(sc) = &mut self.extracted_span_context { + debug!("Trace context was found, not extracting it from incoming headers"); trace_id = sc.trace_id; - span_id = sc.span_id; + parent_id = sc.span_id; + tags.extend(sc.tags.clone()); } - if let Some(header) = headers.get("x-datadog-trace-id") { - trace_id = header.parse::().unwrap_or(0); - } + // We are the root span, so we should extract the trace context + // from the tracer, which has sent it through end invocation headers + if trace_id == 0 { + debug!("No trace context found, extracting it from headers"); + // Extract trace context from headers manually + if let Some(header) = headers.get(DATADOG_TRACE_ID_KEY) { + trace_id = header.parse::().unwrap_or(0); + } + + if let Some(header) = headers.get(DATADOG_PARENT_ID_KEY) { + parent_id = header.parse::().unwrap_or(0); + } + + // TODO: sampling priority extraction - if let Some(header) = headers.get("x-datadog-span-id") { - span_id = header.parse::().unwrap_or(0); + // Extract tags from headers + // Used for 128 bit trace ids + tags = DatadogHeaderPropagator::extract_tags(headers); } - if let Some(header) = headers.get("x-datadog-parent-id") { - parent_id = header.parse::().unwrap_or(0); + // We should always use the generated trace id from the tracer + if let Some(header) = headers.get(DATADOG_SPAN_ID_KEY) { + self.span.span_id = header.parse::().unwrap_or(0); } self.span.trace_id = trace_id; - self.span.span_id = span_id; - // If no inferred span, set the parent id right away - if self.inferrer.inferred_span.is_none() { + if self.inferrer.inferred_span.is_some() { + self.inferrer.extend_meta(tags); + } else { self.span.parent_id = parent_id; + self.span.meta.extend(tags); } } diff --git a/bottlecap/src/lifecycle/invocation/span_inferrer.rs b/bottlecap/src/lifecycle/invocation/span_inferrer.rs index 39314dd83..a4a3a5600 100644 --- a/bottlecap/src/lifecycle/invocation/span_inferrer.rs +++ b/bottlecap/src/lifecycle/invocation/span_inferrer.rs @@ -13,19 +13,27 @@ use crate::lifecycle::invocation::triggers::{ dynamodb_event::DynamoDbRecord, event_bridge_event::EventBridgeEvent, kinesis_event::KinesisRecord, + s3_event::S3Record, sns_event::{SnsEntity, SnsRecord}, sqs_event::SqsRecord, + step_function_event::StepFunctionEvent, Trigger, FUNCTION_TRIGGER_EVENT_SOURCE_ARN_TAG, }; use crate::tags::lambda::tags::{INIT_TYPE, SNAP_START_VALUE}; - -use super::triggers::s3_event::S3Record; +use crate::traces::{context::SpanContext, propagation::Propagator}; pub struct SpanInferrer { + // Span inferred from the Lambda incoming request payload pub inferred_span: Option, + // Nested span inferred from the Lambda incoming request payload pub wrapped_inferred_span: Option, + // If the inferred span is async is_async_span: bool, + // Carrier to extract the span context from carrier: Option>, + // Generated Span Context from Step Functions + generated_span_context: Option, + // Tags generated from the trigger trigger_tags: Option>, } @@ -43,6 +51,7 @@ impl SpanInferrer { wrapped_inferred_span: None, is_async_span: false, carrier: None, + generated_span_context: None, trigger_tags: None, } } @@ -51,11 +60,13 @@ impl SpanInferrer { /// and try matching it to a `Trigger` implementation, which will create /// an inferred span and set it to `self.inferred_span` /// + #[allow(clippy::too_many_lines)] pub fn infer_span(&mut self, payload_value: &Value, aws_config: &AwsConfig) { self.inferred_span = None; self.wrapped_inferred_span = None; self.is_async_span = false; self.carrier = None; + self.generated_span_context = None; self.trigger_tags = None; let mut trigger: Option> = None; @@ -167,6 +178,11 @@ impl SpanInferrer { if let Some(t) = KinesisRecord::new(payload_value.clone()) { t.enrich_span(&mut inferred_span); + trigger = Some(Box::new(t)); + } + } else if StepFunctionEvent::is_match(payload_value) { + if let Some(t) = StepFunctionEvent::new(payload_value.clone()) { + self.generated_span_context = Some(t.get_span_context()); trigger = Some(Box::new(t)); } } else { @@ -184,7 +200,13 @@ impl SpanInferrer { self.trigger_tags = Some(trigger_tags); self.carrier = Some(t.get_carrier()); self.is_async_span = t.is_async(); - self.inferred_span = Some(inferred_span); + + // For Step Functions, there is no inferred span + if self.generated_span_context.is_some() { + self.inferred_span = None; + } else { + self.inferred_span = Some(inferred_span); + } } } @@ -263,11 +285,23 @@ impl SpanInferrer { rng.gen() } - /// Returns a clone of the carrier associated with the inferred span + /// Returns the extracted span context /// - #[must_use] - pub fn get_carrier(&self) -> Option> { - self.carrier.clone() + /// If the carrier is set, it will try to extract the span context, + /// otherwise it will + /// + pub fn get_span_context(&self, propagator: &impl Propagator) -> Option { + // Step Functions `SpanContext` is deterministically generated + if let Some(sc) = &self.generated_span_context { + return Some(sc.clone()); + } + + if let Some(sc) = self.carrier.as_ref().and_then(|c| propagator.extract(c)) { + debug!("Extracted trace context from inferred span"); + return Some(sc); + } + + None } /// Returns a clone of the tags associated with the inferred span diff --git a/bottlecap/src/lifecycle/invocation/triggers/mod.rs b/bottlecap/src/lifecycle/invocation/triggers/mod.rs index a5d0f4ae2..6802174b1 100644 --- a/bottlecap/src/lifecycle/invocation/triggers/mod.rs +++ b/bottlecap/src/lifecycle/invocation/triggers/mod.rs @@ -12,6 +12,7 @@ pub mod kinesis_event; pub mod s3_event; pub mod sns_event; pub mod sqs_event; +pub mod step_function_event; pub const DATADOG_CARRIER_KEY: &str = "_datadog"; pub const FUNCTION_TRIGGER_EVENT_SOURCE_TAG: &str = "function_trigger.event_source"; diff --git a/bottlecap/src/lifecycle/invocation/triggers/step_function_event.rs b/bottlecap/src/lifecycle/invocation/triggers/step_function_event.rs new file mode 100644 index 000000000..91eb2af54 --- /dev/null +++ b/bottlecap/src/lifecycle/invocation/triggers/step_function_event.rs @@ -0,0 +1,366 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::{ + lifecycle::invocation::triggers::{Trigger, FUNCTION_TRIGGER_EVENT_SOURCE_TAG}, + traces::{ + context::{Sampling, SpanContext}, + propagation::text_map_propagator::DATADOG_HIGHER_ORDER_TRACE_ID_BITS_KEY, + }, +}; + +#[allow(clippy::module_name_repetitions)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct LegacyStepFunctionEvent { + #[serde(rename = "Payload")] + pub payload: StepFunctionEvent, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct StepFunctionEvent { + #[serde(rename = "Execution")] + pub execution: Execution, + #[serde(rename = "State")] + pub state: State, + #[serde(rename = "StateMachine")] + pub state_machine: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct Execution { + #[serde(rename = "Id")] + id: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct State { + #[serde(rename = "Name")] + name: String, + #[serde(rename = "EnteredTime")] + entered_time: String, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct StateMachine { + #[serde(rename = "Id")] + id: String, +} + +impl Trigger for StepFunctionEvent { + fn new(payload: serde_json::Value) -> Option + where + Self: Sized, + { + let p = payload.get("Payload").unwrap_or(&payload); + match serde_json::from_value::(p.clone()) { + Ok(event) => Some(event), + Err(e) => { + tracing::debug!("Failed to deserialize Step Function Event: {e}"); + None + } + } + } + + fn is_match(payload: &serde_json::Value) -> bool + where + Self: Sized, + { + // Check first if the payload is a Legacy Step Function event + let p = payload.get("Payload").unwrap_or(payload); + + let execution_id = p + .get("Execution") + .and_then(Value::as_object) + .and_then(|e| e.get("Id")); + let state = p.get("State").and_then(Value::as_object); + let name = state.and_then(|s| s.get("Name")); + let entered_time = state.and_then(|s| s.get("EnteredTime")); + + execution_id.is_some() && name.is_some() && entered_time.is_some() + } + + fn enrich_span(&self, _span: &mut datadog_trace_protobuf::pb::Span) {} + + fn get_tags(&self) -> HashMap { + HashMap::from([( + FUNCTION_TRIGGER_EVENT_SOURCE_TAG.to_string(), + "states".to_string(), + )]) + } + + fn get_arn(&self, _region: &str) -> String { + if let Some(sm) = &self.state_machine { + return sm.id.clone(); + } + + String::new() + } + + fn get_carrier(&self) -> HashMap { + HashMap::new() + } + + fn is_async(&self) -> bool { + true + } +} + +impl StepFunctionEvent { + #[must_use] + pub fn get_span_context(&self) -> SpanContext { + let (lo_tid, hi_tid) = Self::generate_trace_id(self.execution.id.clone()); + let tags = HashMap::from([( + DATADOG_HIGHER_ORDER_TRACE_ID_BITS_KEY.to_string(), + format!("{hi_tid:x}"), + )]); + + let parent_id = Self::generate_parent_id( + self.execution.id.clone(), + self.state.name.clone(), + self.state.entered_time.clone(), + ); + + SpanContext { + trace_id: lo_tid, + span_id: parent_id, + // Priority Auto Keep + sampling: Some(Sampling { + priority: Some(1), + mechanism: None, + }), + origin: Some("states".to_string()), + tags, + links: vec![], + } + } + + /// Generates a random 64 bit ID from the formatted hash of the + /// Step Function Execution ARN, the State Name, and the State Entered Time + /// + fn generate_parent_id( + execution_id: String, + state_name: String, + state_entered_time: String, + ) -> u64 { + let unique_string = format!("{execution_id}#{state_name}#{state_entered_time}"); + + let hash = Sha256::digest(unique_string.as_bytes()); + Self::get_positive_u64(&hash[0..8]) + } + + /// Generates a random 128 bit ID from the Step Function Execution ARN + /// + fn generate_trace_id(execution_arn: String) -> (u64, u64) { + let hash = Sha256::digest(execution_arn.as_bytes()); + + let lower_order_bits = Self::get_positive_u64(&hash[8..16]); + let higher_order_bits = Self::get_positive_u64(&hash[0..8]); + + (lower_order_bits, higher_order_bits) + } + + /// Converts the first 8 bytes of a byte array to a positive `u64` + /// + fn get_positive_u64(hash_bytes: &[u8]) -> u64 { + let mut result: u64 = hash_bytes + .iter() + .take(8) + .fold(0, |acc, &byte| (acc << 8) + u64::from(byte)); + + // Ensure the highest bit is always 0 + result &= !(1u64 << 63); + + // Return 1 if result is 0 + if result == 0 { + 1 + } else { + result + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lifecycle::invocation::triggers::test_utils::read_json_file; + + #[test] + fn test_new() { + let json = read_json_file("step_function_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize into Value"); + let result = StepFunctionEvent::new(payload).expect("Failed to deserialize into Event"); + + let expected = StepFunctionEvent { + execution: Execution { + id: String::from("arn:aws:states:us-east-1:425362996713:execution:agocsTestSF:bc9f281c-3daa-4e5a-9a60-471a3810bf44"), + }, + state: State { + name: String::from("agocsTest1"), + entered_time: String::from("2024-07-30T19:55:53.018Z"), + }, + state_machine: Some(StateMachine { + id: String::from("arn:aws:states:us-east-1:425362996713:stateMachine:agocsTestSF"), + }), + }; + + assert_eq!(result, expected); + } + + #[test] + fn test_new_legacy_event() { + let json = read_json_file("step_function_legacy_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize into Value"); + let result = StepFunctionEvent::new(payload).expect("Failed to deserialize into Event"); + + let expected = StepFunctionEvent { + execution: Execution { + id: String::from("arn:aws:states:us-east-1:425362996713:execution:agocsTestSF:bc9f281c-3daa-4e5a-9a60-471a3810bf44"), + }, + state: State { + name: String::from("agocsTest1"), + entered_time: String::from("2024-07-30T19:55:53.018Z"), + }, + state_machine: Some(StateMachine { + id: String::from("arn:aws:states:us-east-1:425362996713:stateMachine:agocsTestSF"), + }), + }; + + assert_eq!(result, expected); + } + + #[test] + fn test_is_match() { + let json = read_json_file("step_function_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize StepFunctionEvent"); + + assert!(StepFunctionEvent::is_match(&payload)); + } + + #[test] + fn test_is_match_legacy_event() { + let json = read_json_file("step_function_legacy_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize StepFunctionEvent"); + + assert!(StepFunctionEvent::is_match(&payload)); + } + + #[test] + fn test_is_not_match() { + let json = read_json_file("sqs_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize SqsRecord"); + assert!(!StepFunctionEvent::is_match(&payload)); + } + + #[test] + fn test_get_tags() { + let json = read_json_file("step_function_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize into Value"); + let event = + StepFunctionEvent::new(payload).expect("Failed to deserialize StepFunctionEvent"); + let tags = event.get_tags(); + + let expected = HashMap::from([( + "function_trigger.event_source".to_string(), + "states".to_string(), + )]); + + assert_eq!(tags, expected); + } + + #[test] + fn test_get_arn() { + let json = read_json_file("step_function_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize into Value"); + let event = + StepFunctionEvent::new(payload).expect("Failed to deserialize StepFunctionEvent"); + assert_eq!( + event.get_arn("us-east-1"), + "arn:aws:states:us-east-1:425362996713:stateMachine:agocsTestSF" + ); + } + + #[test] + fn test_get_carrier() { + let json = read_json_file("step_function_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize into Value"); + let event = + StepFunctionEvent::new(payload).expect("Failed to deserialize StepFunctionEvent"); + let carrier = event.get_carrier(); + + let expected = HashMap::new(); + + assert_eq!(carrier, expected); + } + + #[test] + fn get_span_context() { + let json = read_json_file("step_function_event.json"); + let payload = serde_json::from_str(&json).expect("Failed to deserialize into Value"); + let event = + StepFunctionEvent::new(payload).expect("Failed to deserialize StepFunctionEvent"); + + let span_context = event.get_span_context(); + + let expected = SpanContext { + trace_id: 5744042798732701615, + span_id: 2902498116043018663, + sampling: Some(Sampling { + priority: Some(1), + mechanism: None, + }), + origin: Some("states".to_string()), + tags: HashMap::from([( + DATADOG_HIGHER_ORDER_TRACE_ID_BITS_KEY.to_string(), + "1914fe7789eb32be".to_string(), + )]), + links: vec![], + }; + + assert_eq!(span_context, expected); + } + + #[test] + fn test_generate_parent_id() { + let parent_id = StepFunctionEvent::generate_parent_id( + String::from("arn:aws:states:sa-east-1:601427271234:express:DatadogStateMachine:acaf1a67-336a-e854-1599-2a627eb2dd8a:c8baf081-31f1-464d-971f-70cb17d01111"), + String::from("step-one"), + String::from("2022-12-08T21:08:19.224Z") + ); + + assert_eq!(parent_id, 4340734536022949921); + + let parent_id = StepFunctionEvent::generate_parent_id( + String::from("arn:aws:states:sa-east-1:601427271234:express:DatadogStateMachine:acaf1a67-336a-e854-1599-2a627eb2dd8a:c8baf081-31f1-464d-971f-70cb17d01111"), + String::from("step-one"), + String::from("2022-12-08T21:08:19.224Y") + ); + + assert_eq!(parent_id, 981693280319792699); + } + + #[test] + fn test_generate_trace_id() { + let (lo_tid, hi_tid) = StepFunctionEvent::generate_trace_id(String::from( + "arn:aws:states:sa-east-1:425362996713:stateMachine:MyStateMachine-b276uka1j", + )); + let hex_tid = format!("{:x}", hi_tid); + + assert_eq!(lo_tid, 1680583253837593461); + assert_eq!(hi_tid, 6984552746569958392); + + assert_eq!(hex_tid, "60ee1db79e4803f8"); + + let (lo_tid, hi_tid) = StepFunctionEvent::generate_trace_id( + String::from("arn:aws:states:us-east-1:425362996713:execution:agocsTestSF:bc9f281c-3daa-4e5a-9a60-471a3810bf44") + ); + let hex_tid = format!("{:x}", hi_tid); + + assert_eq!(lo_tid, 5744042798732701615); + assert_eq!(hi_tid, 1807349139850867390); + + assert_eq!(hex_tid, "1914fe7789eb32be"); + } +} diff --git a/bottlecap/src/metrics/enhanced/statfs.rs b/bottlecap/src/metrics/enhanced/statfs.rs index 0da8a1828..84e7412f1 100644 --- a/bottlecap/src/metrics/enhanced/statfs.rs +++ b/bottlecap/src/metrics/enhanced/statfs.rs @@ -5,6 +5,7 @@ use std::io; use std::path::Path; #[cfg(not(target_os = "windows"))] +#[allow(clippy::cast_lossless)] /// Returns the block size, total number of blocks, and number of blocks available for the specified directory path. /// pub fn statfs_info(path: &str) -> Result<(f64, f64, f64), io::Error> { diff --git a/bottlecap/src/traces/propagation/text_map_propagator.rs b/bottlecap/src/traces/propagation/text_map_propagator.rs index 34b482643..f1c5cbcf4 100644 --- a/bottlecap/src/traces/propagation/text_map_propagator.rs +++ b/bottlecap/src/traces/propagation/text_map_propagator.rs @@ -13,7 +13,8 @@ use crate::traces::propagation::{ // Datadog Keys pub const DATADOG_TRACE_ID_KEY: &str = "x-datadog-trace-id"; -const DATADOG_PARENT_ID_KEY: &str = "x-datadog-parent-id"; +pub const DATADOG_PARENT_ID_KEY: &str = "x-datadog-parent-id"; +pub const DATADOG_SPAN_ID_KEY: &str = "x-datadog-span-id"; pub const DATADOG_SAMPLING_PRIORITY_KEY: &str = "x-datadog-sampling-priority"; const DATADOG_ORIGIN_KEY: &str = "x-datadog-origin"; pub const DATADOG_TAGS_KEY: &str = "x-datadog-tags"; @@ -148,7 +149,7 @@ impl DatadogHeaderPropagator { Some(origin.to_string()) } - fn extract_tags(carrier: &dyn Extractor) -> HashMap { + pub fn extract_tags(carrier: &dyn Extractor) -> HashMap { let carrier_tags = carrier.get(DATADOG_TAGS_KEY).unwrap_or_default(); let mut tags: HashMap = HashMap::new(); diff --git a/bottlecap/tests/payloads/step_function_event.json b/bottlecap/tests/payloads/step_function_event.json new file mode 100644 index 000000000..1461c7164 --- /dev/null +++ b/bottlecap/tests/payloads/step_function_event.json @@ -0,0 +1,19 @@ +{ + "Execution": { + "Id": "arn:aws:states:us-east-1:425362996713:execution:agocsTestSF:bc9f281c-3daa-4e5a-9a60-471a3810bf44", + "Input": {}, + "StartTime": "2024-07-30T19:55:52.976Z", + "Name": "bc9f281c-3daa-4e5a-9a60-471a3810bf44", + "RoleArn": "arn:aws:iam::425362996713:role/test-serverless-stepfunctions-dev-AgocsTestSFRole-tRkeFXScjyk4", + "RedriveCount": 0 + }, + "StateMachine": { + "Id": "arn:aws:states:us-east-1:425362996713:stateMachine:agocsTestSF", + "Name": "agocsTestSF" + }, + "State": { + "Name": "agocsTest1", + "EnteredTime": "2024-07-30T19:55:53.018Z", + "RetryCount": 0 + } +} diff --git a/bottlecap/tests/payloads/step_function_legacy_event.json b/bottlecap/tests/payloads/step_function_legacy_event.json new file mode 100644 index 000000000..74e4c010a --- /dev/null +++ b/bottlecap/tests/payloads/step_function_legacy_event.json @@ -0,0 +1,21 @@ +{ + "Payload": { + "Execution": { + "Id": "arn:aws:states:us-east-1:425362996713:execution:agocsTestSF:bc9f281c-3daa-4e5a-9a60-471a3810bf44", + "Input": {}, + "StartTime": "2024-07-30T19:55:52.976Z", + "Name": "bc9f281c-3daa-4e5a-9a60-471a3810bf44", + "RoleArn": "arn:aws:iam::425362996713:role/test-serverless-stepfunctions-dev-AgocsTestSFRole-tRkeFXScjyk4", + "RedriveCount": 0 + }, + "StateMachine": { + "Id": "arn:aws:states:us-east-1:425362996713:stateMachine:agocsTestSF", + "Name": "agocsTestSF" + }, + "State": { + "Name": "agocsTest1", + "EnteredTime": "2024-07-30T19:55:53.018Z", + "RetryCount": 0 + } + } +}