From 66fb2cacc58fd00f49a7fbe4dd90f9755bc40968 Mon Sep 17 00:00:00 2001 From: Lucas Pimentel Date: Mon, 1 Dec 2025 11:41:13 -0500 Subject: [PATCH 1/7] add failing tests --- .../src/msgpack_decoder/v04/mod.rs | 35 +++++++++ libdd-trace-utils/src/trace_utils.rs | 74 +++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs index 0a214b5997..1094d79a3b 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs @@ -492,6 +492,41 @@ mod tests { assert!(decoded_span.span_links.is_empty()); } + + #[test] + fn test_decoder_meta_with_null_values() { + // SLES-2528: Test that meta HashMap can handle null values + // This reproduces the issue where Java tracer sends null values in meta tags + let mut span = create_test_json_span(1, 2, 0, 0, false); + span["meta"] = json!({ + "key1": "value1", + "key2": null, // This null value causes the error + "key3": "value3" + }); + + let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); + let (decoded_traces, _) = + from_bytes(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + + assert_eq!(1, decoded_traces.len()); + assert_eq!(1, decoded_traces[0].len()); + let decoded_span = &decoded_traces[0][0]; + + // After the fix, null values should be converted to empty strings + assert_eq!( + "value1", + decoded_span.meta[&BytesString::from_slice("key1".as_ref()).unwrap()].as_str() + ); + assert_eq!( + "", + decoded_span.meta[&BytesString::from_slice("key2".as_ref()).unwrap()].as_str() + ); + assert_eq!( + "value3", + decoded_span.meta[&BytesString::from_slice("key3".as_ref()).unwrap()].as_str() + ); + } + #[test] fn test_decoder_read_string_wrong_format() { let span = SpanBytes { diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index 4b74c149fb..849a66da41 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -1197,4 +1197,78 @@ mod tests { 1.0 ); } + + #[test] + fn test_rmp_serde_deserialize_meta_with_null_values() { + // SLES-2528: Test that reproduces the customer's exact error path + // When Java tracer sends null values in meta HashMap, rmp-serde deserialization fails + // with "invalid type: unit value, expected a string" + // + // This test uses the same deserialization path as the customer: + // get_traces_from_request_body() -> rmp_serde::from_read() -> Vec> + + use serde_json::json; + + // Create a JSON representation with null value in meta + let span_json = json!({ + "service": "test-service", + "name": "test_name", + "resource": "test-resource", + "trace_id": 1_u64, + "span_id": 2_u64, + "parent_id": 0_u64, + "start": 0_i64, + "duration": 5_i64, + "error": 0_i32, + "meta": { + "service": "test-service", + "env": "test-env", + "runtime-id": "test-runtime-id-value", + "problematic_key": null // This null value causes the error + }, + "metrics": {}, + "type": "", + "meta_struct": {}, + "span_links": [], + "span_events": [] + }); + + let traces_json = vec![vec![span_json]]; + + // Serialize to MessagePack - this will include the null value + let encoded_data = rmp_serde::to_vec(&traces_json).unwrap(); + + // Deserialize using rmp_serde::from_read with pb::Span target type + // Before fix: This fails with "invalid type: unit value, expected a string" + // After fix: This should succeed and convert null to empty string or skip the entry + let traces: Vec> = rmp_serde::from_read(&encoded_data[..]) + .expect("Failed to deserialize traces with null values in meta"); + + assert_eq!(1, traces.len()); + assert_eq!(1, traces[0].len()); + let decoded_span = &traces[0][0]; + + // Verify basic fields + assert_eq!("test-service", decoded_span.service); + assert_eq!("test_name", decoded_span.name); + assert_eq!("test-resource", decoded_span.resource); + + // After the fix, null values should either be: + // 1. Converted to empty strings, or + // 2. Filtered out (key not present in map) + assert_eq!("test-service", decoded_span.meta.get("service").unwrap()); + assert_eq!("test-env", decoded_span.meta.get("env").unwrap()); + assert_eq!( + "test-runtime-id-value", + decoded_span.meta.get("runtime-id").unwrap() + ); + + // The problematic_key should either be empty string or not present + match decoded_span.meta.get("problematic_key") { + Some(value) => assert_eq!("", value, "Null value should be converted to empty string"), + None => { + // Key was filtered out - also acceptable + } + } + } } From 19de558d51f2560a1ae13c48f63570740d9d5200 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 10 Dec 2025 13:41:28 -0500 Subject: [PATCH 2/7] convert null to empty string --- libdd-trace-protobuf/build.rs | 2 +- libdd-trace-protobuf/src/deserializers.rs | 18 ++++++++++ libdd-trace-protobuf/src/pb.rs | 2 +- .../src/msgpack_decoder/decode/string.rs | 3 +- .../src/msgpack_decoder/v04/mod.rs | 3 -- libdd-trace-utils/src/trace_utils.rs | 33 ++++--------------- 6 files changed, 28 insertions(+), 33 deletions(-) diff --git a/libdd-trace-protobuf/build.rs b/libdd-trace-protobuf/build.rs index 983b1b5ffb..dcecdad2f8 100644 --- a/libdd-trace-protobuf/build.rs +++ b/libdd-trace-protobuf/build.rs @@ -93,7 +93,7 @@ fn generate_protobuf() { ); config.field_attribute( ".pb.Span.meta", - "#[serde(default)] #[serde(deserialize_with = \"crate::deserializers::deserialize_null_into_default\")]", + "#[serde(default)] #[serde(deserialize_with = \"crate::deserializers::deserialize_map_with_nullable_values\")]", ); config.field_attribute( ".pb.Span.metrics", diff --git a/libdd-trace-protobuf/src/deserializers.rs b/libdd-trace-protobuf/src/deserializers.rs index a90652635a..2c239bac03 100644 --- a/libdd-trace-protobuf/src/deserializers.rs +++ b/libdd-trace-protobuf/src/deserializers.rs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 use serde::{Deserialize, Deserializer}; +use std::collections::HashMap; pub fn deserialize_null_into_default<'de, D, T>(deserializer: D) -> Result where @@ -12,6 +13,23 @@ where Ok(opt.unwrap_or_default()) } +/// Deserialize a HashMap where null values are skipped. +pub fn deserialize_map_with_nullable_values<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let opt: Option>> = Option::deserialize(deserializer)?; + match opt { + None => Ok(HashMap::new()), + Some(map) => Ok(map + .into_iter() + .filter_map(|(k, v)| v.map(|val| (k, val))) + .collect()), + } +} + pub fn is_default(t: &T) -> bool { t == &T::default() } diff --git a/libdd-trace-protobuf/src/pb.rs b/libdd-trace-protobuf/src/pb.rs index 5234baf86d..752e2e09ed 100644 --- a/libdd-trace-protobuf/src/pb.rs +++ b/libdd-trace-protobuf/src/pb.rs @@ -301,7 +301,7 @@ pub struct Span { /// @gotags: json:"meta,omitempty" msg:"meta,omitempty" #[prost(map = "string, string", tag = "10")] #[serde(default)] - #[serde(deserialize_with = "crate::deserializers::deserialize_null_into_default")] + #[serde(deserialize_with = "crate::deserializers::deserialize_map_with_nullable_values")] pub meta: ::std::collections::HashMap< ::prost::alloc::string::String, ::prost::alloc::string::String, diff --git a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs index 215fdd941a..da2c0f6aa9 100644 --- a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs +++ b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs @@ -56,6 +56,7 @@ pub fn read_nullable_string<'a>(buf: &mut &'a [u8]) -> Result<&'a str, DecodeErr /// # Errors /// Fails if the buffer does not contain a valid map length prefix, /// or if any key or value is not a valid utf8 msgpack string. +/// Null values are converted to empty strings. #[inline] pub fn read_str_map_to_strings<'a>( buf: &mut &'a [u8], @@ -67,7 +68,7 @@ pub fn read_str_map_to_strings<'a>( let mut map = HashMap::with_capacity(len.try_into().expect("Unable to cast map len to usize")); for _ in 0..len { let key = read_string_ref(buf)?; - let value = read_string_ref(buf)?; + let value = read_nullable_string(buf)?; map.insert(key, value); } Ok(map) diff --git a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs index 1094d79a3b..dbfd8023b7 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs @@ -495,8 +495,6 @@ mod tests { #[test] fn test_decoder_meta_with_null_values() { - // SLES-2528: Test that meta HashMap can handle null values - // This reproduces the issue where Java tracer sends null values in meta tags let mut span = create_test_json_span(1, 2, 0, 0, false); span["meta"] = json!({ "key1": "value1", @@ -512,7 +510,6 @@ mod tests { assert_eq!(1, decoded_traces[0].len()); let decoded_span = &decoded_traces[0][0]; - // After the fix, null values should be converted to empty strings assert_eq!( "value1", decoded_span.meta[&BytesString::from_slice("key1".as_ref()).unwrap()].as_str() diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index 849a66da41..cf4ed3b323 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -1200,13 +1200,6 @@ mod tests { #[test] fn test_rmp_serde_deserialize_meta_with_null_values() { - // SLES-2528: Test that reproduces the customer's exact error path - // When Java tracer sends null values in meta HashMap, rmp-serde deserialization fails - // with "invalid type: unit value, expected a string" - // - // This test uses the same deserialization path as the customer: - // get_traces_from_request_body() -> rmp_serde::from_read() -> Vec> - use serde_json::json; // Create a JSON representation with null value in meta @@ -1224,7 +1217,7 @@ mod tests { "service": "test-service", "env": "test-env", "runtime-id": "test-runtime-id-value", - "problematic_key": null // This null value causes the error + "problematic_key": null // Ensure this null value does not cause an error }, "metrics": {}, "type": "", @@ -1234,13 +1227,7 @@ mod tests { }); let traces_json = vec![vec![span_json]]; - - // Serialize to MessagePack - this will include the null value let encoded_data = rmp_serde::to_vec(&traces_json).unwrap(); - - // Deserialize using rmp_serde::from_read with pb::Span target type - // Before fix: This fails with "invalid type: unit value, expected a string" - // After fix: This should succeed and convert null to empty string or skip the entry let traces: Vec> = rmp_serde::from_read(&encoded_data[..]) .expect("Failed to deserialize traces with null values in meta"); @@ -1248,27 +1235,19 @@ mod tests { assert_eq!(1, traces[0].len()); let decoded_span = &traces[0][0]; - // Verify basic fields assert_eq!("test-service", decoded_span.service); assert_eq!("test_name", decoded_span.name); assert_eq!("test-resource", decoded_span.resource); - - // After the fix, null values should either be: - // 1. Converted to empty strings, or - // 2. Filtered out (key not present in map) assert_eq!("test-service", decoded_span.meta.get("service").unwrap()); assert_eq!("test-env", decoded_span.meta.get("env").unwrap()); assert_eq!( "test-runtime-id-value", decoded_span.meta.get("runtime-id").unwrap() ); - - // The problematic_key should either be empty string or not present - match decoded_span.meta.get("problematic_key") { - Some(value) => assert_eq!("", value, "Null value should be converted to empty string"), - None => { - // Key was filtered out - also acceptable - } - } + // Assert that the null value was filtered out (key not present in map) + assert!( + decoded_span.meta.get("problematic_key").is_none(), + "Null value should be skipped, but key was present" + ); } } From 647e0b1d8b48e37e794178757d02d2618e12b458 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 10 Dec 2025 13:54:03 -0500 Subject: [PATCH 3/7] handle null value consistently --- .../src/msgpack_decoder/decode/string.rs | 9 ++++++--- libdd-trace-utils/src/msgpack_decoder/v04/mod.rs | 12 +++++++----- libdd-trace-utils/src/trace_utils.rs | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs index da2c0f6aa9..a86adaa963 100644 --- a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs +++ b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs @@ -56,7 +56,7 @@ pub fn read_nullable_string<'a>(buf: &mut &'a [u8]) -> Result<&'a str, DecodeErr /// # Errors /// Fails if the buffer does not contain a valid map length prefix, /// or if any key or value is not a valid utf8 msgpack string. -/// Null values are converted to empty strings. +/// Null values are skipped (key not inserted into map). #[inline] pub fn read_str_map_to_strings<'a>( buf: &mut &'a [u8], @@ -68,8 +68,11 @@ pub fn read_str_map_to_strings<'a>( let mut map = HashMap::with_capacity(len.try_into().expect("Unable to cast map len to usize")); for _ in 0..len { let key = read_string_ref(buf)?; - let value = read_nullable_string(buf)?; - map.insert(key, value); + // Skip null values - only insert if value is not null + if !handle_null_marker(buf) { + let value = read_string_ref(buf)?; + map.insert(key, value); + } } Ok(map) } diff --git a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs index dbfd8023b7..26ceb79f8e 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs @@ -492,13 +492,12 @@ mod tests { assert!(decoded_span.span_links.is_empty()); } - #[test] fn test_decoder_meta_with_null_values() { let mut span = create_test_json_span(1, 2, 0, 0, false); span["meta"] = json!({ "key1": "value1", - "key2": null, // This null value causes the error + "key2": null, // This null value should be skipped "key3": "value3" }); @@ -514,9 +513,12 @@ mod tests { "value1", decoded_span.meta[&BytesString::from_slice("key1".as_ref()).unwrap()].as_str() ); - assert_eq!( - "", - decoded_span.meta[&BytesString::from_slice("key2".as_ref()).unwrap()].as_str() + assert!( + decoded_span + .meta + .get(&BytesString::from_slice("key2".as_ref()).unwrap()) + .is_none(), + "Null value should be skipped, but key was present" ); assert_eq!( "value3", diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index cf4ed3b323..502e1e686a 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -1246,7 +1246,7 @@ mod tests { ); // Assert that the null value was filtered out (key not present in map) assert!( - decoded_span.meta.get("problematic_key").is_none(), + !decoded_span.meta.contains_key("problematic_key"), "Null value should be skipped, but key was present" ); } From 2390e7f4f57f22473a00b9b9a8ebf5751fd6517a Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 10 Dec 2025 14:12:41 -0500 Subject: [PATCH 4/7] fixes --- libdd-trace-utils/src/msgpack_decoder/decode/string.rs | 2 +- libdd-trace-utils/src/msgpack_decoder/v04/mod.rs | 2 +- libdd-trace-utils/src/trace_utils.rs | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs index a86adaa963..e512017b4e 100644 --- a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs +++ b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs @@ -68,7 +68,7 @@ pub fn read_str_map_to_strings<'a>( let mut map = HashMap::with_capacity(len.try_into().expect("Unable to cast map len to usize")); for _ in 0..len { let key = read_string_ref(buf)?; - // Skip null values - only insert if value is not null + // Only insert if value is not null if !handle_null_marker(buf) { let value = read_string_ref(buf)?; map.insert(key, value); diff --git a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs index 26ceb79f8e..020cc010a1 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs @@ -503,7 +503,7 @@ mod tests { let encoded_data = rmp_serde::to_vec_named(&vec![vec![span]]).unwrap(); let (decoded_traces, _) = - from_bytes(tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); + from_bytes(libdd_tinybytes::Bytes::from(encoded_data)).expect("Decoding failed"); assert_eq!(1, decoded_traces.len()); assert_eq!(1, decoded_traces[0].len()); diff --git a/libdd-trace-utils/src/trace_utils.rs b/libdd-trace-utils/src/trace_utils.rs index 502e1e686a..00f556f145 100644 --- a/libdd-trace-utils/src/trace_utils.rs +++ b/libdd-trace-utils/src/trace_utils.rs @@ -1200,8 +1200,6 @@ mod tests { #[test] fn test_rmp_serde_deserialize_meta_with_null_values() { - use serde_json::json; - // Create a JSON representation with null value in meta let span_json = json!({ "service": "test-service", From deda6190c2b230a8fc6ff07c683a6f00d5ed5846 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 10 Dec 2025 15:53:49 -0500 Subject: [PATCH 5/7] clippy --- libdd-trace-protobuf/src/pb.rs | 4 +++- libdd-trace-utils/src/msgpack_decoder/v04/mod.rs | 6 ++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libdd-trace-protobuf/src/pb.rs b/libdd-trace-protobuf/src/pb.rs index 752e2e09ed..06f46ba561 100644 --- a/libdd-trace-protobuf/src/pb.rs +++ b/libdd-trace-protobuf/src/pb.rs @@ -301,7 +301,9 @@ pub struct Span { /// @gotags: json:"meta,omitempty" msg:"meta,omitempty" #[prost(map = "string, string", tag = "10")] #[serde(default)] - #[serde(deserialize_with = "crate::deserializers::deserialize_map_with_nullable_values")] + #[serde( + deserialize_with = "crate::deserializers::deserialize_map_with_nullable_values" + )] pub meta: ::std::collections::HashMap< ::prost::alloc::string::String, ::prost::alloc::string::String, diff --git a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs index 020cc010a1..0013082652 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs @@ -514,10 +514,8 @@ mod tests { decoded_span.meta[&BytesString::from_slice("key1".as_ref()).unwrap()].as_str() ); assert!( - decoded_span - .meta - .get(&BytesString::from_slice("key2".as_ref()).unwrap()) - .is_none(), + !decoded_span + .meta.contains_key(&BytesString::from_slice("key2".as_ref()).unwrap()), "Null value should be skipped, but key was present" ); assert_eq!( From 046d2a1d7f1fdcdc8a1fc2b56bc0eb14c37d7163 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Wed, 10 Dec 2025 15:55:45 -0500 Subject: [PATCH 6/7] fmt --- libdd-trace-utils/src/msgpack_decoder/v04/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs index 0013082652..c9dd29892b 100644 --- a/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs +++ b/libdd-trace-utils/src/msgpack_decoder/v04/mod.rs @@ -515,7 +515,8 @@ mod tests { ); assert!( !decoded_span - .meta.contains_key(&BytesString::from_slice("key2".as_ref()).unwrap()), + .meta + .contains_key(&BytesString::from_slice("key2".as_ref()).unwrap()), "Null value should be skipped, but key was present" ); assert_eq!( From 3bf69269e00d9ca85bab57c2dd7bea110e34b875 Mon Sep 17 00:00:00 2001 From: shreyamalpani Date: Thu, 11 Dec 2025 16:30:38 -0500 Subject: [PATCH 7/7] update comment --- libdd-trace-utils/src/msgpack_decoder/decode/string.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs index e512017b4e..9fff188c82 100644 --- a/libdd-trace-utils/src/msgpack_decoder/decode/string.rs +++ b/libdd-trace-utils/src/msgpack_decoder/decode/string.rs @@ -82,6 +82,7 @@ pub fn read_str_map_to_strings<'a>( /// # Errors /// Fails if the buffer does not contain a valid map length prefix, /// or if any key or value is not a valid utf8 msgpack string. +/// Null values are skipped (key not inserted into map). #[inline] pub fn read_nullable_str_map_to_strings<'a>( buf: &mut &'a [u8],