feat: otel process ctxt protobuf encoding

LICENSE-3rdparty.yml

@@ -1,4 +1,4 @@
-root_name: builder, build_common, tools, libdd-alloc, libdd-crashtracker, libdd-common, libdd-telemetry, libdd-ddsketch, libdd-crashtracker-ffi, libdd-common-ffi, datadog-ffe, datadog-ffe-ffi, datadog-ipc, datadog-ipc-macros, libdd-tinybytes, tarpc, tarpc-plugins, spawn_worker, cc_utils, libdd-library-config, libdd-library-config-ffi, datadog-live-debugger, libdd-data-pipeline, libdd-dogstatsd-client, libdd-trace-protobuf, libdd-trace-stats, libdd-trace-utils, libdd-trace-normalization, libdd-log, datadog-live-debugger-ffi, libdd-profiling, libdd-profiling-protobuf, libdd-profiling-ffi, libdd-data-pipeline-ffi, libdd-ddsketch-ffi, libdd-log-ffi, libdd-telemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, libdd-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib, bin_tests
+root_name: builder, build_common, tools, libdd-alloc, libdd-crashtracker, libdd-common, libdd-telemetry, libdd-ddsketch, libdd-crashtracker-ffi, libdd-common-ffi, datadog-ffe, datadog-ffe-ffi, datadog-ipc, datadog-ipc-macros, libdd-tinybytes, tarpc, tarpc-plugins, spawn_worker, cc_utils, libdd-library-config, libdd-trace-protobuf, libdd-library-config-ffi, datadog-live-debugger, libdd-data-pipeline, libdd-dogstatsd-client, libdd-trace-stats, libdd-trace-utils, libdd-trace-normalization, libdd-log, datadog-live-debugger-ffi, libdd-profiling, libdd-profiling-protobuf, libdd-profiling-ffi, libdd-data-pipeline-ffi, libdd-ddsketch-ffi, libdd-log-ffi, libdd-telemetry-ffi, symbolizer-ffi, datadog-profiling-replayer, datadog-remote-config, datadog-sidecar, datadog-sidecar-macros, datadog-sidecar-ffi, libdd-trace-obfuscation, datadog-tracer-flare, sidecar_mockgen, test_spawn_from_lib, bin_tests
 third_party_libraries:
 - package_name: addr2line
   package_version: 0.24.2
@@ -32028,9 +32028,9 @@ third_party_libraries:
 - package_name: stringmetrics
   package_version: 2.2.2
   repository: https://github.com/pluots/stringmetrics
-  license: License specified in file ($CARGO_HOME/registry/src/github.com-25cdd57fae9f0462/stringmetrics-2.2.2/LICENSE)
+  license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/stringmetrics-2.2.2/LICENSE)
   licenses:
-  - license: License specified in file ($CARGO_HOME/registry/src/github.com-25cdd57fae9f0462/stringmetrics-2.2.2/LICENSE)
+  - license: License specified in file ($CARGO_HOME/registry/src/index.crates.io-1949cf8c6b5b557f/stringmetrics-2.2.2/LICENSE)
     text: |
       Copyright 2022 Trevor Gross
 

libdd-library-config/Cargo.toml

@@ -17,12 +17,15 @@ bench = false
 [dependencies]
 serde = { version = "1.0", features = ["derive"] }
 serde_yaml = "0.9.34"
+prost = "0.14.1"
 anyhow = "1.0"
 
 rand = "0.8.3"
 rmp = "0.8.14"
 rmp-serde = "1.3.0"
 
+libdd-trace-protobuf = { version = "1.0.0", path = "../libdd-trace-protobuf" }
+
 [dev-dependencies]
 tempfile = { version = "3.3" }
 serial_test = "3.2"

libdd-library-config/src/otel_process_ctx.rs

@@ -33,6 +33,9 @@ pub mod linux {
         process::{getpid, set_virtual_memory_region_name, Pid},
     };
 
+    use libdd_trace_protobuf::opentelemetry::proto::common::v1::ProcessContext;
+    use prost::Message;
+
     /// Current version of the process context format
     pub const PROCESS_CTX_VERSION: u32 = 2;
     /// Signature bytes for identifying process context mappings
@@ -368,7 +371,12 @@ pub mod linux {
     /// is Undefined Behavior, for example. We assume that a forking runtime (such as Python or
     /// Ruby) that doesn't follow with an immediate `exec` is already "taking that risk", so to
     /// speak (typically, if no thread is ever spawned before the fork, things are mostly fine).
-    pub fn publish(payload: Vec<u8>) -> anyhow::Result<()> {
+    #[inline]
+    pub fn publish(context: &ProcessContext) -> anyhow::Result<()> {
+        publish_raw_payload(context.encode_to_vec())
+    }
+
+    fn publish_raw_payload(payload: Vec<u8>) -> anyhow::Result<()> {
         let mut guard = lock_context_handle()?;
 
         match &mut *guard {
@@ -497,7 +505,7 @@ pub mod linux {
             let payload_v1 = "example process context payload";
             let payload_v2 = "another example process context payload of different size";
 
-            super::publish(payload_v1.as_bytes().to_vec())
+            super::publish_raw_payload(payload_v1.as_bytes().to_vec())
                 .expect("couldn't publish the process context");
 
             let header = read_process_context().expect("couldn't read back the process context");
@@ -522,7 +530,8 @@ pub mod linux {
             let published_at_ns_v1 = header.published_at_ns;
             // Ensure the clock advances so the updated timestamp is strictly greater
             std::thread::sleep(std::time::Duration::from_nanos(10));
-            super::publish(payload_v2.as_bytes().to_vec())
+
+            super::publish_raw_payload(payload_v2.as_bytes().to_vec())
                 .expect("couldn't update the process context");
 
             let header = read_process_context().expect("couldn't read back the process context");
@@ -555,7 +564,7 @@ pub mod linux {
         fn unpublish_process_context() {
             let payload = "example process context payload";
 
-            super::publish(payload.as_bytes().to_vec())
+            super::publish_raw_payload(payload.as_bytes().to_vec())
                 .expect("couldn't publish the process context");
 
             // The mapping must be discoverable right after publishing

libdd-trace-protobuf/build.rs

@@ -261,6 +261,7 @@ fn generate_protobuf() {
                 "src/pb/span.proto",
                 "src/pb/stats.proto",
                 "src/pb/remoteconfig.proto",
+                "src/pb/opentelemetry/proto/common/v1/process_context.proto",
                 "src/pb/idx/tracer_payload.proto",
                 "src/pb/idx/span.proto",
             ],
@@ -288,6 +289,23 @@ fn generate_protobuf() {
     prepend_to_file(serde_uses, &output_path.join("pb.rs"));
     prepend_to_file(serde_uses, &output_path.join("remoteconfig.rs"));
     prepend_to_file(serde_uses, &output_path.join("pb.idx.rs"));
+
+    // We vendored a few OTel protobuf definitions, which requires their own copyright header,
+    // although they thankfully have the same Apache license.
+    let otel_license = "// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+"
+    .as_bytes();
+
+    prepend_to_file(
+        otel_license,
+        &output_path.join("opentelemetry.proto.resource.v1.rs"),
+    );
+    prepend_to_file(
+        otel_license,
+        &output_path.join("opentelemetry.proto.common.v1.rs"),
+    );
 }
 
 #[cfg(feature = "generate-protobuf")]

libdd-trace-protobuf/src/_includes.rs

@@ -2,6 +2,20 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // This file is @generated by prost-build.
+pub mod opentelemetry {
+    pub mod proto {
+        pub mod common {
+            pub mod v1 {
+                include!("opentelemetry.proto.common.v1.rs");
+            }
+        }
+        pub mod resource {
+            pub mod v1 {
+                include!("opentelemetry.proto.resource.v1.rs");
+            }
+        }
+    }
+}
 pub mod pb {
     include!("pb.rs");
     pub mod idx {

libdd-trace-protobuf/src/opentelemetry.proto.common.v1.rs

@@ -0,0 +1,186 @@
+// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// This file is @generated by prost-build.
+/// Represents any type of attribute value. AnyValue may contain a
+/// primitive value such as a string or integer or it may contain an arbitrary nested
+/// object containing arrays, key-value lists and primitives.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct AnyValue {
+    /// The value is one of the listed fields. It is valid for all values to be unspecified
+    /// in which case this AnyValue is considered to be "empty".
+    #[prost(oneof = "any_value::Value", tags = "1, 2, 3, 4, 5, 6, 7, 8")]
+    pub value: ::core::option::Option<any_value::Value>,
+}
+/// Nested message and enum types in `AnyValue`.
+pub mod any_value {
+    /// The value is one of the listed fields. It is valid for all values to be unspecified
+    /// in which case this AnyValue is considered to be "empty".
+    #[derive(Clone, PartialEq, ::prost::Oneof)]
+    pub enum Value {
+        #[prost(string, tag = "1")]
+        StringValue(::prost::alloc::string::String),
+        #[prost(bool, tag = "2")]
+        BoolValue(bool),
+        #[prost(int64, tag = "3")]
+        IntValue(i64),
+        #[prost(double, tag = "4")]
+        DoubleValue(f64),
+        #[prost(message, tag = "5")]
+        ArrayValue(super::ArrayValue),
+        #[prost(message, tag = "6")]
+        KvlistValue(super::KeyValueList),
+        #[prost(bytes, tag = "7")]
+        BytesValue(::prost::alloc::vec::Vec<u8>),
+        /// Reference to the string value in ProfilesDictionary.string_table.
+        ///
+        /// Note: This is currently used exclusively in the Profiling signal.
+        /// Implementers of OTLP receivers for signals other than Profiling should
+        /// treat the presence of this value as a non-fatal issue.
+        /// Log an error or warning indicating an unexpected field intended for the
+        /// Profiling signal and process the data as if this value were absent or
+        /// empty, ignoring its semantic content for the non-Profiling signal.
+        ///
+        /// Status: \[Development\]
+        #[prost(int32, tag = "8")]
+        StringValueRef(i32),
+    }
+}
+/// ArrayValue is a list of AnyValue messages. We need ArrayValue as a message
+/// since oneof in AnyValue does not allow repeated fields.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ArrayValue {
+    /// Array of values. The array may be empty (contain 0 elements).
+    #[prost(message, repeated, tag = "1")]
+    pub values: ::prost::alloc::vec::Vec<AnyValue>,
+}
+/// KeyValueList is a list of KeyValue messages. We need KeyValueList as a message
+/// since `oneof` in AnyValue does not allow repeated fields. Everywhere else where we need
+/// a list of KeyValue messages (e.g. in Span) we use `repeated KeyValue` directly to
+/// avoid unnecessary extra wrapping (which slows down the protocol). The 2 approaches
+/// are semantically equivalent.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct KeyValueList {
+    /// A collection of key/value pairs of key-value pairs. The list may be empty (may
+    /// contain 0 elements).
+    ///
+    /// The keys MUST be unique (it is not allowed to have more than one
+    /// value with the same key).
+    /// The behavior of software that receives duplicated keys can be unpredictable.
+    #[prost(message, repeated, tag = "1")]
+    pub values: ::prost::alloc::vec::Vec<KeyValue>,
+}
+/// Represents a key-value pair that is used to store Span attributes, Link
+/// attributes, etc.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct KeyValue {
+    /// The key name of the pair.
+    /// key_ref MUST NOT be set if key is used.
+    #[prost(string, tag = "1")]
+    pub key: ::prost::alloc::string::String,
+    /// The value of the pair.
+    #[prost(message, optional, tag = "2")]
+    pub value: ::core::option::Option<AnyValue>,
+    /// Reference to the string key in ProfilesDictionary.string_table.
+    /// key MUST NOT be set if key_ref is used.
+    ///
+    /// Note: This is currently used exclusively in the Profiling signal.
+    /// Implementers of OTLP receivers for signals other than Profiling should
+    /// treat the presence of this key as a non-fatal issue.
+    /// Log an error or warning indicating an unexpected field intended for the
+    /// Profiling signal and process the data as if this value were absent or
+    /// empty, ignoring its semantic content for the non-Profiling signal.
+    ///
+    /// Status: \[Development\]
+    #[prost(int32, tag = "3")]
+    pub key_ref: i32,
+}
+/// InstrumentationScope is a message representing the instrumentation scope information
+/// such as the fully qualified name and version.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct InstrumentationScope {
+    /// A name denoting the Instrumentation scope.
+    /// An empty instrumentation scope name means the name is unknown.
+    #[prost(string, tag = "1")]
+    pub name: ::prost::alloc::string::String,
+    /// Defines the version of the instrumentation scope.
+    /// An empty instrumentation scope version means the version is unknown.
+    #[prost(string, tag = "2")]
+    pub version: ::prost::alloc::string::String,
+    /// Additional attributes that describe the scope. \[Optional\].
+    /// Attribute keys MUST be unique (it is not allowed to have more than one
+    /// attribute with the same key).
+    /// The behavior of software that receives duplicated keys can be unpredictable.
+    #[prost(message, repeated, tag = "3")]
+    pub attributes: ::prost::alloc::vec::Vec<KeyValue>,
+    /// The number of attributes that were discarded. Attributes
+    /// can be discarded because their keys are too long or because there are too many
+    /// attributes. If this value is 0, then no attributes were dropped.
+    #[prost(uint32, tag = "4")]
+    pub dropped_attributes_count: u32,
+}
+/// A reference to an Entity.
+/// Entity represents an object of interest associated with produced telemetry: e.g spans, metrics, profiles, or logs.
+///
+/// Status: \[Development\]
+#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
+pub struct EntityRef {
+    /// The Schema URL, if known. This is the identifier of the Schema that the entity data
+    /// is recorded in. To learn more about Schema URL see
+    /// <https://opentelemetry.io/docs/specs/otel/schemas/#schema-url>
+    ///
+    /// This schema_url applies to the data in this message and to the Resource attributes
+    /// referenced by id_keys and description_keys.
+    /// TODO: discuss if we are happy with this somewhat complicated definition of what
+    /// the schema_url applies to.
+    ///
+    /// This field obsoletes the schema_url field in ResourceMetrics/ResourceSpans/ResourceLogs.
+    #[prost(string, tag = "1")]
+    pub schema_url: ::prost::alloc::string::String,
+    /// Defines the type of the entity. MUST not change during the lifetime of the entity.
+    /// For example: "service" or "host". This field is required and MUST not be empty
+    /// for valid entities.
+    #[prost(string, tag = "2")]
+    pub r#type: ::prost::alloc::string::String,
+    /// Attribute Keys that identify the entity.
+    /// MUST not change during the lifetime of the entity. The Id must contain at least one attribute.
+    /// These keys MUST exist in the containing {message}.attributes.
+    #[prost(string, repeated, tag = "3")]
+    pub id_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+    /// Descriptive (non-identifying) attribute keys of the entity.
+    /// MAY change over the lifetime of the entity. MAY be empty.
+    /// These attribute keys are not part of entity's identity.
+    /// These keys MUST exist in the containing {message}.attributes.
+    #[prost(string, repeated, tag = "4")]
+    pub description_keys: ::prost::alloc::vec::Vec<::prost::alloc::string::String>,
+}
+/// ProcessContext represents the payload for the process context sharing mechanism.
+///
+/// This message is designed to be published by OpenTelemetry SDKs via a memory-mapped
+/// region, allowing external readers (such as the OpenTelemetry eBPF Profiler) to
+/// discover and read resource attributes from instrumented processes without requiring
+/// direct integration or process activity.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct ProcessContext {
+    /// The resource attributes describing this process.
+    ///
+    /// Attribute keys MUST be unique (it is not allowed to have more than one
+    /// attribute with the same key). The behavior of software that receives
+    /// duplicated keys can be unpredictable.
+    ///
+    /// Attributes SHOULD follow OpenTelemetry semantic conventions where applicable.
+    /// See: <https://opentelemetry.io/docs/specs/semconv/>
+    #[prost(message, optional, tag = "1")]
+    pub resource: ::core::option::Option<super::super::resource::v1::Resource>,
+    /// Additional attributes to share with external readers that are not part of
+    /// the standard Resource. \[Optional\]
+    ///
+    /// This field allows publishers to include supplementary key-value pairs that
+    /// may be useful for external readers but are not part of the SDK's configured
+    /// Resource.
+    ///
+    /// Consider adding any keys here to the profiles semantic conventions in
+    /// <https://opentelemetry.io/docs/specs/semconv/general/profiles/>
+    #[prost(message, repeated, tag = "2")]
+    pub extra_attributes: ::prost::alloc::vec::Vec<KeyValue>,
+}

libdd-trace-protobuf/src/opentelemetry.proto.resource.v1.rs

@@ -0,0 +1,25 @@
+// Copyright 2019, OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+// This file is @generated by prost-build.
+/// Resource information.
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct Resource {
+    /// Set of attributes that describe the resource.
+    /// Attribute keys MUST be unique (it is not allowed to have more than one
+    /// attribute with the same key).
+    /// The behavior of software that receives duplicated keys can be unpredictable.
+    #[prost(message, repeated, tag = "1")]
+    pub attributes: ::prost::alloc::vec::Vec<super::super::common::v1::KeyValue>,
+    /// The number of dropped attributes. If the value is 0, then
+    /// no attributes were dropped.
+    #[prost(uint32, tag = "2")]
+    pub dropped_attributes_count: u32,
+    /// Set of entities that participate in this Resource.
+    ///
+    /// Note: keys in the references MUST exist in attributes of this message.
+    ///
+    /// Status: \[Development\]
+    #[prost(message, repeated, tag = "3")]
+    pub entity_refs: ::prost::alloc::vec::Vec<super::super::common::v1::EntityRef>,
+}