feat(appsec): add endpoints collection

datadog-sidecar-ffi/Cargo.toml

@@ -27,6 +27,7 @@ paste = "1"
 libc = "0.2"
 tracing = { version = "0.1", default-features = false }
 rmp-serde = "1.1.1"
+serde_json = "1.0"
 
 
 [target.'cfg(windows)'.dependencies]

datadog-sidecar-ffi/src/lib.rs

@@ -402,6 +402,35 @@ pub unsafe extern "C" fn ddog_sidecar_telemetry_enqueueConfig(
     MaybeError::None
 }
 
+/// Reports an endpoint to the telemetry.
+#[no_mangle]
+#[allow(clippy::missing_safety_doc)]
+pub unsafe extern "C" fn ddog_sidecar_telemetry_addEndpoint(
+    transport: &mut Box<SidecarTransport>,
+    instance_id: &InstanceId,
+    queue_id: &QueueId,
+    method: libdd_telemetry::data::Method,
+    path: CharSlice,
+    operation_name: CharSlice,
+    resource_name: CharSlice,
+) -> MaybeError {
+    let endpoint = TelemetryActions::AddEndpoint(libdd_telemetry::data::Endpoint {
+        method: Some(method),
+        path: Some(path.to_utf8_lossy().into_owned()),
+        operation_name: operation_name.to_utf8_lossy().into_owned(),
+        resource_name: resource_name.to_utf8_lossy().into_owned(),
+    });
+
+    try_c!(blocking::enqueue_actions(
+        transport,
+        instance_id,
+        queue_id,
+        vec![SidecarAction::Telemetry(endpoint)],
+    ));
+
+    MaybeError::None
+}
+
 /// Reports a dependency to the telemetry.
 #[no_mangle]
 #[allow(clippy::missing_safety_doc)]

datadog-sidecar/src/service/sidecar_server.rs

@@ -29,7 +29,7 @@ use std::collections::{HashMap, HashSet};
 use std::pin::Pin;
 use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
-use std::time::Duration;
+use std::time::{Duration, SystemTime};
 use tracing::{debug, error, info, trace, warn};
 
 use futures::FutureExt;
@@ -441,7 +441,7 @@ impl SidecarInterface for SidecarServer {
             );
             let mut telemetry = telemetry_mutex.lock_or_panic();
 
-            let mut actions_to_process = vec![];
+            let mut actions_to_process: Vec<SidecarAction> = vec![];
             let mut composer_paths_to_process = vec![];
             let mut buffered_info_changed = false;
             let mut remove_entry = false;
@@ -469,6 +469,11 @@ impl SidecarInterface for SidecarServer {
                     SidecarAction::ClearQueueId => {
                         remove_entry = true;
                     }
+                    SidecarAction::Telemetry(TelemetryActions::AddEndpoint(_)) => {
+                        telemetry.last_endpoints_push = SystemTime::now();
+                        buffered_info_changed = true;
+                        actions_to_process.push(action);
+                    }
                     SidecarAction::Telemetry(TelemetryActions::Lifecycle(
                         LifecycleAction::Stop,
                     )) => {

datadog-sidecar/src/service/telemetry.rs

@@ -65,6 +65,7 @@ pub struct TelemetryCachedClient {
     pub config_sent: bool,
     pub buffered_integrations: HashSet<Integration>,
     pub buffered_composer_paths: HashSet<PathBuf>,
+    pub last_endpoints_push: SystemTime,
     pub telemetry_metrics: HashMap<String, ContextKey>,
     pub handle: Option<JoinHandle<()>>,
 }
@@ -101,6 +102,7 @@ impl TelemetryCachedClient {
             config_sent: false,
             buffered_integrations: HashSet::new(),
             buffered_composer_paths: HashSet::new(),
+            last_endpoints_push: SystemTime::UNIX_EPOCH,
             telemetry_metrics: Default::default(),
             handle: None,
         }
@@ -111,6 +113,7 @@ impl TelemetryCachedClient {
             &self.config_sent,
             &self.buffered_integrations,
             &self.buffered_composer_paths,
+            &self.last_endpoints_push,
         )) {
             self.shm_writer.write(&buf);
         } else {

examples/cxx/build-and-run-crashinfo.ps1

@@ -7,7 +7,7 @@ $SCRIPT_DIR = Split-Path -Parent $MyInvocation.MyCommand.Path
 & "$SCRIPT_DIR\build-and-run.ps1" `
     -CrateName "libdd-crashtracker" `
     -ExampleName "crashinfo" `
-    -ExtraMsvcLibs "dbghelp.lib psapi.lib powrprof.lib" `
+    -ExtraMsvcLibs "dbghelp.lib psapi.lib powrprof.lib user32.lib oleaut32.lib secur32.lib ncrypt.lib runtimeobject.lib" `
     -ExtraGnuLibs "-ldbghelp -lpsapi -lole32 -lpowrprof"
 
 exit $LASTEXITCODE

libdd-telemetry/src/data/payload.rs

@@ -12,6 +12,7 @@ pub enum Payload {
     AppDependenciesLoaded(AppDependenciesLoaded),
     AppIntegrationsChange(AppIntegrationsChange),
     AppClientConfigurationChange(AppClientConfigurationChange),
+    AppEndpoints(AppEndpoints),
     AppHeartbeat(#[serde(skip_serializing)] ()),
     AppClosing(#[serde(skip_serializing)] ()),
     GenerateMetrics(GenerateMetrics),
@@ -29,6 +30,7 @@ impl Payload {
             AppDependenciesLoaded(_) => "app-dependencies-loaded",
             AppIntegrationsChange(_) => "app-integrations-change",
             AppClientConfigurationChange(_) => "app-client-configuration-change",
+            AppEndpoints(_) => "app-endpoints",
             AppHeartbeat(_) => "app-heartbeat",
             AppClosing(_) => "app-closing",
             GenerateMetrics(_) => "generate-metrics",

libdd-telemetry/src/data/payloads.rs

@@ -1,6 +1,8 @@
 // Copyright 2021-Present Datadog, Inc. https://www.datadoghq.com/
 // SPDX-License-Identifier: Apache-2.0
 
+use std::hash::Hasher;
+
 use crate::data::metrics;
 
 use serde::{Deserialize, Serialize};
@@ -63,6 +65,12 @@ pub struct AppClientConfigurationChange {
     pub configuration: Vec<Configuration>,
 }
 
+#[derive(Debug, Serialize)]
+pub struct AppEndpoints {
+    pub is_first: bool,
+    pub endpoints: Vec<serde_json::Value>,
+}
+
 #[derive(Serialize, Debug)]
 pub struct GenerateMetrics {
     pub series: Vec<metrics::Serie>,
@@ -97,3 +105,49 @@ pub enum LogLevel {
     Warn,
     Debug,
 }
+
+#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, Hash, Clone)]
+#[serde(rename_all = "UPPERCASE")]
+#[repr(C)]
+pub enum Method {
+    Get = 0,
+    Post = 1,
+    Put = 2,
+    Delete = 3,
+    Patch = 4,
+    Head = 5,
+    Options = 6,
+    Trace = 7,
+    Connect = 8,
+    Other = 9, //This is specified as "*" in the OpenAPI spec
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Default)]
+pub struct Endpoint {
+    #[serde(default)]
+    pub method: Option<Method>,
+    #[serde(default)]
+    pub path: Option<String>,
+    pub operation_name: String,
+    pub resource_name: String,
+}
+
+impl PartialEq for Endpoint {
+    fn eq(&self, other: &Self) -> bool {
+        self.resource_name == other.resource_name
+    }
+}
+
+impl Eq for Endpoint {}
+
+impl std::hash::Hash for Endpoint {
+    fn hash<H: Hasher>(&self, state: &mut H) {
+        self.resource_name.hash(state);
+    }
+}
+
+impl Endpoint {
+    pub fn to_json_value(&self) -> serde_json::Result<serde_json::Value> {
+        serde_json::to_value(self)
+    }
+}

libdd-telemetry/src/worker/mod.rs

@@ -7,10 +7,10 @@ pub mod store;
 
 use crate::{
     config::Config,
-    data::{self, Application, Dependency, Host, Integration, Log, Payload, Telemetry},
+    data::{self, Application, Dependency, Endpoint, Host, Integration, Log, Payload, Telemetry},
     metrics::{ContextKey, MetricBuckets, MetricContexts},
 };
-use libdd_common::Endpoint;
+
 use libdd_common::{hyper_migration, tag::Tag, worker::Worker};
 
 use std::iter::Sum;
@@ -25,7 +25,7 @@ use std::{
     },
     time,
 };
-use std::{fmt::Debug, time::Duration};
+use std::{collections::HashSet, fmt::Debug, time::Duration};
 
 use crate::metrics::MetricBucketStats;
 use futures::{
@@ -89,6 +89,7 @@ pub enum TelemetryActions {
     AddDependency(Dependency),
     AddIntegration(Integration),
     AddLog((LogIdentifier, Log)),
+    AddEndpoint(Endpoint),
     Lifecycle(LifecycleAction),
     #[serde(skip)]
     CollectStats(oneshot::Sender<TelemetryWorkerStats>),
@@ -120,6 +121,7 @@ struct TelemetryWorkerData {
     dependencies: store::Store<Dependency>,
     configurations: store::Store<data::Configuration>,
     integrations: store::Store<data::Integration>,
+    endpoints: HashSet<data::Endpoint>,
     logs: store::QueueHashMap<LogIdentifier, Log>,
     metric_contexts: MetricContexts,
     metric_buckets: MetricBuckets,
@@ -352,7 +354,11 @@ impl TelemetryWorker {
                     }
                 }
             }
-            AddConfig(_) | AddDependency(_) | AddIntegration(_) | Lifecycle(ExtendedHeartbeat) => {}
+            AddConfig(_)
+            | AddDependency(_)
+            | AddIntegration(_)
+            | AddEndpoint(_)
+            | Lifecycle(ExtendedHeartbeat) => {}
             Lifecycle(Stop) => {
                 if !self.data.started {
                     return BREAK;
@@ -409,6 +415,9 @@ impl TelemetryWorker {
             AddDependency(dep) => self.data.dependencies.insert(dep),
             AddIntegration(integration) => self.data.integrations.insert(integration),
             AddConfig(cfg) => self.data.configurations.insert(cfg),
+            AddEndpoint(endpoint) => {
+                self.data.endpoints.insert(endpoint);
+            }
             AddLog((identifier, log)) => {
                 let (l, new) = self.data.logs.get_mut_or_insert(identifier, log);
                 if !new {
@@ -553,6 +562,18 @@ impl TelemetryWorker {
                 },
             ))
         }
+        if !self.data.endpoints.is_empty() {
+            payloads.push(data::Payload::AppEndpoints(data::AppEndpoints {
+                is_first: true,
+                endpoints: self
+                    .data
+                    .endpoints
+                    .iter()
+                    .map(|e| e.to_json_value().unwrap_or_default())
+                    .filter(|e| e.is_object())
+                    .collect(),
+            }));
+        }
         payloads
     }
 
@@ -655,6 +676,7 @@ impl TelemetryWorker {
                 .data
                 .configurations
                 .removed_flushed(p.configuration.len()),
+            AppEndpoints(_) => self.data.endpoints.clear(),
             MessageBatch(batch) => {
                 for p in batch {
                     self.payload_sent_success(p);
@@ -758,7 +780,7 @@ impl TelemetryWorker {
         let timeout_ms = if let Some(endpoint) = self.config.endpoint.as_ref() {
             endpoint.timeout_ms
         } else {
-            Endpoint::DEFAULT_TIMEOUT
+            libdd_common::Endpoint::DEFAULT_TIMEOUT
         };
 
         debug!(
@@ -1017,6 +1039,7 @@ pub struct TelemetryWorkerBuilder {
     pub dependencies: store::Store<data::Dependency>,
     pub integrations: store::Store<data::Integration>,
     pub configurations: store::Store<data::Configuration>,
+    pub endpoints: HashSet<data::Endpoint>,
     pub native_deps: bool,
     pub rust_shared_lib_deps: bool,
     pub config: Config,
@@ -1067,6 +1090,7 @@ impl TelemetryWorkerBuilder {
             dependencies: store::Store::new(MAX_ITEMS),
             integrations: store::Store::new(MAX_ITEMS),
             configurations: store::Store::new(MAX_ITEMS),
+            endpoints: HashSet::new(),
             native_deps: true,
             rust_shared_lib_deps: false,
             config: Config::default(),
@@ -1100,6 +1124,7 @@ impl TelemetryWorkerBuilder {
                 dependencies: self.dependencies,
                 integrations: self.integrations,
                 configurations: self.configurations,
+                endpoints: self.endpoints,
                 logs: store::QueueHashMap::default(),
                 metric_contexts: contexts.clone(),
                 metric_buckets: MetricBuckets::default(),