From fa7652a77b147c9ce4fb286f613d730bc30ac6ec Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Thu, 27 Mar 2025 16:01:22 -0400 Subject: [PATCH 01/12] add llmobs endpoints passthroughs --- bottlecap/src/traces/trace_agent.rs | 64 +++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 611133066..ddc15e283 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -33,6 +33,8 @@ const DSM_ENDPOINT_PATH: &str = "/api/v0.1/pipeline_stats"; const DSM_AGENT_PATH: &str = "/v0.1/pipeline_stats"; const PROFILING_ENDPOINT_PATH: &str = "/profiling/v1/input"; const PROFILING_BACKEND_PATH: &str = "/api/v2/profile"; +const LLM_OBS_EVAL_METRIC_ENDPOINT_PATH: &str = "/evp_proxy/v2/api/intake/llm-obs/v1/eval-metric"; +const LLM_OBS_SPANS_ENDPOINT_PATH: &str = "/evp_proxy/v2/api/v2/llmobs"; const DD_ADDITIONAL_TAGS_HEADER: &str = "X-Datadog-Additional-Tags"; const INFO_ENDPOINT_PATH: &str = "/info"; const TRACER_PAYLOAD_CHANNEL_BUFFER_SIZE: usize = 10; @@ -256,6 +258,28 @@ impl TraceAgent { ), } } + (&Method::POST, LLM_OBS_EVAL_METRIC_ENDPOINT_PATH) => { + match Self::handle_llm_obs_eval_metric_proxy(config, tags_provider, api_key, client, req) + .await + { + Ok(result) => Ok(result), + Err(err) => log_and_create_http_response( + &format!("LLM OBS Eval Metric endpoint error: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ), + } + } + (&Method::POST, LLM_OBS_SPANS_ENDPOINT_PATH) => { + match Self::handle_llm_obs_spans_proxy(config, tags_provider, api_key, client, req) + .await + { + Ok(result) => Ok(result), + Err(err) => log_and_create_http_response( + &format!("LLM OBS Spans endpoint error: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ), + } + } (_, INFO_ENDPOINT_PATH) => match Self::info_handler() { Ok(result) => Ok(result), Err(err) => log_and_create_http_response( @@ -480,6 +504,46 @@ impl TraceAgent { .await } + async fn handle_llm_obs_eval_metric_proxy( + config: Arc, + tags_provider: Arc, + api_key: String, + client: reqwest::Client, + req: Request, + ) -> http::Result> { + Self::handle_proxy( + config, + client, + api_key, + tags_provider, + req, + "api", + "/api/intake/llm-obs/v1/eval-metric", + "llm_obs_eval_metric", + ) + .await + } + + async fn handle_llm_obs_spans_proxy( + config: Arc, + tags_provider: Arc, + api_key: String, + client: reqwest::Client, + req: Request, + ) -> http::Result> { + Self::handle_proxy( + config, + client, + api_key, + tags_provider, + req, + "llmobs-intake", + "/api/intake/llm-obs", + "LLM OBS Spans", + ) + .await + } + #[must_use] pub fn get_sender_copy(&self) -> Sender { self.tx.clone() From bf225db998569b1b5087b08642d92d541fb0603a Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Thu, 27 Mar 2025 16:21:49 -0400 Subject: [PATCH 02/12] fmt --- bottlecap/src/traces/trace_agent.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index ddc15e283..53a7cf004 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -259,8 +259,14 @@ impl TraceAgent { } } (&Method::POST, LLM_OBS_EVAL_METRIC_ENDPOINT_PATH) => { - match Self::handle_llm_obs_eval_metric_proxy(config, tags_provider, api_key, client, req) - .await + match Self::handle_llm_obs_eval_metric_proxy( + config, + tags_provider, + api_key, + client, + req, + ) + .await { Ok(result) => Ok(result), Err(err) => log_and_create_http_response( @@ -270,8 +276,14 @@ impl TraceAgent { } } (&Method::POST, LLM_OBS_SPANS_ENDPOINT_PATH) => { - match Self::handle_llm_obs_spans_proxy(config, tags_provider, api_key, client, req) - .await + match Self::handle_llm_obs_spans_proxy( + config, + tags_provider, + api_key, + client, + req, + ) + .await { Ok(result) => Ok(result), Err(err) => log_and_create_http_response( From 933d1b7e5407f1ff538ee43623aa8ee4cae3a6f4 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Thu, 27 Mar 2025 16:23:09 -0400 Subject: [PATCH 03/12] update spans endpoint --- bottlecap/src/traces/trace_agent.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 53a7cf004..0fcc7eddd 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -550,7 +550,7 @@ impl TraceAgent { tags_provider, req, "llmobs-intake", - "/api/intake/llm-obs", + "/api/v2/llm-obs", "LLM OBS Spans", ) .await From 744deeb1ec3f6846e147c3f2ace01191139adbe3 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Thu, 27 Mar 2025 16:27:19 -0400 Subject: [PATCH 04/12] revert spans proxy fmt --- bottlecap/src/traces/trace_agent.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 0fcc7eddd..25c3f4e3f 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -276,13 +276,7 @@ impl TraceAgent { } } (&Method::POST, LLM_OBS_SPANS_ENDPOINT_PATH) => { - match Self::handle_llm_obs_spans_proxy( - config, - tags_provider, - api_key, - client, - req, - ) + match Self::handle_llm_obs_spans_proxy(config, tags_provider, api_key, client, req) .await { Ok(result) => Ok(result), From f01c9eea787de5ac818dbe40b28261f2e5fddb3b Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Thu, 27 Mar 2025 16:32:38 -0400 Subject: [PATCH 05/12] fmt --- bottlecap/src/traces/trace_agent.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 25c3f4e3f..05170c9fe 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -277,7 +277,7 @@ impl TraceAgent { } (&Method::POST, LLM_OBS_SPANS_ENDPOINT_PATH) => { match Self::handle_llm_obs_spans_proxy(config, tags_provider, api_key, client, req) - .await + .await { Ok(result) => Ok(result), Err(err) => log_and_create_http_response( From 022c456f640510f5fc710b3a55102ce9de05ab96 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Thu, 27 Mar 2025 16:41:13 -0400 Subject: [PATCH 06/12] allow pedantic --- bottlecap/src/traces/trace_agent.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 05170c9fe..0bfe6f479 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -182,6 +182,7 @@ impl TraceAgent { } #[allow(clippy::too_many_arguments)] + #[allow(clippy::pedantic)] async fn trace_endpoint_handler( config: Arc, req: Request, From ffc7130adf78ca3fdada176c1f6721ea66bfcfb2 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 28 Mar 2025 09:59:22 -0400 Subject: [PATCH 07/12] add llmobs endpoints to info path --- bottlecap/src/traces/trace_agent.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 0bfe6f479..23834eb08 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -375,7 +375,9 @@ impl TraceAgent { STATS_ENDPOINT_PATH, DSM_AGENT_PATH, PROFILING_ENDPOINT_PATH, - INFO_ENDPOINT_PATH + INFO_ENDPOINT_PATH, + LLM_OBS_EVAL_METRIC_ENDPOINT_PATH, + LLM_OBS_SPANS_ENDPOINT_PATH, ], "client_drop_p0s": true, } From a914aa6eb04aa28db59bf0f1af059e2bfa302680 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 28 Mar 2025 11:56:17 -0400 Subject: [PATCH 08/12] add some temp debug logs --- bottlecap/src/traces/trace_agent.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 23834eb08..0bd7ed647 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -416,6 +416,8 @@ impl TraceAgent { let target_url = format!("https://{}.{}{}", backend_domain, config.site, backend_path); let mut request_builder = client.post(&target_url); + debug!("Sending request to {target_url}"); + for (name, value) in &parts.headers { if name.as_str().to_lowercase() != "host" && name.as_str().to_lowercase() != "content-length" @@ -450,6 +452,8 @@ impl TraceAgent { let status = StatusCode::from_u16(response.status().as_u16()) .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); + debug!("Got response from {target_url} backend: {status}"); + let mut builder = Response::builder().status(status); for (name, value) in response.headers() { @@ -540,6 +544,7 @@ impl TraceAgent { client: reqwest::Client, req: Request, ) -> http::Result> { + debug!("Sending request to llmobs-intake"); Self::handle_proxy( config, client, @@ -548,7 +553,7 @@ impl TraceAgent { req, "llmobs-intake", "/api/v2/llm-obs", - "LLM OBS Spans", + "llm_obs_spans", ) .await } From 08ebe377b105c5273d260a40a9bc07ab65e3265f Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 28 Mar 2025 15:13:38 -0400 Subject: [PATCH 09/12] add some more debugging --- bottlecap/src/traces/trace_agent.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 0bd7ed647..6ef68cd46 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -417,6 +417,16 @@ impl TraceAgent { let mut request_builder = client.post(&target_url); debug!("Sending request to {target_url}"); + debug!("Request headers: {:#?}", parts.headers); + if let Ok(json_str) = String::from_utf8(body_bytes.to_vec()) { + if let Ok(json_value) = serde_json::from_str::(&json_str) { + debug!("Request body JSON: {:#?}", json_value); + } else { + debug!("Request body: {:#?}", body_bytes); + } + } else { + debug!("Request body: {:#?}", body_bytes); + } for (name, value) in &parts.headers { if name.as_str().to_lowercase() != "host" From 34ad973f3bb695df8d84b7a7c91b73aeb7a57942 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 28 Mar 2025 15:42:37 -0400 Subject: [PATCH 10/12] remove debug logs, fix typo --- bottlecap/src/traces/trace_agent.rs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 6ef68cd46..4c473a520 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -416,18 +416,6 @@ impl TraceAgent { let target_url = format!("https://{}.{}{}", backend_domain, config.site, backend_path); let mut request_builder = client.post(&target_url); - debug!("Sending request to {target_url}"); - debug!("Request headers: {:#?}", parts.headers); - if let Ok(json_str) = String::from_utf8(body_bytes.to_vec()) { - if let Ok(json_value) = serde_json::from_str::(&json_str) { - debug!("Request body JSON: {:#?}", json_value); - } else { - debug!("Request body: {:#?}", body_bytes); - } - } else { - debug!("Request body: {:#?}", body_bytes); - } - for (name, value) in &parts.headers { if name.as_str().to_lowercase() != "host" && name.as_str().to_lowercase() != "content-length" @@ -462,8 +450,6 @@ impl TraceAgent { let status = StatusCode::from_u16(response.status().as_u16()) .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR); - debug!("Got response from {target_url} backend: {status}"); - let mut builder = Response::builder().status(status); for (name, value) in response.headers() { @@ -554,7 +540,6 @@ impl TraceAgent { client: reqwest::Client, req: Request, ) -> http::Result> { - debug!("Sending request to llmobs-intake"); Self::handle_proxy( config, client, @@ -562,7 +547,7 @@ impl TraceAgent { tags_provider, req, "llmobs-intake", - "/api/v2/llm-obs", + "/api/v2/llmobs", "llm_obs_spans", ) .await From d723a853012db217d846a8b85f5d5c6c59eea929 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Mon, 31 Mar 2025 13:43:09 -0400 Subject: [PATCH 11/12] add eval metric v2 path and llmobs intake consts --- bottlecap/src/traces/trace_agent.rs | 47 +++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index 4c473a520..aa2ef84a7 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -33,7 +33,11 @@ const DSM_ENDPOINT_PATH: &str = "/api/v0.1/pipeline_stats"; const DSM_AGENT_PATH: &str = "/v0.1/pipeline_stats"; const PROFILING_ENDPOINT_PATH: &str = "/profiling/v1/input"; const PROFILING_BACKEND_PATH: &str = "/api/v2/profile"; +const LLM_OBS_SPANS_INTAKE_PATH: &str = "/api/v2/llmobs"; +const LLM_OBS_EVAL_METRIC_INTAKE_PATH: &str = "/api/intake/llm-obs/v1/eval-metric"; +const LLM_OBS_EVAL_METRIC_INTAKE_PATH_V2: &str = "/api/intake/llm-obs/v2/eval-metric"; const LLM_OBS_EVAL_METRIC_ENDPOINT_PATH: &str = "/evp_proxy/v2/api/intake/llm-obs/v1/eval-metric"; +const LLM_OBS_EVAL_METRIC_ENDPOINT_PATH_V2: &str = "/evp_proxy/v2/api/intake/llm-obs/v2/eval-metric"; const LLM_OBS_SPANS_ENDPOINT_PATH: &str = "/evp_proxy/v2/api/v2/llmobs"; const DD_ADDITIONAL_TAGS_HEADER: &str = "X-Datadog-Additional-Tags"; const INFO_ENDPOINT_PATH: &str = "/info"; @@ -276,6 +280,23 @@ impl TraceAgent { ), } } + (&Method::POST, LLM_OBS_EVAL_METRIC_ENDPOINT_PATH_V2) => { + match Self::handle_llm_obs_eval_metric_proxy_v2( + config, + tags_provider, + api_key, + client, + req, + ) + .await + { + Ok(result) => Ok(result), + Err(err) => log_and_create_http_response( + &format!("LLM OBS Eval Metric endpoint error: {err}"), + StatusCode::INTERNAL_SERVER_ERROR, + ), + } + } (&Method::POST, LLM_OBS_SPANS_ENDPOINT_PATH) => { match Self::handle_llm_obs_spans_proxy(config, tags_provider, api_key, client, req) .await @@ -377,6 +398,7 @@ impl TraceAgent { PROFILING_ENDPOINT_PATH, INFO_ENDPOINT_PATH, LLM_OBS_EVAL_METRIC_ENDPOINT_PATH, + LLM_OBS_EVAL_METRIC_ENDPOINT_PATH_V2, LLM_OBS_SPANS_ENDPOINT_PATH, ], "client_drop_p0s": true, @@ -527,11 +549,32 @@ impl TraceAgent { tags_provider, req, "api", - "/api/intake/llm-obs/v1/eval-metric", + LLM_OBS_EVAL_METRIC_INTAKE_PATH, + "llm_obs_eval_metric", + ) + .await + } + + async fn handle_llm_obs_eval_metric_proxy_v2( + config: Arc, + tags_provider: Arc, + api_key: String, + client: reqwest::Client, + req: Request, + ) -> http::Result> { + Self::handle_proxy( + config, + client, + api_key, + tags_provider, + req, + "api", + LLM_OBS_EVAL_METRIC_INTAKE_PATH_V2, "llm_obs_eval_metric", ) .await } + async fn handle_llm_obs_spans_proxy( config: Arc, @@ -547,7 +590,7 @@ impl TraceAgent { tags_provider, req, "llmobs-intake", - "/api/v2/llmobs", + LLM_OBS_SPANS_INTAKE_PATH, "llm_obs_spans", ) .await From 825ec8bd80a64d82b8a0b22946919a9f25be7ab0 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Mon, 31 Mar 2025 13:44:54 -0400 Subject: [PATCH 12/12] fmt --- bottlecap/src/traces/trace_agent.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bottlecap/src/traces/trace_agent.rs b/bottlecap/src/traces/trace_agent.rs index aa2ef84a7..cf86a9e70 100644 --- a/bottlecap/src/traces/trace_agent.rs +++ b/bottlecap/src/traces/trace_agent.rs @@ -37,7 +37,8 @@ const LLM_OBS_SPANS_INTAKE_PATH: &str = "/api/v2/llmobs"; const LLM_OBS_EVAL_METRIC_INTAKE_PATH: &str = "/api/intake/llm-obs/v1/eval-metric"; const LLM_OBS_EVAL_METRIC_INTAKE_PATH_V2: &str = "/api/intake/llm-obs/v2/eval-metric"; const LLM_OBS_EVAL_METRIC_ENDPOINT_PATH: &str = "/evp_proxy/v2/api/intake/llm-obs/v1/eval-metric"; -const LLM_OBS_EVAL_METRIC_ENDPOINT_PATH_V2: &str = "/evp_proxy/v2/api/intake/llm-obs/v2/eval-metric"; +const LLM_OBS_EVAL_METRIC_ENDPOINT_PATH_V2: &str = + "/evp_proxy/v2/api/intake/llm-obs/v2/eval-metric"; const LLM_OBS_SPANS_ENDPOINT_PATH: &str = "/evp_proxy/v2/api/v2/llmobs"; const DD_ADDITIONAL_TAGS_HEADER: &str = "X-Datadog-Additional-Tags"; const INFO_ENDPOINT_PATH: &str = "/info"; @@ -574,7 +575,6 @@ impl TraceAgent { ) .await } - async fn handle_llm_obs_spans_proxy( config: Arc,