From 65e60ec2c79250181fdd5f339e4ffbcf4947bc48 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 Aug 2025 09:55:42 -0400 Subject: [PATCH 01/49] wip --- src/sources/opentelemetry/grpc.rs | 40 +++++++++---- src/sources/opentelemetry/http.rs | 59 ++++++++++++------- .../opentelemetry/integration_tests.rs | 3 + src/sources/opentelemetry/mod.rs | 51 +++++++++++----- src/sources/opentelemetry/tests.rs | 2 + src/sources/util/http/prelude.rs | 2 +- .../e2e/opentelemetry/logs/output/.gitignore | 0 7 files changed, 108 insertions(+), 49 deletions(-) mode change 100644 => 100755 tests/data/e2e/opentelemetry/logs/output/.gitignore diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 32a4d766c17a2..297245d4c6801 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,5 +1,14 @@ +use crate::{ + internal_events::{EventsReceived, StreamClosedError}, + sources::opentelemetry::{LOGS, METRICS, TRACES}, + SourceSender, +}; use futures::TryFutureExt; +use prost::Message; +use std::path::PathBuf; use tonic::{Request, Response, Status}; +use vector_lib::codecs::decoding::format::Deserializer; +use vector_lib::codecs::decoding::{ProtobufDeserializerConfig, ProtobufDeserializerOptions}; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ @@ -19,12 +28,6 @@ use vector_lib::{ EstimatedJsonEncodedSizeOf, }; -use crate::{ - internal_events::{EventsReceived, StreamClosedError}, - sources::opentelemetry::{LOGS, METRICS, TRACES}, - SourceSender, -}; - #[derive(Clone)] pub(super) struct Service { pub pipeline: SourceSender, @@ -59,12 +62,25 @@ impl LogsService for Service { &self, request: Request, ) -> Result, Status> { - let events: Vec = request - .into_inner() - .resource_logs - .into_iter() - .flat_map(|v| v.into_event_iter(self.log_namespace)) - .collect(); + println!("{request:#?}"); + let raw_bytes = request.get_ref().encode_to_vec(); + println!("{raw_bytes:?}"); + + let bytes = bytes::Bytes::from(raw_bytes); + let deserializer = ProtobufDeserializerConfig { + protobuf: ProtobufDeserializerOptions { + desc_file: PathBuf::from("/Users/pavlos.rontidis/CLionProjects/vector/pront/otel/proto/vector-proto-related/otlp.desc"), + message_type: "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest".to_string(), + } + }.build().unwrap(); + let events = deserializer.parse(bytes, self.log_namespace).unwrap().into_vec(); + + // let events: Vec = request + // .into_inner() + // .resource_logs + // .into_iter() + // .flat_map(|v| v.into_event_iter(self.log_namespace)) + // .collect(); self.handle_events(events, LOGS).await?; Ok(Response::new(ExportLogsServiceResponse { diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index d3c3798dd14d6..1f415bea50630 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -1,13 +1,30 @@ use std::time::Duration; use std::{convert::Infallible, net::SocketAddr}; -use bytes::Bytes; +use super::OpentelemetryConfig; +use super::{reply::protobuf, status::Status}; +use crate::codecs::Decoder as VectorDecoder; +use crate::common::http::ErrorMessage; +use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; +use crate::sources::http_server::HttpConfigParamKind; +use crate::sources::util::add_headers; +use crate::{ + event::Event, + http::build_http_trace_layer, + internal_events::{EventsReceived, StreamClosedError}, + shutdown::ShutdownSignal, + sources::util::decode, + tls::MaybeTlsSettings, + SourceSender, +}; +use bytes::{Bytes, BytesMut}; use futures_util::FutureExt; use http::StatusCode; use hyper::{service::make_service_fn, Server}; use prost::Message; use snafu::Snafu; use tokio::net::TcpStream; +use tokio_util::codec::Decoder; use tower::ServiceBuilder; use tracing::Span; use vector_lib::internal_event::{ @@ -28,23 +45,6 @@ use warp::{ filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, Filter, Reply, }; -use crate::common::http::ErrorMessage; -use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; -use crate::sources::http_server::HttpConfigParamKind; -use crate::sources::util::add_headers; -use crate::{ - event::Event, - http::build_http_trace_layer, - internal_events::{EventsReceived, StreamClosedError}, - shutdown::ShutdownSignal, - sources::util::decode, - tls::MaybeTlsSettings, - SourceSender, -}; - -use super::OpentelemetryConfig; -use super::{reply::protobuf, status::Status}; - #[derive(Clone, Copy, Debug, Snafu)] pub(crate) enum ApiError { ServerShutdown, @@ -94,6 +94,7 @@ pub(crate) fn build_warp_filter( bytes_received: Registered, events_received: Registered, headers: Vec, + decoder: Option ) -> BoxedFilter<(Response,)> { let log_filters = build_warp_log_filter( acknowledgements, @@ -102,6 +103,7 @@ pub(crate) fn build_warp_filter( bytes_received.clone(), events_received.clone(), headers.clone(), + decoder ); let metrics_filters = build_warp_metrics_filter( acknowledgements, @@ -145,6 +147,7 @@ fn build_warp_log_filter( bytes_received: Registered, events_received: Registered, headers: Vec, + decoder: Option ) -> BoxedFilter<(Response,)> { warp::post() .and(warp::path!("v1" / "logs")) @@ -157,7 +160,21 @@ fn build_warp_log_filter( .and(warp::body::bytes()) .and_then( move |encoding_header: Option, headers_config: HeaderMap, body: Bytes| { - let events = decode(encoding_header.as_deref(), body) + println!("📦 Raw bytes (len={}): {:02x?}", body.len(), body); + let events = if let Some(mut decoder) = decoder.clone() + { + let mut body = BytesMut::from(body); + decoder.decode(&mut body).map(|result| + { + if let Some(result) = result { + result.0.into_vec() + } else { + Vec::new() + } + } + ).map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) + } else { + decode(encoding_header.as_deref(), body) .and_then(|body| { bytes_received.emit(ByteSize(body.len())); decode_log_body(body, log_namespace, &events_received) @@ -165,8 +182,10 @@ fn build_warp_log_filter( .map(|mut events| { enrich_events(&mut events, &headers, &headers_config, log_namespace); events - }); + }) + }; + println!("{:#?}", events); handle_request( events, acknowledgements, diff --git a/src/sources/opentelemetry/integration_tests.rs b/src/sources/opentelemetry/integration_tests.rs index feb7717460884..c5997db4a6eb7 100644 --- a/src/sources/opentelemetry/integration_tests.rs +++ b/src/sources/opentelemetry/integration_tests.rs @@ -61,6 +61,7 @@ async fn receive_logs_legacy_namespace() { }, acknowledgements: Default::default(), log_namespace: Default::default(), + decoding: None, }; let (sender, logs_output, _) = new_source(EventStatus::Delivered, LOGS.to_string()); @@ -159,6 +160,7 @@ async fn receive_trace() { }, acknowledgements: Default::default(), log_namespace: Default::default(), + decoding: None, }; let (sender, trace_output, _) = new_source(EventStatus::Delivered, TRACES.to_string()); @@ -263,6 +265,7 @@ async fn receive_metric() { }, acknowledgements: Default::default(), log_namespace: Default::default(), + decoding: None, }; let (sender, metrics_output, _) = new_source(EventStatus::Delivered, METRICS.to_string()); diff --git a/src/sources/opentelemetry/mod.rs b/src/sources/opentelemetry/mod.rs index 8f79673d86186..3a8a2bf5def4c 100644 --- a/src/sources/opentelemetry/mod.rs +++ b/src/sources/opentelemetry/mod.rs @@ -18,24 +18,12 @@ use vector_lib::opentelemetry::logs::{ SEVERITY_NUMBER_KEY, SEVERITY_TEXT_KEY, SPAN_ID_KEY, TRACE_ID_KEY, }; -use tonic::transport::server::RoutesBuilder; -use vector_lib::configurable::configurable_component; -use vector_lib::internal_event::{BytesReceived, EventsReceived, Protocol}; -use vector_lib::opentelemetry::proto::collector::{ - logs::v1::logs_service_server::LogsServiceServer, - metrics::v1::metrics_service_server::MetricsServiceServer, - trace::v1::trace_service_server::TraceServiceServer, -}; -use vector_lib::{ - config::{log_schema, LegacyKey, LogNamespace}, - schema::Definition, -}; -use vrl::value::{kind::Collection, Kind}; - use self::{ grpc::Service, http::{build_warp_filter, run_http_server}, }; +use super::http_server::{build_param_matcher, remove_duplicates}; +use crate::codecs::DecodingConfig; use crate::{ config::{ DataType, GenerateConfig, Resource, SourceAcknowledgementsConfig, SourceConfig, @@ -46,8 +34,20 @@ use crate::{ sources::{util::grpc::run_grpc_server_with_routes, Source}, tls::{MaybeTlsSettings, TlsEnableableConfig}, }; - -use super::http_server::{build_param_matcher, remove_duplicates}; +use tonic::transport::server::RoutesBuilder; +use vector_lib::codecs::decoding::{DeserializerConfig, FramingConfig}; +use vector_lib::configurable::configurable_component; +use vector_lib::internal_event::{BytesReceived, EventsReceived, Protocol}; +use vector_lib::opentelemetry::proto::collector::{ + logs::v1::logs_service_server::LogsServiceServer, + metrics::v1::metrics_service_server::MetricsServiceServer, + trace::v1::trace_service_server::TraceServiceServer, +}; +use vector_lib::{ + config::{log_schema, LegacyKey, LogNamespace}, + schema::Definition, +}; +use vrl::value::{kind::Collection, Kind}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -72,6 +72,10 @@ pub struct OpentelemetryConfig { #[configurable(metadata(docs::hidden))] #[serde(default)] log_namespace: Option, + + #[configurable(derived)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decoding: Option, } /// Configuration for the `opentelemetry` gRPC server. @@ -149,6 +153,7 @@ impl GenerateConfig for OpentelemetryConfig { http: example_http_config(), acknowledgements: Default::default(), log_namespace: None, + decoding: None, }) .unwrap() } @@ -196,6 +201,16 @@ impl SourceConfig for OpentelemetryConfig { .add_service(log_service) .add_service(metrics_service) .add_service(trace_service); + + let decoder = if let Some(deserializer_config) = &self.decoding { + let decoding_config = DecodingConfig::new(FramingConfig::NewlineDelimited(Default::default()), + deserializer_config.clone(), + log_namespace); + Some(decoding_config.build()?) + } else { + None + }; + let grpc_source = run_grpc_server_with_routes( self.grpc.address, grpc_tls_settings, @@ -206,11 +221,13 @@ impl SourceConfig for OpentelemetryConfig { error!(message = "Source future failed.", %error); }); + let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; let protocol = http_tls_settings.http_protocol_name(); let bytes_received = register!(BytesReceived::from(Protocol::from(protocol))); let headers = build_param_matcher(&remove_duplicates(self.http.headers.clone(), "headers"))?; + let filters = build_warp_filter( acknowledgements, log_namespace, @@ -218,7 +235,9 @@ impl SourceConfig for OpentelemetryConfig { bytes_received, events_received, headers, + decoder, ); + let http_source = run_http_server( self.http.address, http_tls_settings, diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index a0fa9868acc58..dea7596df5160 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -1132,6 +1132,7 @@ async fn http_headers() { }, acknowledgements: Default::default(), log_namespace: Default::default(), + decoding: None, }; let schema_definitions = source .outputs(LogNamespace::Legacy) @@ -1237,6 +1238,7 @@ pub async fn build_otlp_test_env( }, acknowledgements: Default::default(), log_namespace, + decoding: None, }; let (sender, output, _) = new_source(EventStatus::Delivered, event_name.to_string()); diff --git a/src/sources/util/http/prelude.rs b/src/sources/util/http/prelude.rs index 9fc83559fe887..eefee840f2f96 100644 --- a/src/sources/util/http/prelude.rs +++ b/src/sources/util/http/prelude.rs @@ -129,7 +129,7 @@ pub trait HttpSource: Clone + Send + Sync + 'static { addr: Option| { debug!(message = "Handling HTTP request.", headers = ?headers); let http_path = path.as_str(); - + println!("📦 Raw bytes (len={}): {:02x?}", body.len(), body); let events = auth_matcher .as_ref() .map_or(Ok(()), |a| { diff --git a/tests/data/e2e/opentelemetry/logs/output/.gitignore b/tests/data/e2e/opentelemetry/logs/output/.gitignore old mode 100644 new mode 100755 From 8621a68b5afc25d55259f7df2cd5f236e308a9d9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 Aug 2025 12:48:41 -0400 Subject: [PATCH 02/49] WIP2 --- src/sinks/http/encoder.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sinks/http/encoder.rs b/src/sinks/http/encoder.rs index a48c1c19151f2..bf75d8065b32f 100644 --- a/src/sinks/http/encoder.rs +++ b/src/sinks/http/encoder.rs @@ -92,6 +92,9 @@ impl SinkEncoder> for HttpEncoder { let body = body.freeze(); + println!("📦 Final encoded HTTP payload ({} bytes):", body.len()); + println!("{:?}", body); + write_all(writer, n_events, body.as_ref()).map(|()| (body.len(), byte_size)) } } From 83c585bbad40102955ee2fe62ad390b53d7841fd Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 Aug 2025 15:49:18 -0400 Subject: [PATCH 03/49] wip3 --- lib/codecs/src/encoding/format/protobuf.rs | 2 +- src/sinks/http/config.rs | 1 + src/sinks/http/encoder.rs | 4 ++++ src/sources/opentelemetry/grpc.rs | 23 ++++++++++++++++------ 4 files changed, 23 insertions(+), 7 deletions(-) diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index c014a529455e1..9d65ede9f7f60 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -30,7 +30,7 @@ impl ProtobufSerializerConfig { /// The data type of events that are accepted by `ProtobufSerializer`. pub fn input_type(&self) -> DataType { - DataType::Log + DataType::all_bits() } /// The schema required by the serializer. diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index e2d5c0840659e..7c8a3d863ebc1 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -241,6 +241,7 @@ impl SinkConfig for HttpSinkConfig { let batch_settings = self.batch.validate()?.into_batcher_settings()?; let encoder = self.build_encoder()?; + println!("🚧 encoder: {encoder:?}"); let transformer = self.encoding.transformer(); let mut request = self.request.clone(); diff --git a/src/sinks/http/encoder.rs b/src/sinks/http/encoder.rs index bf75d8065b32f..76ba363ac686d 100644 --- a/src/sinks/http/encoder.rs +++ b/src/sinks/http/encoder.rs @@ -71,6 +71,10 @@ impl SinkEncoder> for HttpEncoder { .map_err(|_| io::Error::other("unable to encode event"))?; } + println!("🚧 encoder: {:?}", self.encoder); + println!("📦 First encoded HTTP payload ({} bytes):", body.len()); + println!("{:?}", body); + match (self.encoder.serializer(), self.encoder.framer()) { (Json(_), NewlineDelimited(_)) => { if !body.is_empty() { diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 297245d4c6801..51ac8ededfc21 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -42,12 +42,23 @@ impl TraceService for Service { &self, request: Request, ) -> Result, Status> { - let events: Vec = request - .into_inner() - .resource_spans - .into_iter() - .flat_map(|v| v.into_event_iter()) - .collect(); + let raw_bytes = request.get_ref().encode_to_vec(); + let bytes = bytes::Bytes::from(raw_bytes); + let deserializer = ProtobufDeserializerConfig { + protobuf: ProtobufDeserializerOptions { + desc_file: PathBuf::from("/Users/pavlos.rontidis/CLionProjects/vector/pront/otel/proto/vector-proto-related/trace_service.desc"), + message_type: "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest".to_string(), + } + }.build().unwrap(); + let events = deserializer.parse(bytes, self.log_namespace).unwrap().into_vec(); + + + // let events: Vec = request + // .into_inner() + // .resource_spans + // .into_iter() + // .flat_map(|v| v.into_event_iter()) + // .collect(); self.handle_events(events, TRACES).await?; Ok(Response::new(ExportTraceServiceResponse { From d94874062bc7d33c3c04c8ff9df563739eebaf19 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 4 Aug 2025 15:49:32 -0400 Subject: [PATCH 04/49] ran cargo fmt --- src/sources/opentelemetry/grpc.rs | 11 +++++++--- src/sources/opentelemetry/http.rs | 34 +++++++++++++++---------------- src/sources/opentelemetry/mod.rs | 9 ++++---- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 51ac8ededfc21..c53633ea67ff5 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -50,8 +50,10 @@ impl TraceService for Service { message_type: "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest".to_string(), } }.build().unwrap(); - let events = deserializer.parse(bytes, self.log_namespace).unwrap().into_vec(); - + let events = deserializer + .parse(bytes, self.log_namespace) + .unwrap() + .into_vec(); // let events: Vec = request // .into_inner() @@ -84,7 +86,10 @@ impl LogsService for Service { message_type: "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest".to_string(), } }.build().unwrap(); - let events = deserializer.parse(bytes, self.log_namespace).unwrap().into_vec(); + let events = deserializer + .parse(bytes, self.log_namespace) + .unwrap() + .into_vec(); // let events: Vec = request // .into_inner() diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 1f415bea50630..f91fde599a365 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -94,7 +94,7 @@ pub(crate) fn build_warp_filter( bytes_received: Registered, events_received: Registered, headers: Vec, - decoder: Option + decoder: Option, ) -> BoxedFilter<(Response,)> { let log_filters = build_warp_log_filter( acknowledgements, @@ -103,7 +103,7 @@ pub(crate) fn build_warp_filter( bytes_received.clone(), events_received.clone(), headers.clone(), - decoder + decoder, ); let metrics_filters = build_warp_metrics_filter( acknowledgements, @@ -147,7 +147,7 @@ fn build_warp_log_filter( bytes_received: Registered, events_received: Registered, headers: Vec, - decoder: Option + decoder: Option, ) -> BoxedFilter<(Response,)> { warp::post() .and(warp::path!("v1" / "logs")) @@ -161,28 +161,28 @@ fn build_warp_log_filter( .and_then( move |encoding_header: Option, headers_config: HeaderMap, body: Bytes| { println!("📦 Raw bytes (len={}): {:02x?}", body.len(), body); - let events = if let Some(mut decoder) = decoder.clone() - { + let events = if let Some(mut decoder) = decoder.clone() { let mut body = BytesMut::from(body); - decoder.decode(&mut body).map(|result| - { + decoder + .decode(&mut body) + .map(|result| { if let Some(result) = result { result.0.into_vec() } else { Vec::new() } - } - ).map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) + }) + .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) } else { decode(encoding_header.as_deref(), body) - .and_then(|body| { - bytes_received.emit(ByteSize(body.len())); - decode_log_body(body, log_namespace, &events_received) - }) - .map(|mut events| { - enrich_events(&mut events, &headers, &headers_config, log_namespace); - events - }) + .and_then(|body| { + bytes_received.emit(ByteSize(body.len())); + decode_log_body(body, log_namespace, &events_received) + }) + .map(|mut events| { + enrich_events(&mut events, &headers, &headers_config, log_namespace); + events + }) }; println!("{:#?}", events); diff --git a/src/sources/opentelemetry/mod.rs b/src/sources/opentelemetry/mod.rs index 3a8a2bf5def4c..d703dc6e948d5 100644 --- a/src/sources/opentelemetry/mod.rs +++ b/src/sources/opentelemetry/mod.rs @@ -203,9 +203,11 @@ impl SourceConfig for OpentelemetryConfig { .add_service(trace_service); let decoder = if let Some(deserializer_config) = &self.decoding { - let decoding_config = DecodingConfig::new(FramingConfig::NewlineDelimited(Default::default()), - deserializer_config.clone(), - log_namespace); + let decoding_config = DecodingConfig::new( + FramingConfig::NewlineDelimited(Default::default()), + deserializer_config.clone(), + log_namespace, + ); Some(decoding_config.build()?) } else { None @@ -221,7 +223,6 @@ impl SourceConfig for OpentelemetryConfig { error!(message = "Source future failed.", %error); }); - let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; let protocol = http_tls_settings.http_protocol_name(); let bytes_received = register!(BytesReceived::from(Protocol::from(protocol))); From 2e03966084accc47d0e76f9751dbedbd82ee7dfb Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 8 Aug 2025 09:06:31 -0400 Subject: [PATCH 05/49] refactoring otel source --- src/sources/opentelemetry/config.rs | 348 +++++++++++++++++++++++++++ src/sources/opentelemetry/grpc.rs | 4 +- src/sources/opentelemetry/http.rs | 8 +- src/sources/opentelemetry/mod.rs | 352 +--------------------------- 4 files changed, 355 insertions(+), 357 deletions(-) create mode 100644 src/sources/opentelemetry/config.rs diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs new file mode 100644 index 0000000000000..30b49ec70389a --- /dev/null +++ b/src/sources/opentelemetry/config.rs @@ -0,0 +1,348 @@ +use std::net::SocketAddr; + +use futures::{future::join, FutureExt, TryFutureExt}; +use tonic::codec::CompressionEncoding; +use vector_lib::lookup::{owned_value_path, OwnedTargetPath}; +use vector_lib::opentelemetry::logs::{ + ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, RESOURCE_KEY, + SEVERITY_NUMBER_KEY, SEVERITY_TEXT_KEY, SPAN_ID_KEY, TRACE_ID_KEY, +}; + +use crate::codecs::DecodingConfig; +use crate::sources::http_server::{build_param_matcher, remove_duplicates}; +use crate::sources::opentelemetry::grpc::Service; +use crate::sources::opentelemetry::http::{build_warp_filter, run_http_server}; +use crate::{ + config::{ + DataType, GenerateConfig, Resource, SourceAcknowledgementsConfig, SourceConfig, + SourceContext, SourceOutput, + }, + http::KeepaliveConfig, + serde::bool_or_struct, + sources::{util::grpc::run_grpc_server_with_routes, Source}, + tls::{MaybeTlsSettings, TlsEnableableConfig}, +}; +use tonic::transport::server::RoutesBuilder; +use vector_lib::codecs::decoding::{DeserializerConfig, FramingConfig}; +use vector_lib::configurable::configurable_component; +use vector_lib::internal_event::{BytesReceived, EventsReceived, Protocol}; +use vector_lib::opentelemetry::proto::collector::{ + logs::v1::logs_service_server::LogsServiceServer, + metrics::v1::metrics_service_server::MetricsServiceServer, + trace::v1::trace_service_server::TraceServiceServer, +}; +use vector_lib::{ + config::{log_schema, LegacyKey, LogNamespace}, + schema::Definition, +}; +use vrl::value::{kind::Collection, Kind}; + +pub const LOGS: &str = "logs"; +pub const METRICS: &str = "metrics"; +pub const TRACES: &str = "traces"; + +/// Configuration for the `opentelemetry` source. +#[configurable_component(source("opentelemetry", "Receive OTLP data through gRPC or HTTP."))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +pub struct OpentelemetryConfig { + #[configurable(derived)] + grpc: GrpcConfig, + + #[configurable(derived)] + http: HttpConfig, + + #[configurable(derived)] + #[serde(default, deserialize_with = "bool_or_struct")] + acknowledgements: SourceAcknowledgementsConfig, + + /// The namespace to use for logs. This overrides the global setting. + #[configurable(metadata(docs::hidden))] + #[serde(default)] + log_namespace: Option, + + #[configurable(derived)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decoding: Option, +} + +/// Configuration for the `opentelemetry` gRPC server. +#[configurable_component] +#[configurable(metadata(docs::examples = "example_grpc_config()"))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +struct GrpcConfig { + /// The socket address to listen for connections on. + /// + /// It _must_ include a port. + #[configurable(metadata(docs::examples = "0.0.0.0:4317", docs::examples = "localhost:4317"))] + address: SocketAddr, + + #[configurable(derived)] + #[serde(default, skip_serializing_if = "Option::is_none")] + tls: Option, +} + +fn example_grpc_config() -> GrpcConfig { + GrpcConfig { + address: "0.0.0.0:4317".parse().unwrap(), + tls: None, + } +} + +/// Configuration for the `opentelemetry` HTTP server. +#[configurable_component] +#[configurable(metadata(docs::examples = "example_http_config()"))] +#[derive(Clone, Debug)] +#[serde(deny_unknown_fields)] +struct HttpConfig { + /// The socket address to listen for connections on. + /// + /// It _must_ include a port. + #[configurable(metadata(docs::examples = "0.0.0.0:4318", docs::examples = "localhost:4318"))] + address: SocketAddr, + + #[configurable(derived)] + #[serde(default, skip_serializing_if = "Option::is_none")] + tls: Option, + + #[configurable(derived)] + #[serde(default)] + keepalive: KeepaliveConfig, + + /// A list of HTTP headers to include in the log event. + /// + /// Accepts the wildcard (`*`) character for headers matching a specified pattern. + /// + /// Specifying "*" results in all headers included in the log event. + /// + /// These headers are not included in the JSON payload if a field with a conflicting name exists. + #[serde(default)] + #[configurable(metadata(docs::examples = "User-Agent"))] + #[configurable(metadata(docs::examples = "X-My-Custom-Header"))] + #[configurable(metadata(docs::examples = "X-*"))] + #[configurable(metadata(docs::examples = "*"))] + headers: Vec, +} + +fn example_http_config() -> HttpConfig { + HttpConfig { + address: "0.0.0.0:4318".parse().unwrap(), + tls: None, + keepalive: KeepaliveConfig::default(), + headers: vec![], + } +} + +impl GenerateConfig for OpentelemetryConfig { + fn generate_config() -> toml::Value { + toml::Value::try_from(Self { + grpc: example_grpc_config(), + http: example_http_config(), + acknowledgements: Default::default(), + log_namespace: None, + decoding: None, + }) + .unwrap() + } +} + +#[async_trait::async_trait] +#[typetag::serde(name = "opentelemetry")] +impl SourceConfig for OpentelemetryConfig { + async fn build(&self, cx: SourceContext) -> crate::Result { + let acknowledgements = cx.do_acknowledgements(self.acknowledgements); + let events_received = register!(EventsReceived); + let log_namespace = cx.log_namespace(self.log_namespace); + + let grpc_tls_settings = MaybeTlsSettings::from_config(self.grpc.tls.as_ref(), true)?; + + let log_service = LogsServiceServer::new(Service { + pipeline: cx.out.clone(), + acknowledgements, + log_namespace, + events_received: events_received.clone(), + }) + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); + + let trace_service = TraceServiceServer::new(Service { + pipeline: cx.out.clone(), + acknowledgements, + log_namespace, + events_received: events_received.clone(), + }) + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); + + let metrics_service = MetricsServiceServer::new(Service { + pipeline: cx.out.clone(), + acknowledgements, + log_namespace, + events_received: events_received.clone(), + }) + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); + + let mut builder = RoutesBuilder::default(); + builder + .add_service(log_service) + .add_service(metrics_service) + .add_service(trace_service); + + let decoder = if let Some(deserializer_config) = &self.decoding { + let decoding_config = DecodingConfig::new( + FramingConfig::NewlineDelimited(Default::default()), + deserializer_config.clone(), + log_namespace, + ); + Some(decoding_config.build()?) + } else { + None + }; + + let grpc_source = run_grpc_server_with_routes( + self.grpc.address, + grpc_tls_settings, + builder.routes(), + cx.shutdown.clone(), + ) + .map_err(|error| { + error!(message = "Source future failed.", %error); + }); + + let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; + let protocol = http_tls_settings.http_protocol_name(); + let bytes_received = register!(BytesReceived::from(Protocol::from(protocol))); + let headers = + build_param_matcher(&remove_duplicates(self.http.headers.clone(), "headers"))?; + + let filters = build_warp_filter( + acknowledgements, + log_namespace, + cx.out, + bytes_received, + events_received, + headers, + decoder, + ); + + let http_source = run_http_server( + self.http.address, + http_tls_settings, + filters, + cx.shutdown, + self.http.keepalive.clone(), + ); + + Ok(join(grpc_source, http_source).map(|_| Ok(())).boxed()) + } + + // TODO: appropriately handle "severity" meaning across both "severity_text" and "severity_number", + // as both are optional and can be converted to/from. + fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { + let log_namespace = global_log_namespace.merge(self.log_namespace); + let schema_definition = Definition::new_with_default_metadata(Kind::any(), [log_namespace]) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(RESOURCE_KEY))), + &owned_value_path!(RESOURCE_KEY), + Kind::object(Collection::from_unknown(Kind::any())).or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(ATTRIBUTES_KEY))), + &owned_value_path!(ATTRIBUTES_KEY), + Kind::object(Collection::from_unknown(Kind::any())).or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(TRACE_ID_KEY))), + &owned_value_path!(TRACE_ID_KEY), + Kind::bytes().or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(SPAN_ID_KEY))), + &owned_value_path!(SPAN_ID_KEY), + Kind::bytes().or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(SEVERITY_TEXT_KEY))), + &owned_value_path!(SEVERITY_TEXT_KEY), + Kind::bytes().or_undefined(), + Some("severity"), + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(SEVERITY_NUMBER_KEY))), + &owned_value_path!(SEVERITY_NUMBER_KEY), + Kind::integer().or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!(FLAGS_KEY))), + &owned_value_path!(FLAGS_KEY), + Kind::integer().or_undefined(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!( + DROPPED_ATTRIBUTES_COUNT_KEY + ))), + &owned_value_path!(DROPPED_ATTRIBUTES_COUNT_KEY), + Kind::integer(), + None, + ) + .with_source_metadata( + Self::NAME, + Some(LegacyKey::Overwrite(owned_value_path!( + OBSERVED_TIMESTAMP_KEY + ))), + &owned_value_path!(OBSERVED_TIMESTAMP_KEY), + Kind::timestamp(), + None, + ) + .with_source_metadata( + Self::NAME, + None, + &owned_value_path!("timestamp"), + Kind::timestamp(), + Some("timestamp"), + ) + .with_standard_vector_source_metadata(); + + let schema_definition = match log_namespace { + LogNamespace::Vector => { + schema_definition.with_meaning(OwnedTargetPath::event_root(), "message") + } + LogNamespace::Legacy => { + schema_definition.with_meaning(log_schema().owned_message_path(), "message") + } + }; + + vec![ + SourceOutput::new_maybe_logs(DataType::Log, schema_definition).with_port(LOGS), + SourceOutput::new_metrics().with_port(METRICS), + SourceOutput::new_traces().with_port(TRACES), + ] + } + + fn resources(&self) -> Vec { + vec![ + Resource::tcp(self.grpc.address), + Resource::tcp(self.http.address), + ] + } + + fn can_acknowledge(&self) -> bool { + true + } +} diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index c53633ea67ff5..11a4da0f67f7c 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,6 +1,6 @@ +use crate::sources::opentelemetry::config::{LOGS, METRICS, TRACES}; use crate::{ internal_events::{EventsReceived, StreamClosedError}, - sources::opentelemetry::{LOGS, METRICS, TRACES}, SourceSender, }; use futures::TryFutureExt; @@ -46,7 +46,7 @@ impl TraceService for Service { let bytes = bytes::Bytes::from(raw_bytes); let deserializer = ProtobufDeserializerConfig { protobuf: ProtobufDeserializerOptions { - desc_file: PathBuf::from("/Users/pavlos.rontidis/CLionProjects/vector/pront/otel/proto/vector-proto-related/trace_service.desc"), + desc_file: PathBuf::from("src/proto/opentelemetry-proto/opentelemetry-proto.desc"), message_type: "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest".to_string(), } }.build().unwrap(); diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index f91fde599a365..6f5f8d93f4809 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -1,12 +1,12 @@ use std::time::Duration; use std::{convert::Infallible, net::SocketAddr}; -use super::OpentelemetryConfig; use super::{reply::protobuf, status::Status}; use crate::codecs::Decoder as VectorDecoder; use crate::common::http::ErrorMessage; use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; use crate::sources::http_server::HttpConfigParamKind; +use crate::sources::opentelemetry::config::{OpentelemetryConfig, LOGS, METRICS, TRACES}; use crate::sources::util::add_headers; use crate::{ event::Event, @@ -190,7 +190,7 @@ fn build_warp_log_filter( events, acknowledgements, out.clone(), - super::LOGS, + LOGS, ExportLogsServiceResponse::default(), ) }, @@ -222,7 +222,7 @@ fn build_warp_metrics_filter( events, acknowledgements, out.clone(), - super::METRICS, + METRICS, ExportMetricsServiceResponse::default(), ) }) @@ -253,7 +253,7 @@ fn build_warp_trace_filter( events, acknowledgements, out.clone(), - super::TRACES, + TRACES, ExportTraceServiceResponse::default(), ) }) diff --git a/src/sources/opentelemetry/mod.rs b/src/sources/opentelemetry/mod.rs index d703dc6e948d5..f65513e448839 100644 --- a/src/sources/opentelemetry/mod.rs +++ b/src/sources/opentelemetry/mod.rs @@ -7,354 +7,4 @@ mod grpc; mod http; mod reply; mod status; - -use std::net::SocketAddr; - -use futures::{future::join, FutureExt, TryFutureExt}; -use tonic::codec::CompressionEncoding; -use vector_lib::lookup::{owned_value_path, OwnedTargetPath}; -use vector_lib::opentelemetry::logs::{ - ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, RESOURCE_KEY, - SEVERITY_NUMBER_KEY, SEVERITY_TEXT_KEY, SPAN_ID_KEY, TRACE_ID_KEY, -}; - -use self::{ - grpc::Service, - http::{build_warp_filter, run_http_server}, -}; -use super::http_server::{build_param_matcher, remove_duplicates}; -use crate::codecs::DecodingConfig; -use crate::{ - config::{ - DataType, GenerateConfig, Resource, SourceAcknowledgementsConfig, SourceConfig, - SourceContext, SourceOutput, - }, - http::KeepaliveConfig, - serde::bool_or_struct, - sources::{util::grpc::run_grpc_server_with_routes, Source}, - tls::{MaybeTlsSettings, TlsEnableableConfig}, -}; -use tonic::transport::server::RoutesBuilder; -use vector_lib::codecs::decoding::{DeserializerConfig, FramingConfig}; -use vector_lib::configurable::configurable_component; -use vector_lib::internal_event::{BytesReceived, EventsReceived, Protocol}; -use vector_lib::opentelemetry::proto::collector::{ - logs::v1::logs_service_server::LogsServiceServer, - metrics::v1::metrics_service_server::MetricsServiceServer, - trace::v1::trace_service_server::TraceServiceServer, -}; -use vector_lib::{ - config::{log_schema, LegacyKey, LogNamespace}, - schema::Definition, -}; -use vrl::value::{kind::Collection, Kind}; - -pub const LOGS: &str = "logs"; -pub const METRICS: &str = "metrics"; -pub const TRACES: &str = "traces"; - -/// Configuration for the `opentelemetry` source. -#[configurable_component(source("opentelemetry", "Receive OTLP data through gRPC or HTTP."))] -#[derive(Clone, Debug)] -#[serde(deny_unknown_fields)] -pub struct OpentelemetryConfig { - #[configurable(derived)] - grpc: GrpcConfig, - - #[configurable(derived)] - http: HttpConfig, - - #[configurable(derived)] - #[serde(default, deserialize_with = "bool_or_struct")] - acknowledgements: SourceAcknowledgementsConfig, - - /// The namespace to use for logs. This overrides the global setting. - #[configurable(metadata(docs::hidden))] - #[serde(default)] - log_namespace: Option, - - #[configurable(derived)] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub decoding: Option, -} - -/// Configuration for the `opentelemetry` gRPC server. -#[configurable_component] -#[configurable(metadata(docs::examples = "example_grpc_config()"))] -#[derive(Clone, Debug)] -#[serde(deny_unknown_fields)] -struct GrpcConfig { - /// The socket address to listen for connections on. - /// - /// It _must_ include a port. - #[configurable(metadata(docs::examples = "0.0.0.0:4317", docs::examples = "localhost:4317"))] - address: SocketAddr, - - #[configurable(derived)] - #[serde(default, skip_serializing_if = "Option::is_none")] - tls: Option, -} - -fn example_grpc_config() -> GrpcConfig { - GrpcConfig { - address: "0.0.0.0:4317".parse().unwrap(), - tls: None, - } -} - -/// Configuration for the `opentelemetry` HTTP server. -#[configurable_component] -#[configurable(metadata(docs::examples = "example_http_config()"))] -#[derive(Clone, Debug)] -#[serde(deny_unknown_fields)] -struct HttpConfig { - /// The socket address to listen for connections on. - /// - /// It _must_ include a port. - #[configurable(metadata(docs::examples = "0.0.0.0:4318", docs::examples = "localhost:4318"))] - address: SocketAddr, - - #[configurable(derived)] - #[serde(default, skip_serializing_if = "Option::is_none")] - tls: Option, - - #[configurable(derived)] - #[serde(default)] - keepalive: KeepaliveConfig, - - /// A list of HTTP headers to include in the log event. - /// - /// Accepts the wildcard (`*`) character for headers matching a specified pattern. - /// - /// Specifying "*" results in all headers included in the log event. - /// - /// These headers are not included in the JSON payload if a field with a conflicting name exists. - #[serde(default)] - #[configurable(metadata(docs::examples = "User-Agent"))] - #[configurable(metadata(docs::examples = "X-My-Custom-Header"))] - #[configurable(metadata(docs::examples = "X-*"))] - #[configurable(metadata(docs::examples = "*"))] - headers: Vec, -} - -fn example_http_config() -> HttpConfig { - HttpConfig { - address: "0.0.0.0:4318".parse().unwrap(), - tls: None, - keepalive: KeepaliveConfig::default(), - headers: vec![], - } -} - -impl GenerateConfig for OpentelemetryConfig { - fn generate_config() -> toml::Value { - toml::Value::try_from(Self { - grpc: example_grpc_config(), - http: example_http_config(), - acknowledgements: Default::default(), - log_namespace: None, - decoding: None, - }) - .unwrap() - } -} - -#[async_trait::async_trait] -#[typetag::serde(name = "opentelemetry")] -impl SourceConfig for OpentelemetryConfig { - async fn build(&self, cx: SourceContext) -> crate::Result { - let acknowledgements = cx.do_acknowledgements(self.acknowledgements); - let events_received = register!(EventsReceived); - let log_namespace = cx.log_namespace(self.log_namespace); - - let grpc_tls_settings = MaybeTlsSettings::from_config(self.grpc.tls.as_ref(), true)?; - - let log_service = LogsServiceServer::new(Service { - pipeline: cx.out.clone(), - acknowledgements, - log_namespace, - events_received: events_received.clone(), - }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); - - let trace_service = TraceServiceServer::new(Service { - pipeline: cx.out.clone(), - acknowledgements, - log_namespace, - events_received: events_received.clone(), - }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); - - let metrics_service = MetricsServiceServer::new(Service { - pipeline: cx.out.clone(), - acknowledgements, - log_namespace, - events_received: events_received.clone(), - }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); - - let mut builder = RoutesBuilder::default(); - builder - .add_service(log_service) - .add_service(metrics_service) - .add_service(trace_service); - - let decoder = if let Some(deserializer_config) = &self.decoding { - let decoding_config = DecodingConfig::new( - FramingConfig::NewlineDelimited(Default::default()), - deserializer_config.clone(), - log_namespace, - ); - Some(decoding_config.build()?) - } else { - None - }; - - let grpc_source = run_grpc_server_with_routes( - self.grpc.address, - grpc_tls_settings, - builder.routes(), - cx.shutdown.clone(), - ) - .map_err(|error| { - error!(message = "Source future failed.", %error); - }); - - let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; - let protocol = http_tls_settings.http_protocol_name(); - let bytes_received = register!(BytesReceived::from(Protocol::from(protocol))); - let headers = - build_param_matcher(&remove_duplicates(self.http.headers.clone(), "headers"))?; - - let filters = build_warp_filter( - acknowledgements, - log_namespace, - cx.out, - bytes_received, - events_received, - headers, - decoder, - ); - - let http_source = run_http_server( - self.http.address, - http_tls_settings, - filters, - cx.shutdown, - self.http.keepalive.clone(), - ); - - Ok(join(grpc_source, http_source).map(|_| Ok(())).boxed()) - } - - // TODO: appropriately handle "severity" meaning across both "severity_text" and "severity_number", - // as both are optional and can be converted to/from. - fn outputs(&self, global_log_namespace: LogNamespace) -> Vec { - let log_namespace = global_log_namespace.merge(self.log_namespace); - let schema_definition = Definition::new_with_default_metadata(Kind::any(), [log_namespace]) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(RESOURCE_KEY))), - &owned_value_path!(RESOURCE_KEY), - Kind::object(Collection::from_unknown(Kind::any())).or_undefined(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(ATTRIBUTES_KEY))), - &owned_value_path!(ATTRIBUTES_KEY), - Kind::object(Collection::from_unknown(Kind::any())).or_undefined(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(TRACE_ID_KEY))), - &owned_value_path!(TRACE_ID_KEY), - Kind::bytes().or_undefined(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(SPAN_ID_KEY))), - &owned_value_path!(SPAN_ID_KEY), - Kind::bytes().or_undefined(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(SEVERITY_TEXT_KEY))), - &owned_value_path!(SEVERITY_TEXT_KEY), - Kind::bytes().or_undefined(), - Some("severity"), - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(SEVERITY_NUMBER_KEY))), - &owned_value_path!(SEVERITY_NUMBER_KEY), - Kind::integer().or_undefined(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!(FLAGS_KEY))), - &owned_value_path!(FLAGS_KEY), - Kind::integer().or_undefined(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!( - DROPPED_ATTRIBUTES_COUNT_KEY - ))), - &owned_value_path!(DROPPED_ATTRIBUTES_COUNT_KEY), - Kind::integer(), - None, - ) - .with_source_metadata( - Self::NAME, - Some(LegacyKey::Overwrite(owned_value_path!( - OBSERVED_TIMESTAMP_KEY - ))), - &owned_value_path!(OBSERVED_TIMESTAMP_KEY), - Kind::timestamp(), - None, - ) - .with_source_metadata( - Self::NAME, - None, - &owned_value_path!("timestamp"), - Kind::timestamp(), - Some("timestamp"), - ) - .with_standard_vector_source_metadata(); - - let schema_definition = match log_namespace { - LogNamespace::Vector => { - schema_definition.with_meaning(OwnedTargetPath::event_root(), "message") - } - LogNamespace::Legacy => { - schema_definition.with_meaning(log_schema().owned_message_path(), "message") - } - }; - - vec![ - SourceOutput::new_maybe_logs(DataType::Log, schema_definition).with_port(LOGS), - SourceOutput::new_metrics().with_port(METRICS), - SourceOutput::new_traces().with_port(TRACES), - ] - } - - fn resources(&self) -> Vec { - vec![ - Resource::tcp(self.grpc.address), - Resource::tcp(self.http.address), - ] - } - - fn can_acknowledge(&self) -> bool { - true - } -} +pub mod config; From f250f8b04d2c1026ebf6ff0ac8ca56df4c95a229 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 8 Aug 2025 13:02:41 -0400 Subject: [PATCH 06/49] wip --- src/sources/opentelemetry/config.rs | 7 ++++++ src/sources/opentelemetry/grpc.rs | 33 ++++++++++++----------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 30b49ec70389a..4b1703996e947 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -61,6 +61,7 @@ pub struct OpentelemetryConfig { #[serde(default)] log_namespace: Option, + /// Setting this field will override the legacy mapping of OTEL protos to Vector events. #[configurable(derived)] #[serde(default, skip_serializing_if = "Option::is_none")] pub decoding: Option, @@ -154,6 +155,9 @@ impl SourceConfig for OpentelemetryConfig { let acknowledgements = cx.do_acknowledgements(self.acknowledgements); let events_received = register!(EventsReceived); let log_namespace = cx.log_namespace(self.log_namespace); + let decoder = self.decoding.map(|deserializer| + DecodingConfig::new(FramingConfig::Bytes, deserializer, self.log_namespace)); + let grpc_tls_settings = MaybeTlsSettings::from_config(self.grpc.tls.as_ref(), true)?; @@ -162,6 +166,7 @@ impl SourceConfig for OpentelemetryConfig { acknowledgements, log_namespace, events_received: events_received.clone(), + decoder, }) .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); @@ -171,6 +176,7 @@ impl SourceConfig for OpentelemetryConfig { acknowledgements, log_namespace, events_received: events_received.clone(), + decoder }) .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); @@ -180,6 +186,7 @@ impl SourceConfig for OpentelemetryConfig { acknowledgements, log_namespace, events_received: events_received.clone(), + decoder }) .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 11a4da0f67f7c..46d1128cf87f2 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,3 +1,4 @@ +use crate::codecs::Decoder; use crate::sources::opentelemetry::config::{LOGS, METRICS, TRACES}; use crate::{ internal_events::{EventsReceived, StreamClosedError}, @@ -34,6 +35,7 @@ pub(super) struct Service { pub acknowledgements: bool, pub events_received: Registered, pub log_namespace: LogNamespace, + pub decoder: Option } #[tonic::async_trait] @@ -42,25 +44,18 @@ impl TraceService for Service { &self, request: Request, ) -> Result, Status> { - let raw_bytes = request.get_ref().encode_to_vec(); - let bytes = bytes::Bytes::from(raw_bytes); - let deserializer = ProtobufDeserializerConfig { - protobuf: ProtobufDeserializerOptions { - desc_file: PathBuf::from("src/proto/opentelemetry-proto/opentelemetry-proto.desc"), - message_type: "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest".to_string(), - } - }.build().unwrap(); - let events = deserializer - .parse(bytes, self.log_namespace) - .unwrap() - .into_vec(); - - // let events: Vec = request - // .into_inner() - // .resource_spans - // .into_iter() - // .flat_map(|v| v.into_event_iter()) - // .collect(); + let events = if let Some(decoder) = self.decoder.as_deref() { + let raw_bytes = request.get_ref().encode_to_vec(); + let bytes = bytes::Bytes::from(raw_bytes); + decoder.dexcode(bytes) + } else { + request + .into_inner() + .resource_spans + .into_iter() + .flat_map(|v| v.into_event_iter()) + .collect() + }; self.handle_events(events, TRACES).await?; Ok(Response::new(ExportTraceServiceResponse { From 64eecdf5ad25257fdae655bcb289a5d46c261721 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 11 Aug 2025 14:18:14 -0400 Subject: [PATCH 07/49] ran cargo fmt --- src/sources/opentelemetry/config.rs | 30 ++++++++++++++--------------- src/sources/opentelemetry/grpc.rs | 2 +- src/sources/opentelemetry/mod.rs | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 4b1703996e947..cd2bac7be620b 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -144,7 +144,7 @@ impl GenerateConfig for OpentelemetryConfig { log_namespace: None, decoding: None, }) - .unwrap() + .unwrap() } } @@ -155,9 +155,9 @@ impl SourceConfig for OpentelemetryConfig { let acknowledgements = cx.do_acknowledgements(self.acknowledgements); let events_received = register!(EventsReceived); let log_namespace = cx.log_namespace(self.log_namespace); - let decoder = self.decoding.map(|deserializer| - DecodingConfig::new(FramingConfig::Bytes, deserializer, self.log_namespace)); - + let decoder = self.decoding.map(|deserializer| { + DecodingConfig::new(FramingConfig::Bytes, deserializer, self.log_namespace) + }); let grpc_tls_settings = MaybeTlsSettings::from_config(self.grpc.tls.as_ref(), true)?; @@ -168,28 +168,28 @@ impl SourceConfig for OpentelemetryConfig { events_received: events_received.clone(), decoder, }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); let trace_service = TraceServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, log_namespace, events_received: events_received.clone(), - decoder + decoder, }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); let metrics_service = MetricsServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, log_namespace, events_received: events_received.clone(), - decoder + decoder, }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); let mut builder = RoutesBuilder::default(); builder @@ -214,9 +214,9 @@ impl SourceConfig for OpentelemetryConfig { builder.routes(), cx.shutdown.clone(), ) - .map_err(|error| { - error!(message = "Source future failed.", %error); - }); + .map_err(|error| { + error!(message = "Source future failed.", %error); + }); let http_tls_settings = MaybeTlsSettings::from_config(self.http.tls.as_ref(), true)?; let protocol = http_tls_settings.http_protocol_name(); diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 46d1128cf87f2..5d58e1c75132a 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -35,7 +35,7 @@ pub(super) struct Service { pub acknowledgements: bool, pub events_received: Registered, pub log_namespace: LogNamespace, - pub decoder: Option + pub decoder: Option, } #[tonic::async_trait] diff --git a/src/sources/opentelemetry/mod.rs b/src/sources/opentelemetry/mod.rs index f65513e448839..0e4129798b525 100644 --- a/src/sources/opentelemetry/mod.rs +++ b/src/sources/opentelemetry/mod.rs @@ -3,8 +3,8 @@ mod integration_tests; #[cfg(test)] mod tests; +pub mod config; mod grpc; mod http; mod reply; mod status; -pub mod config; From f976a74c1ff18bb5cd7f4e9435a31e8b1f84a93b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 11 Aug 2025 14:24:54 -0400 Subject: [PATCH 08/49] commit the otel proto desc --- lib/opentelemetry-proto/build.rs | 5 ++--- tests/data/e2e/opentelemetry/logs/output/.gitignore | 0 2 files changed, 2 insertions(+), 3 deletions(-) mode change 100755 => 100644 tests/data/e2e/opentelemetry/logs/output/.gitignore diff --git a/lib/opentelemetry-proto/build.rs b/lib/opentelemetry-proto/build.rs index 19fc3d533bbc7..94a730f743f5e 100644 --- a/lib/opentelemetry-proto/build.rs +++ b/lib/opentelemetry-proto/build.rs @@ -1,5 +1,5 @@ use glob::glob; -use std::{env, io::Result, path::PathBuf}; +use std::{io::Result, path::PathBuf}; fn main() -> Result<()> { let proto_root = PathBuf::from("src/proto/opentelemetry-proto"); @@ -15,8 +15,7 @@ fn main() -> Result<()> { println!("cargo:rerun-if-changed={}", proto.display()); } - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); - let descriptor_path = out_dir.join("opentelemetry-proto.desc"); + let descriptor_path = proto_root.join("opentelemetry-proto.desc"); tonic_build::configure() .build_client(true) diff --git a/tests/data/e2e/opentelemetry/logs/output/.gitignore b/tests/data/e2e/opentelemetry/logs/output/.gitignore old mode 100755 new mode 100644 From 98fc30d61d18becbabd1ecbeb82c984e351b8b1a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 11 Aug 2025 16:31:52 -0400 Subject: [PATCH 09/49] refactoring --- Cargo.lock | 1 - Cargo.toml | 2 +- lib/codecs/src/decoding/format/protobuf.rs | 6 ++ src/sources/opentelemetry/config.rs | 87 ++++++++++++---------- src/sources/opentelemetry/grpc.rs | 9 +-- src/sources/opentelemetry/http.rs | 25 +++---- src/sources/opentelemetry/tests.rs | 6 +- 7 files changed, 70 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 308aa895e1dc1..9efe8fda52d56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12408,7 +12408,6 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.26.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#40d3f6dfa395ef5dd306432e6cfd03af9966da00" dependencies = [ "aes", "aes-siv", diff --git a/Cargo.toml b/Cargo.toml index ca22f73b017e8..88298277bd119 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -178,7 +178,7 @@ vector-lib = { path = "lib/vector-lib", default-features = false, features = ["v vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { path = "../vrl", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] cfg-if.workspace = true diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index cacd7912b9cbc..19b69a71e58d4 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -89,6 +89,12 @@ impl ProtobufDeserializer { pub fn new(message_descriptor: MessageDescriptor) -> Self { Self { message_descriptor } } + + /// Creates a new deserializer instance using the descriptor bytes directly. + pub fn new_from_bytes(desc_bytes: &[u8], message_type: &str) -> vector_common::Result { + let message_descriptor = vrl::protobuf::get_message_descriptor_from_bytes(desc_bytes, message_type)?; + Ok(Self { message_descriptor }) + } } impl Deserializer for ProtobufDeserializer { diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index cd2bac7be620b..9bd1f89ec501d 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,6 +1,8 @@ use std::net::SocketAddr; -use futures::{future::join, FutureExt, TryFutureExt}; +use futures::FutureExt; +use futures_util::future::join; +use futures_util::TryFutureExt; use tonic::codec::CompressionEncoding; use vector_lib::lookup::{owned_value_path, OwnedTargetPath}; use vector_lib::opentelemetry::logs::{ @@ -8,7 +10,6 @@ use vector_lib::opentelemetry::logs::{ SEVERITY_NUMBER_KEY, SEVERITY_TEXT_KEY, SPAN_ID_KEY, TRACE_ID_KEY, }; -use crate::codecs::DecodingConfig; use crate::sources::http_server::{build_param_matcher, remove_duplicates}; use crate::sources::opentelemetry::grpc::Service; use crate::sources::opentelemetry::http::{build_warp_filter, run_http_server}; @@ -23,7 +24,7 @@ use crate::{ tls::{MaybeTlsSettings, TlsEnableableConfig}, }; use tonic::transport::server::RoutesBuilder; -use vector_lib::codecs::decoding::{DeserializerConfig, FramingConfig}; +use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::configurable::configurable_component; use vector_lib::internal_event::{BytesReceived, EventsReceived, Protocol}; use vector_lib::opentelemetry::proto::collector::{ @@ -41,30 +42,35 @@ pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; pub const TRACES: &str = "traces"; +pub static OTEL_PROTO_DESCRIPTOR_BYTES: &[u8] = include_bytes!("../../../lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry-proto.desc"); +pub const OTEL_PROTO_LOGS_REQUEST: &str = "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest"; +pub const OTEL_PROTO_TRACES_REQUEST: &str = "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest"; +pub const OTEL_PROTO_METRICS_REQUEST: &str = "opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest"; + /// Configuration for the `opentelemetry` source. #[configurable_component(source("opentelemetry", "Receive OTLP data through gRPC or HTTP."))] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct OpentelemetryConfig { #[configurable(derived)] - grpc: GrpcConfig, + pub grpc: GrpcConfig, #[configurable(derived)] - http: HttpConfig, + pub http: HttpConfig, #[configurable(derived)] #[serde(default, deserialize_with = "bool_or_struct")] - acknowledgements: SourceAcknowledgementsConfig, + pub acknowledgements: SourceAcknowledgementsConfig, /// The namespace to use for logs. This overrides the global setting. #[configurable(metadata(docs::hidden))] #[serde(default)] - log_namespace: Option, + pub log_namespace: Option, - /// Setting this field will override the legacy mapping of OTEL protos to Vector events. + /// Setting this field will override the legacy mapping of OTEL protos to Vector events and use the proto directly. #[configurable(derived)] - #[serde(default, skip_serializing_if = "Option::is_none")] - pub decoding: Option, + #[serde(default)] + pub use_oltp_decoding: bool, } /// Configuration for the `opentelemetry` gRPC server. @@ -72,16 +78,16 @@ pub struct OpentelemetryConfig { #[configurable(metadata(docs::examples = "example_grpc_config()"))] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -struct GrpcConfig { +pub struct GrpcConfig { /// The socket address to listen for connections on. /// /// It _must_ include a port. #[configurable(metadata(docs::examples = "0.0.0.0:4317", docs::examples = "localhost:4317"))] - address: SocketAddr, + pub address: SocketAddr, #[configurable(derived)] #[serde(default, skip_serializing_if = "Option::is_none")] - tls: Option, + pub tls: Option, } fn example_grpc_config() -> GrpcConfig { @@ -96,20 +102,20 @@ fn example_grpc_config() -> GrpcConfig { #[configurable(metadata(docs::examples = "example_http_config()"))] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] -struct HttpConfig { +pub struct HttpConfig { /// The socket address to listen for connections on. /// /// It _must_ include a port. #[configurable(metadata(docs::examples = "0.0.0.0:4318", docs::examples = "localhost:4318"))] - address: SocketAddr, + pub address: SocketAddr, #[configurable(derived)] #[serde(default, skip_serializing_if = "Option::is_none")] - tls: Option, + pub tls: Option, #[configurable(derived)] #[serde(default)] - keepalive: KeepaliveConfig, + pub keepalive: KeepaliveConfig, /// A list of HTTP headers to include in the log event. /// @@ -123,7 +129,7 @@ struct HttpConfig { #[configurable(metadata(docs::examples = "X-My-Custom-Header"))] #[configurable(metadata(docs::examples = "X-*"))] #[configurable(metadata(docs::examples = "*"))] - headers: Vec, + pub headers: Vec, } fn example_http_config() -> HttpConfig { @@ -142,12 +148,21 @@ impl GenerateConfig for OpentelemetryConfig { http: example_http_config(), acknowledgements: Default::default(), log_namespace: None, - decoding: None, + use_oltp_decoding: false, }) .unwrap() } } +impl OpentelemetryConfig { + fn get_deserializer(&self, message_type: &str) -> vector_common::Result> { + if self.use_oltp_decoding { + let deserializer = ProtobufDeserializer::new_from_bytes(OTEL_PROTO_DESCRIPTOR_BYTES, message_type)?; + Ok(Some(deserializer)) + } else { Ok(None) } + } +} + #[async_trait::async_trait] #[typetag::serde(name = "opentelemetry")] impl SourceConfig for OpentelemetryConfig { @@ -155,38 +170,39 @@ impl SourceConfig for OpentelemetryConfig { let acknowledgements = cx.do_acknowledgements(self.acknowledgements); let events_received = register!(EventsReceived); let log_namespace = cx.log_namespace(self.log_namespace); - let decoder = self.decoding.map(|deserializer| { - DecodingConfig::new(FramingConfig::Bytes, deserializer, self.log_namespace) - }); let grpc_tls_settings = MaybeTlsSettings::from_config(self.grpc.tls.as_ref(), true)?; + let log_deserializer = self.get_deserializer(OTEL_PROTO_LOGS_REQUEST)?; let log_service = LogsServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, log_namespace, events_received: events_received.clone(), - decoder, + deserializer: log_deserializer.clone(), }) .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); - let trace_service = TraceServiceServer::new(Service { + let metric_deserializer = self.get_deserializer(OTEL_PROTO_METRICS_REQUEST)?; + let metrics_service = MetricsServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, log_namespace, events_received: events_received.clone(), - decoder, + deserializer: metric_deserializer, }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); - let metrics_service = MetricsServiceServer::new(Service { + + let trace_deserializer = self.get_deserializer(OTEL_PROTO_TRACES_REQUEST)?; + let trace_service = TraceServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, log_namespace, events_received: events_received.clone(), - decoder, + deserializer: trace_deserializer, }) .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); @@ -197,17 +213,6 @@ impl SourceConfig for OpentelemetryConfig { .add_service(metrics_service) .add_service(trace_service); - let decoder = if let Some(deserializer_config) = &self.decoding { - let decoding_config = DecodingConfig::new( - FramingConfig::NewlineDelimited(Default::default()), - deserializer_config.clone(), - log_namespace, - ); - Some(decoding_config.build()?) - } else { - None - }; - let grpc_source = run_grpc_server_with_routes( self.grpc.address, grpc_tls_settings, @@ -231,7 +236,7 @@ impl SourceConfig for OpentelemetryConfig { bytes_received, events_received, headers, - decoder, + log_deserializer, ); let http_source = run_http_server( diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 5d58e1c75132a..63fcf3439ab18 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,4 +1,3 @@ -use crate::codecs::Decoder; use crate::sources::opentelemetry::config::{LOGS, METRICS, TRACES}; use crate::{ internal_events::{EventsReceived, StreamClosedError}, @@ -9,7 +8,7 @@ use prost::Message; use std::path::PathBuf; use tonic::{Request, Response, Status}; use vector_lib::codecs::decoding::format::Deserializer; -use vector_lib::codecs::decoding::{ProtobufDeserializerConfig, ProtobufDeserializerOptions}; +use vector_lib::codecs::decoding::{ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions}; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ @@ -35,7 +34,7 @@ pub(super) struct Service { pub acknowledgements: bool, pub events_received: Registered, pub log_namespace: LogNamespace, - pub decoder: Option, + pub deserializer: Option, } #[tonic::async_trait] @@ -44,10 +43,10 @@ impl TraceService for Service { &self, request: Request, ) -> Result, Status> { - let events = if let Some(decoder) = self.decoder.as_deref() { + let events = if let Some(deserializer) = self.deserializer.as_ref() { let raw_bytes = request.get_ref().encode_to_vec(); let bytes = bytes::Bytes::from(raw_bytes); - decoder.dexcode(bytes) + deserializer.parse(bytes, self.log_namespace).map_err(|e| Status::invalid_argument(e.to_string()))?.into_vec() } else { request .into_inner() diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 6f5f8d93f4809..b0690bb4647f4 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -2,7 +2,6 @@ use std::time::Duration; use std::{convert::Infallible, net::SocketAddr}; use super::{reply::protobuf, status::Status}; -use crate::codecs::Decoder as VectorDecoder; use crate::common::http::ErrorMessage; use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; use crate::sources::http_server::HttpConfigParamKind; @@ -17,16 +16,17 @@ use crate::{ tls::MaybeTlsSettings, SourceSender, }; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use futures_util::FutureExt; use http::StatusCode; use hyper::{service::make_service_fn, Server}; use prost::Message; use snafu::Snafu; use tokio::net::TcpStream; -use tokio_util::codec::Decoder; use tower::ServiceBuilder; use tracing::Span; +use vector_lib::codecs::decoding::format::Deserializer; +use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Registered, }; @@ -94,7 +94,7 @@ pub(crate) fn build_warp_filter( bytes_received: Registered, events_received: Registered, headers: Vec, - decoder: Option, + deserializer: Option, ) -> BoxedFilter<(Response,)> { let log_filters = build_warp_log_filter( acknowledgements, @@ -103,7 +103,7 @@ pub(crate) fn build_warp_filter( bytes_received.clone(), events_received.clone(), headers.clone(), - decoder, + deserializer, ); let metrics_filters = build_warp_metrics_filter( acknowledgements, @@ -147,7 +147,7 @@ fn build_warp_log_filter( bytes_received: Registered, events_received: Registered, headers: Vec, - decoder: Option, + deserializer: Option, ) -> BoxedFilter<(Response,)> { warp::post() .and(warp::path!("v1" / "logs")) @@ -161,16 +161,11 @@ fn build_warp_log_filter( .and_then( move |encoding_header: Option, headers_config: HeaderMap, body: Bytes| { println!("📦 Raw bytes (len={}): {:02x?}", body.len(), body); - let events = if let Some(mut decoder) = decoder.clone() { - let mut body = BytesMut::from(body); - decoder - .decode(&mut body) + let events = if let Some(deserializer) = deserializer.as_ref() { + deserializer + .parse(body, log_namespace) .map(|result| { - if let Some(result) = result { - result.0.into_vec() - } else { - Vec::new() - } + result.into_vec() }) .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) } else { diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index dea7596df5160..ca0c81b423bfd 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -6,7 +6,7 @@ use crate::{ event::{ into_event_stream, Event, EventStatus, LogEvent, Metric as MetricEvent, ObjectMap, Value, }, - sources::opentelemetry::{GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS}, + sources::opentelemetry::config::{GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS}, test_util::{ self, components::{assert_source_compliance, SOURCE_TAGS}, @@ -1132,7 +1132,7 @@ async fn http_headers() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - decoding: None, + use_oltp_decoding: false, }; let schema_definitions = source .outputs(LogNamespace::Legacy) @@ -1238,7 +1238,7 @@ pub async fn build_otlp_test_env( }, acknowledgements: Default::default(), log_namespace, - decoding: None, + use_oltp_decoding: false, }; let (sender, output, _) = new_source(EventStatus::Delivered, event_name.to_string()); From 35140c6ef3e86444830b531f74e7126eb2e49935 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 12 Aug 2025 10:44:28 -0400 Subject: [PATCH 10/49] fix re-build mess --- lib/opentelemetry-proto/build.rs | 27 +++++++++++++++++++++------ lib/opentelemetry-proto/src/proto.rs | 3 +++ src/sources/opentelemetry/config.rs | 3 +-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/lib/opentelemetry-proto/build.rs b/lib/opentelemetry-proto/build.rs index 94a730f743f5e..7e5fcb5f40279 100644 --- a/lib/opentelemetry-proto/build.rs +++ b/lib/opentelemetry-proto/build.rs @@ -1,4 +1,6 @@ use glob::glob; +use std::fs::{read_to_string, write}; +use std::path::Path; use std::{io::Result, path::PathBuf}; fn main() -> Result<()> { @@ -10,12 +12,8 @@ fn main() -> Result<()> { .filter_map(|result| result.ok()) .collect(); - // Set up re-run triggers - for proto in &proto_paths { - println!("cargo:rerun-if-changed={}", proto.display()); - } - - let descriptor_path = proto_root.join("opentelemetry-proto.desc"); + let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let descriptor_path = out_dir.join("opentelemetry-proto.desc"); tonic_build::configure() .build_client(true) @@ -23,5 +21,22 @@ fn main() -> Result<()> { .file_descriptor_set_path(&descriptor_path) .compile(&proto_paths, &[include_path])?; + write_static_descriptor_reference(&descriptor_path, &out_dir)?; + + Ok(()) +} + +fn write_static_descriptor_reference(descriptor_path: &Path, out_dir: &Path) -> Result<()> { + let include_line = format!( + "pub static DESCRIPTOR_BYTES: &[u8] = include_bytes!(r\"{}\");\n", + descriptor_path.display() + ); + + let include_file = out_dir.join("opentelemetry-proto.rs"); + let existing = read_to_string(&include_file).ok(); + if existing.as_deref() != Some(&include_line) { + write(&include_file, include_line)?; + } + Ok(()) } diff --git a/lib/opentelemetry-proto/src/proto.rs b/lib/opentelemetry-proto/src/proto.rs index 4c2b1acf340c6..5559113bd14db 100644 --- a/lib/opentelemetry-proto/src/proto.rs +++ b/lib/opentelemetry-proto/src/proto.rs @@ -51,3 +51,6 @@ pub mod resource { tonic::include_proto!("opentelemetry.proto.resource.v1"); } } + +/// The raw descriptor bytes for all the above. +include!(concat!(env!("OUT_DIR"), "/opentelemetry-proto.rs")); diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 9bd1f89ec501d..1d8542ded7ba2 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -42,7 +42,6 @@ pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; pub const TRACES: &str = "traces"; -pub static OTEL_PROTO_DESCRIPTOR_BYTES: &[u8] = include_bytes!("../../../lib/opentelemetry-proto/src/proto/opentelemetry-proto/opentelemetry-proto.desc"); pub const OTEL_PROTO_LOGS_REQUEST: &str = "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest"; pub const OTEL_PROTO_TRACES_REQUEST: &str = "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest"; pub const OTEL_PROTO_METRICS_REQUEST: &str = "opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest"; @@ -157,7 +156,7 @@ impl GenerateConfig for OpentelemetryConfig { impl OpentelemetryConfig { fn get_deserializer(&self, message_type: &str) -> vector_common::Result> { if self.use_oltp_decoding { - let deserializer = ProtobufDeserializer::new_from_bytes(OTEL_PROTO_DESCRIPTOR_BYTES, message_type)?; + let deserializer = ProtobufDeserializer::new_from_bytes(vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, message_type)?; Ok(Some(deserializer)) } else { Ok(None) } } From 4c11c870b1660dec2b9defedd4f828d239c2785c Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 12 Aug 2025 11:02:28 -0400 Subject: [PATCH 11/49] unrelated web-playground build fix --- lib/vector-vrl/web-playground/Cargo.toml | 3 ++ lib/vector-vrl/web-playground/build.rs | 59 +++++++++++++----------- 2 files changed, 36 insertions(+), 26 deletions(-) diff --git a/lib/vector-vrl/web-playground/Cargo.toml b/lib/vector-vrl/web-playground/Cargo.toml index 544359ff7fac6..30a2c44a7716c 100644 --- a/lib/vector-vrl/web-playground/Cargo.toml +++ b/lib/vector-vrl/web-playground/Cargo.toml @@ -2,6 +2,9 @@ name = "vector-vrl-web-playground" version = "0.1.0" edition = "2021" +repository = "https://github.com/vectordotdev/vector" +description = "A playground for experimenting with VRL" +license = "MPL-2.0" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/lib/vector-vrl/web-playground/build.rs b/lib/vector-vrl/web-playground/build.rs index b6bc46a33cc80..0fff8acd4f494 100644 --- a/lib/vector-vrl/web-playground/build.rs +++ b/lib/vector-vrl/web-playground/build.rs @@ -55,40 +55,47 @@ fn write_vrl_constants(lockfile: &Lockfile, output_file: &mut File) { .find(|&package| package.name.as_str() == "vrl") .expect("missing VRL dependency"); - let vrl_source = vrl_dep.source.clone().expect("missing VRL source id"); - - let (version, link) = match vrl_source.kind() { - SourceKind::Git(_) => { - let precise = vrl_source - .precise() - .expect("git reference should have precise") - .to_string(); - ( - precise.clone(), - format!("{}/tree/{precise}", vrl_source.url()), - ) - } - SourceKind::Registry if vrl_source.is_default_registry() => { - let version = vrl_dep.version.to_string(); - ( - version.to_string(), - format!("https://crates.io/crates/vrl/{version}"), - ) + let (version, link) = if let Some(vrl_source) = vrl_dep.source.clone() { + match vrl_source.kind() { + SourceKind::Git(_) => { + let precise = vrl_source + .precise() + .expect("git reference should have precise") + .to_string(); + ( + precise.clone(), + Some(format!("{}/tree/{precise}", vrl_source.url())), + ) + } + SourceKind::Registry if vrl_source.is_default_registry() => { + let version = vrl_dep.version.to_string(); + ( + version.clone(), + Some(format!("https://crates.io/crates/vrl/{version}")), + ) + } + SourceKind::Path => { + ( + vrl_dep.version.to_string(), + None, + ) + } + kind => unimplemented!("unhandled source kind: {:?}", kind), } - SourceKind::Path - | SourceKind::Registry - | SourceKind::SparseRegistry - | SourceKind::LocalRegistry - | SourceKind::Directory => unimplemented!("unhandled source kind: {:?}", vrl_source.kind()), - _ => unimplemented!("unknown source kind: {:?}", vrl_source.kind()), + } else { + ( + vrl_dep.version.to_string(), + None, + ) }; output_file .write_all(create_const_statement("VRL_VERSION", &version).as_bytes()) .expect("Failed to write VRL version constant"); + let link_str = link.as_deref().unwrap_or(""); output_file - .write_all(create_const_statement("VRL_LINK", &link).as_bytes()) + .write_all(create_const_statement("VRL_LINK", link_str).as_bytes()) .expect("Failed to write VRL_LINK constant"); } From 0592b7342f41b273f080074c8e8dc679d773eff7 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 12 Aug 2025 14:41:53 -0400 Subject: [PATCH 12/49] avoid duplication in tests --- lib/vector-core/src/schema/definition.rs | 2 + src/sources/opentelemetry/tests.rs | 176 +++++++++-------------- 2 files changed, 67 insertions(+), 111 deletions(-) diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index aff0522606774..a05f8e69a874f 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -560,6 +560,8 @@ mod test_utils { } let actual_metadata_kind = Kind::from(log.metadata().value()); + println!("pavlos actual {actual_metadata_kind:#?}"); + println!("pavlos self {:#?}", self.metadata_kind); if let Err(path) = self.metadata_kind.is_superset(&actual_metadata_kind) { // return Result::Err(format!("Event metadata doesn't match definition.\n\nDefinition type=\n{:?}\n\nActual event metadata type=\n{:?}\n", // self.metadata_kind.debug_info(), actual_metadata_kind.debug_info())); diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index ca0c81b423bfd..f27482ea3df0f 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -24,27 +24,75 @@ use std::time::{SystemTime, UNIX_EPOCH}; use tonic::Request; use vector_lib::config::LogNamespace; use vector_lib::lookup::path; -use vector_lib::opentelemetry::proto::collector::metrics::v1::metrics_service_client::MetricsServiceClient; -use vector_lib::opentelemetry::proto::collector::metrics::v1::ExportMetricsServiceRequest; -use vector_lib::opentelemetry::proto::common::v1::any_value::Value::StringValue; -use vector_lib::opentelemetry::proto::metrics::v1::exponential_histogram_data_point::Buckets; -use vector_lib::opentelemetry::proto::metrics::v1::metric::Data; -use vector_lib::opentelemetry::proto::metrics::v1::summary_data_point::ValueAtQuantile; -use vector_lib::opentelemetry::proto::metrics::v1::{ - AggregationTemporality, ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, Histogram, - HistogramDataPoint, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, Summary, - SummaryDataPoint, -}; -use vector_lib::opentelemetry::proto::resource::v1::Resource; use vector_lib::opentelemetry::proto::{ - collector::logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, - common::v1::{any_value, AnyValue, InstrumentationScope, KeyValue}, + collector::{ + logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, + metrics::v1::{metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest}, + }, + common::v1::{any_value, any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue}, logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, - resource::v1::Resource as OtelResource, + metrics::v1::{ + exponential_histogram_data_point::Buckets, metric::Data, summary_data_point::ValueAtQuantile, AggregationTemporality, + ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, Histogram, HistogramDataPoint, Metric, + NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, + Summary, SummaryDataPoint, + }, + resource::v1::{Resource, Resource as OtelResource}, }; use vrl::value; use warp::http::HeaderMap; +fn create_test_logs_request() -> Request { + Request::new(ExportLogsServiceRequest { + resource_logs: vec![ResourceLogs { + resource: Some(OtelResource { + attributes: vec![KeyValue { + key: "res_key".into(), + value: Some(AnyValue { + value: Some(StringValue("res_val".into())), + }), + }], + dropped_attributes_count: 0, + }), + scope_logs: vec![ScopeLogs { + scope: Some(InstrumentationScope { + name: "some.scope.name".into(), + version: "1.2.3".into(), + attributes: vec![KeyValue { + key: "scope_attr".into(), + value: Some(AnyValue { + value: Some(StringValue("scope_val".into())), + }), + }], + dropped_attributes_count: 7, + }), + log_records: vec![LogRecord { + time_unix_nano: 1, + observed_time_unix_nano: 2, + severity_number: 9, + severity_text: "info".into(), + body: Some(AnyValue { + value: Some(StringValue("log body".into())), + }), + attributes: vec![KeyValue { + key: "attr_key".into(), + value: Some(AnyValue { + value: Some(StringValue("attr_val".into())), + }), + }], + dropped_attributes_count: 3, + flags: 4, + // opentelemetry sdk will hex::decode the given trace_id and span_id + trace_id: str_into_hex_bytes("4ac52aadf321c2e531db005df08792f5"), + span_id: str_into_hex_bytes("0b9e4bda2a55530d"), + }], + schema_url: "v1".into(), + }], + schema_url: "v1".into(), + }], + }) +} + #[test] fn generate_config() { test_util::test_generate_config::(); @@ -64,54 +112,7 @@ async fn receive_grpc_logs_vector_namespace() { let mut client = LogsServiceClient::connect(format!("http://{}", env.grpc_addr)) .await .unwrap(); - let req = Request::new(ExportLogsServiceRequest { - resource_logs: vec![ResourceLogs { - resource: Some(OtelResource { - attributes: vec![KeyValue { - key: "res_key".into(), - value: Some(AnyValue { - value: Some(StringValue("res_val".into())), - }), - }], - dropped_attributes_count: 0, - }), - scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: "some.scope.name".into(), - version: "1.2.3".into(), - attributes: vec![KeyValue { - key: "scope_attr".into(), - value: Some(AnyValue { - value: Some(StringValue("scope_val".into())), - }), - }], - dropped_attributes_count: 7, - }), - log_records: vec![LogRecord { - time_unix_nano: 1, - observed_time_unix_nano: 2, - severity_number: 9, - severity_text: "info".into(), - body: Some(AnyValue { - value: Some(StringValue("log body".into())), - }), - attributes: vec![KeyValue { - key: "attr_key".into(), - value: Some(AnyValue { - value: Some(StringValue("attr_val".into())), - }), - }], - dropped_attributes_count: 3, - flags: 4, - // opentelemetry sdk will hex::decode the given trace_id and span_id - trace_id: str_into_hex_bytes("4ac52aadf321c2e531db005df08792f5"), - span_id: str_into_hex_bytes("0b9e4bda2a55530d"), - }], - schema_url: "v1".into(), - }], - schema_url: "v1".into(), - }], - }); + let req = create_test_logs_request(); _ = client.export(req).await; let mut output = test_util::collect_ready(env.output).await; // we just send one, so only one output @@ -209,54 +210,7 @@ async fn receive_grpc_logs_legacy_namespace() { let mut client = LogsServiceClient::connect(format!("http://{}", env.grpc_addr)) .await .unwrap(); - let req = Request::new(ExportLogsServiceRequest { - resource_logs: vec![ResourceLogs { - resource: Some(OtelResource { - attributes: vec![KeyValue { - key: "res_key".into(), - value: Some(AnyValue { - value: Some(StringValue("res_val".into())), - }), - }], - dropped_attributes_count: 0, - }), - scope_logs: vec![ScopeLogs { - scope: Some(InstrumentationScope { - name: "some.scope.name".into(), - version: "1.2.3".into(), - attributes: vec![KeyValue { - key: "scope_attr".into(), - value: Some(AnyValue { - value: Some(StringValue("scope_val".into())), - }), - }], - dropped_attributes_count: 7, - }), - log_records: vec![LogRecord { - time_unix_nano: 1, - observed_time_unix_nano: 2, - severity_number: 9, - severity_text: "info".into(), - body: Some(AnyValue { - value: Some(StringValue("log body".into())), - }), - attributes: vec![KeyValue { - key: "attr_key".into(), - value: Some(AnyValue { - value: Some(StringValue("attr_val".into())), - }), - }], - dropped_attributes_count: 3, - flags: 4, - // opentelemetry sdk will hex::decode the given trace_id and span_id - trace_id: str_into_hex_bytes("4ac52aadf321c2e531db005df08792f5"), - span_id: str_into_hex_bytes("0b9e4bda2a55530d"), - }], - schema_url: "v1".into(), - }], - schema_url: "v1".into(), - }], - }); + let req = create_test_logs_request(); _ = client.export(req).await; let mut output = test_util::collect_ready(env.output).await; // we just send one, so only one output From 200bb7936443119811e35f0dbdc05be071cb735b Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 12 Aug 2025 14:49:49 -0400 Subject: [PATCH 13/49] dbg tests --- lib/vector-core/src/schema/definition.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index a05f8e69a874f..37bc4d154ea70 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -552,6 +552,8 @@ mod test_utils { let actual_kind = Kind::from(log.value()); if let Err(path) = self.event_kind.is_superset(&actual_kind) { + println!("pavlos actual {actual_kind:#?}"); + println!("pavlos self {:#?}", self.event_kind); return Result::Err(format!("Event value doesn't match at path: {}\n\nEvent type at path = {:?}\n\nDefinition at path = {:?}", path, actual_kind.at_path(&path).debug_info(), From a763a80458ebdea9729a16b4fa3c1c01db3505df Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 12 Aug 2025 14:49:49 -0400 Subject: [PATCH 14/49] dbg tests --- src/sources/opentelemetry/tests.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index f27482ea3df0f..4de8c28df2ddb 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -204,7 +204,7 @@ async fn receive_grpc_logs_legacy_namespace() { .config .outputs(LogNamespace::Legacy) .remove(0) - .schema_definition(true); + .schema_definition(true).unwrap(); // send request via grpc client let mut client = LogsServiceClient::connect(format!("http://{}", env.grpc_addr)) @@ -216,9 +216,7 @@ async fn receive_grpc_logs_legacy_namespace() { // we just send one, so only one output assert_eq!(output.len(), 1); let actual_event = output.pop().unwrap(); - schema_definitions - .unwrap() - .assert_valid_for_event(&actual_event); + schema_definitions.assert_valid_for_event(&actual_event); let expect_vec = vec_into_btmap(vec![ ( "attributes", From 8dc36742d5145bce89581431d8deeea5bffaba3a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 18 Aug 2025 14:08:24 -0400 Subject: [PATCH 15/49] ran cargo fmt --- lib/codecs/src/decoding/format/protobuf.rs | 3 ++- lib/vector-vrl/web-playground/build.rs | 12 ++-------- src/sources/opentelemetry/config.rs | 28 +++++++++++++++------- src/sources/opentelemetry/grpc.rs | 9 +++++-- src/sources/opentelemetry/http.rs | 4 +--- src/sources/opentelemetry/tests.rs | 15 +++++++----- 6 files changed, 40 insertions(+), 31 deletions(-) diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 19b69a71e58d4..9f52f83c18f81 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -92,7 +92,8 @@ impl ProtobufDeserializer { /// Creates a new deserializer instance using the descriptor bytes directly. pub fn new_from_bytes(desc_bytes: &[u8], message_type: &str) -> vector_common::Result { - let message_descriptor = vrl::protobuf::get_message_descriptor_from_bytes(desc_bytes, message_type)?; + let message_descriptor = + vrl::protobuf::get_message_descriptor_from_bytes(desc_bytes, message_type)?; Ok(Self { message_descriptor }) } } diff --git a/lib/vector-vrl/web-playground/build.rs b/lib/vector-vrl/web-playground/build.rs index 0fff8acd4f494..9f6bc26293068 100644 --- a/lib/vector-vrl/web-playground/build.rs +++ b/lib/vector-vrl/web-playground/build.rs @@ -74,19 +74,11 @@ fn write_vrl_constants(lockfile: &Lockfile, output_file: &mut File) { Some(format!("https://crates.io/crates/vrl/{version}")), ) } - SourceKind::Path => { - ( - vrl_dep.version.to_string(), - None, - ) - } + SourceKind::Path => (vrl_dep.version.to_string(), None), kind => unimplemented!("unhandled source kind: {:?}", kind), } } else { - ( - vrl_dep.version.to_string(), - None, - ) + (vrl_dep.version.to_string(), None) }; output_file diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 1d8542ded7ba2..979da68a0f802 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -42,9 +42,12 @@ pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; pub const TRACES: &str = "traces"; -pub const OTEL_PROTO_LOGS_REQUEST: &str = "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest"; -pub const OTEL_PROTO_TRACES_REQUEST: &str = "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest"; -pub const OTEL_PROTO_METRICS_REQUEST: &str = "opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest"; +pub const OTEL_PROTO_LOGS_REQUEST: &str = + "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest"; +pub const OTEL_PROTO_TRACES_REQUEST: &str = + "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest"; +pub const OTEL_PROTO_METRICS_REQUEST: &str = + "opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest"; /// Configuration for the `opentelemetry` source. #[configurable_component(source("opentelemetry", "Receive OTLP data through gRPC or HTTP."))] @@ -154,11 +157,19 @@ impl GenerateConfig for OpentelemetryConfig { } impl OpentelemetryConfig { - fn get_deserializer(&self, message_type: &str) -> vector_common::Result> { + fn get_deserializer( + &self, + message_type: &str, + ) -> vector_common::Result> { if self.use_oltp_decoding { - let deserializer = ProtobufDeserializer::new_from_bytes(vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, message_type)?; + let deserializer = ProtobufDeserializer::new_from_bytes( + vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, + message_type, + )?; Ok(Some(deserializer)) - } else { Ok(None) } + } else { + Ok(None) + } } } @@ -191,9 +202,8 @@ impl SourceConfig for OpentelemetryConfig { events_received: events_received.clone(), deserializer: metric_deserializer, }) - .accept_compressed(CompressionEncoding::Gzip) - .max_decoding_message_size(usize::MAX); - + .accept_compressed(CompressionEncoding::Gzip) + .max_decoding_message_size(usize::MAX); let trace_deserializer = self.get_deserializer(OTEL_PROTO_TRACES_REQUEST)?; let trace_service = TraceServiceServer::new(Service { diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 63fcf3439ab18..1e85a5824e0a0 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -8,7 +8,9 @@ use prost::Message; use std::path::PathBuf; use tonic::{Request, Response, Status}; use vector_lib::codecs::decoding::format::Deserializer; -use vector_lib::codecs::decoding::{ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions}; +use vector_lib::codecs::decoding::{ + ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions, +}; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ @@ -46,7 +48,10 @@ impl TraceService for Service { let events = if let Some(deserializer) = self.deserializer.as_ref() { let raw_bytes = request.get_ref().encode_to_vec(); let bytes = bytes::Bytes::from(raw_bytes); - deserializer.parse(bytes, self.log_namespace).map_err(|e| Status::invalid_argument(e.to_string()))?.into_vec() + deserializer + .parse(bytes, self.log_namespace) + .map_err(|e| Status::invalid_argument(e.to_string()))? + .into_vec() } else { request .into_inner() diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index b0690bb4647f4..1cff48aa5d4c8 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -164,9 +164,7 @@ fn build_warp_log_filter( let events = if let Some(deserializer) = deserializer.as_ref() { deserializer .parse(body, log_namespace) - .map(|result| { - result.into_vec() - }) + .map(|result| result.into_vec()) .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) } else { decode(encoding_header.as_deref(), body) diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 4de8c28df2ddb..6471ca4be00d7 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -29,13 +29,15 @@ use vector_lib::opentelemetry::proto::{ logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, metrics::v1::{metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest}, }, - common::v1::{any_value, any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue}, + common::v1::{ + any_value, any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue, + }, logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, metrics::v1::{ - exponential_histogram_data_point::Buckets, metric::Data, summary_data_point::ValueAtQuantile, AggregationTemporality, - ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, Histogram, HistogramDataPoint, Metric, - NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, - Summary, SummaryDataPoint, + exponential_histogram_data_point::Buckets, metric::Data, + summary_data_point::ValueAtQuantile, AggregationTemporality, ExponentialHistogram, + ExponentialHistogramDataPoint, Gauge, Histogram, HistogramDataPoint, Metric, + NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, Summary, SummaryDataPoint, }, resource::v1::{Resource, Resource as OtelResource}, }; @@ -204,7 +206,8 @@ async fn receive_grpc_logs_legacy_namespace() { .config .outputs(LogNamespace::Legacy) .remove(0) - .schema_definition(true).unwrap(); + .schema_definition(true) + .unwrap(); // send request via grpc client let mut client = LogsServiceClient::connect(format!("http://{}", env.grpc_addr)) From de174455f6720cb484de8463b6783fd93cd71fb4 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 18 Aug 2025 14:08:40 -0400 Subject: [PATCH 16/49] chore(dev): cargo vdev build licenses --- LICENSE-3rdparty.csv | 1 - 1 file changed, 1 deletion(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index f4c960eb06064..70dea50b1f1e1 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -722,7 +722,6 @@ uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix -vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton From c4366c7dc1138cc53fb5d3db885d13955532729a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Mon, 18 Aug 2025 14:13:36 -0400 Subject: [PATCH 17/49] wip - cleanup --- lib/codecs/src/decoding/format/mod.rs | 9 +- lib/codecs/src/decoding/format/protobuf.rs | 30 +++++-- lib/vector-core/src/event/trace.rs | 7 ++ src/sources/opentelemetry/config.rs | 1 + src/sources/opentelemetry/grpc.rs | 55 +++++-------- src/sources/opentelemetry/http.rs | 95 +++++++++++----------- 6 files changed, 109 insertions(+), 88 deletions(-) diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 9e2dee7de1ce2..32b26ba3b6aa4 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -15,8 +15,8 @@ mod protobuf; mod syslog; mod vrl; -use ::bytes::Bytes; pub use avro::{AvroDeserializer, AvroDeserializerConfig, AvroDeserializerOptions}; +use ::bytes::Bytes; use dyn_clone::DynClone; pub use gelf::{GelfDeserializer, GelfDeserializerConfig, GelfDeserializerOptions}; pub use influxdb::{InfluxdbDeserializer, InfluxdbDeserializerConfig}; @@ -44,11 +44,18 @@ pub trait Deserializer: DynClone + Send + Sync { /// frame can potentially hold multiple events, e.g. when parsing a JSON /// array. However, we optimize the most common case of emitting one event /// by not requiring heap allocations for it. + /// + /// **Note**: The type of the produced events depends on the implementation. fn parse( &self, bytes: Bytes, log_namespace: LogNamespace, ) -> vector_common::Result>; + + /// Parses trace events from bytes. + fn parse_traces(&self, _bytes: Bytes) -> vector_common::Result> { + unimplemented!() + } } dyn_clone::clone_trait_object!(Deserializer); diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 9f52f83c18f81..849dc988ddfde 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -6,13 +6,13 @@ use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; use smallvec::{smallvec, SmallVec}; use vector_config::configurable_component; -use vector_core::event::LogEvent; +use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ config::{log_schema, DataType, LogNamespace}, event::Event, schema, }; -use vrl::value::Kind; +use vrl::value::{Kind, Value}; use super::Deserializer; @@ -98,18 +98,28 @@ impl ProtobufDeserializer { } } +fn extract_vrl_value( + bytes: Bytes, + message_descriptor: &MessageDescriptor, +) -> vector_common::Result { + let dynamic_message = DynamicMessage::decode(message_descriptor.clone(), bytes) + .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; + + Ok(vrl::protobuf::proto_to_value( + &prost_reflect::Value::Message(dynamic_message), + None, + )?) +} + impl Deserializer for ProtobufDeserializer { fn parse( &self, bytes: Bytes, log_namespace: LogNamespace, ) -> vector_common::Result> { - let dynamic_message = DynamicMessage::decode(self.message_descriptor.clone(), bytes) - .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; + let vrl_value = extract_vrl_value(bytes, &self.message_descriptor)?; + let mut event = Event::Log(LogEvent::from(vrl_value)); - let proto_vrl = - vrl::protobuf::proto_to_value(&prost_reflect::Value::Message(dynamic_message), None)?; - let mut event = Event::Log(LogEvent::from(proto_vrl)); let event = match log_namespace { LogNamespace::Vector => event, LogNamespace::Legacy => { @@ -126,6 +136,12 @@ impl Deserializer for ProtobufDeserializer { Ok(smallvec![event]) } + + fn parse_traces(&self, bytes: Bytes) -> vector_common::Result> { + let vrl_value = extract_vrl_value(bytes, &self.message_descriptor)?; + let trace_event = Event::Trace(TraceEvent::from(vrl_value)); + Ok(smallvec![trace_event]) + } } impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer { diff --git a/lib/vector-core/src/event/trace.rs b/lib/vector-core/src/event/trace.rs index 3956a9532315b..40a016af7adad 100644 --- a/lib/vector-core/src/event/trace.rs +++ b/lib/vector-core/src/event/trace.rs @@ -118,6 +118,13 @@ impl TraceEvent { } } +impl From for TraceEvent { + fn from(value: Value) -> Self { + let log_event = LogEvent::from(value); + Self(log_event) + } +} + impl From for TraceEvent { fn from(log: LogEvent) -> Self { Self(log) diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 079a1f844da86..2c86e914a065b 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -6,6 +6,7 @@ use futures_util::{future::join, TryFutureExt}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ + codecs::decoding::ProtobufDeserializer, config::{log_schema, LegacyKey, LogNamespace}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 2e28a98785d74..2bf70db93f1fa 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,17 +1,14 @@ -use crate::sources::opentelemetry::config::{LOGS, METRICS, TRACES}; +use crate::sources::opentelemetry::config::METRICS; use crate::{ internal_events::{EventsReceived, StreamClosedError}, - sources::opentelemetry::config::{LOGS, METRICS, TRACES}, + sources::opentelemetry::config::{LOGS, TRACES}, SourceSender, }; use futures::TryFutureExt; use prost::Message; -use std::path::PathBuf; use tonic::{Request, Response, Status}; use vector_lib::codecs::decoding::format::Deserializer; -use vector_lib::codecs::decoding::{ - ProtobufDeserializer, ProtobufDeserializerConfig, ProtobufDeserializerOptions, -}; +use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ @@ -31,8 +28,6 @@ use vector_lib::{ EstimatedJsonEncodedSizeOf, }; - - #[derive(Clone)] pub(super) struct Service { pub pipeline: SourceSender, @@ -52,9 +47,9 @@ impl TraceService for Service { let raw_bytes = request.get_ref().encode_to_vec(); let bytes = bytes::Bytes::from(raw_bytes); deserializer - .parse(bytes, self.log_namespace) - .map_err(|e| Status::invalid_argument(e.to_string()))? - .into_vec() + .parse_traces(bytes) + .map_err(|e| Status::invalid_argument(e.to_string())) + .map(|buf| buf.into_vec())? } else { request .into_inner() @@ -77,28 +72,21 @@ impl LogsService for Service { &self, request: Request, ) -> Result, Status> { - println!("{request:#?}"); - let raw_bytes = request.get_ref().encode_to_vec(); - println!("{raw_bytes:?}"); - - let bytes = bytes::Bytes::from(raw_bytes); - let deserializer = ProtobufDeserializerConfig { - protobuf: ProtobufDeserializerOptions { - desc_file: PathBuf::from("/Users/pavlos.rontidis/CLionProjects/vector/pront/otel/proto/vector-proto-related/otlp.desc"), - message_type: "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest".to_string(), - } - }.build().unwrap(); - let events = deserializer - .parse(bytes, self.log_namespace) - .unwrap() - .into_vec(); - - // let events: Vec = request - // .into_inner() - // .resource_logs - // .into_iter() - // .flat_map(|v| v.into_event_iter(self.log_namespace)) - // .collect(); + let events = if let Some(deserializer) = self.deserializer.as_ref() { + let raw_bytes = request.get_ref().encode_to_vec(); + let bytes = bytes::Bytes::from(raw_bytes); + deserializer + .parse(bytes, self.log_namespace) + .map_err(|e| Status::invalid_argument(e.to_string())) + .map(|buf| buf.into_vec())? + } else { + request + .into_inner() + .resource_logs + .into_iter() + .flat_map(|v| v.into_event_iter(self.log_namespace)) + .collect() + }; self.handle_events(events, LOGS).await?; Ok(Response::new(ExportLogsServiceResponse { @@ -113,6 +101,7 @@ impl MetricsService for Service { &self, request: Request, ) -> Result, Status> { + // Protobuf deserializer doesn't support metrics. let events: Vec = request .into_inner() .resource_metrics diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index e1b2739d42fa6..55459b92bcea7 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -45,23 +45,6 @@ use warp::{ filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, Filter, Reply, }; -use crate::common::http::ErrorMessage; -use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; -use crate::sources::http_server::HttpConfigParamKind; -use crate::sources::util::add_headers; -use crate::{ - event::Event, - http::build_http_trace_layer, - internal_events::{EventsReceived, StreamClosedError}, - shutdown::ShutdownSignal, - sources::util::decode, - tls::MaybeTlsSettings, - SourceSender, -}; - -use super::{reply::protobuf, status::Status}; -use crate::sources::opentelemetry::config::{OpentelemetryConfig, LOGS, METRICS, TRACES}; - #[derive(Clone, Copy, Debug, Snafu)] pub(crate) enum ApiError { ServerShutdown, @@ -157,17 +140,24 @@ fn enrich_events( ); } -fn build_warp_log_filter( +fn build_ingest_filter( + telemetry_type: &'static str, acknowledgements: bool, - log_namespace: LogNamespace, out: SourceSender, - bytes_received: Registered, - events_received: Registered, - headers: Vec, - deserializer: Option, -) -> BoxedFilter<(Response,)> { + make_events: F, +) -> BoxedFilter<(Response,)> +where + Resp: prost::Message + Default + Send + 'static, + F: Clone + + Send + + Sync + + 'static + + Fn(Option, HeaderMap, Bytes) -> Result, ErrorMessage>, +{ warp::post() - .and(warp::path!("v1" / "logs")) + .and(warp::path("v1")) + .and(warp::path(telemetry_type)) + .and(warp::path::end()) .and(warp::header::exact_ignore_case( "content-type", "application/x-protobuf", @@ -176,38 +166,49 @@ fn build_warp_log_filter( .and(warp::header::headers_cloned()) .and(warp::body::bytes()) .and_then( - move |encoding_header: Option, headers_config: HeaderMap, body: Bytes| { - println!("📦 Raw bytes (len={}): {:02x?}", body.len(), body); - let events = if let Some(deserializer) = deserializer.as_ref() { - deserializer - .parse(body, log_namespace) - .map(|result| result.into_vec()) - .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) - } else { - decode(encoding_header.as_deref(), body) - .and_then(|body| { - bytes_received.emit(ByteSize(body.len())); - decode_log_body(body, log_namespace, &events_received) - }) - .map(|mut events| { - enrich_events(&mut events, &headers, &headers_config, log_namespace); - events - }) - }; - - println!("{:#?}", events); + move |encoding_header: Option, headers: HeaderMap, body: Bytes| { + let events = make_events(encoding_header, headers, body); handle_request( events, acknowledgements, out.clone(), - LOGS, - ExportLogsServiceResponse::default(), + telemetry_type, + Resp::default(), ) }, ) .boxed() } +fn build_warp_log_filter( + acknowledgements: bool, + log_namespace: LogNamespace, + out: SourceSender, + bytes_received: Registered, + events_received: Registered, + headers_cfg: Vec, + deserializer: Option, +) -> BoxedFilter<(Response,)> { + let make_events = move |encoding_header: Option, headers: HeaderMap, body: Bytes| { + if let Some(d) = deserializer.as_ref() { + d.parse(body, log_namespace) + .map(|r| r.into_vec()) + .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) + } else { + decode(encoding_header.as_deref(), body) + .and_then(|body| { + bytes_received.emit(ByteSize(body.len())); + decode_log_body(body, log_namespace, &events_received) + }) + .map(|mut events| { + enrich_events(&mut events, &headers_cfg, &headers, log_namespace); + events + }) + } + }; + + build_ingest_filter::(LOGS, acknowledgements, out, make_events) +} fn build_warp_metrics_filter( acknowledgements: bool, out: SourceSender, From 4e7f5d9832c943bd1f13d5cbf0008460c06ba1e8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 19 Aug 2025 13:30:38 -0400 Subject: [PATCH 18/49] ran cargo fmt --- lib/codecs/src/decoding/format/mod.rs | 2 +- lib/vector-core/src/schema/definition.rs | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/codecs/src/decoding/format/mod.rs b/lib/codecs/src/decoding/format/mod.rs index 32b26ba3b6aa4..783aa5d79d19d 100644 --- a/lib/codecs/src/decoding/format/mod.rs +++ b/lib/codecs/src/decoding/format/mod.rs @@ -15,8 +15,8 @@ mod protobuf; mod syslog; mod vrl; -pub use avro::{AvroDeserializer, AvroDeserializerConfig, AvroDeserializerOptions}; use ::bytes::Bytes; +pub use avro::{AvroDeserializer, AvroDeserializerConfig, AvroDeserializerOptions}; use dyn_clone::DynClone; pub use gelf::{GelfDeserializer, GelfDeserializerConfig, GelfDeserializerOptions}; pub use influxdb::{InfluxdbDeserializer, InfluxdbDeserializerConfig}; diff --git a/lib/vector-core/src/schema/definition.rs b/lib/vector-core/src/schema/definition.rs index 37bc4d154ea70..aff0522606774 100644 --- a/lib/vector-core/src/schema/definition.rs +++ b/lib/vector-core/src/schema/definition.rs @@ -552,8 +552,6 @@ mod test_utils { let actual_kind = Kind::from(log.value()); if let Err(path) = self.event_kind.is_superset(&actual_kind) { - println!("pavlos actual {actual_kind:#?}"); - println!("pavlos self {:#?}", self.event_kind); return Result::Err(format!("Event value doesn't match at path: {}\n\nEvent type at path = {:?}\n\nDefinition at path = {:?}", path, actual_kind.at_path(&path).debug_info(), @@ -562,8 +560,6 @@ mod test_utils { } let actual_metadata_kind = Kind::from(log.metadata().value()); - println!("pavlos actual {actual_metadata_kind:#?}"); - println!("pavlos self {:#?}", self.metadata_kind); if let Err(path) = self.metadata_kind.is_superset(&actual_metadata_kind) { // return Result::Err(format!("Event metadata doesn't match definition.\n\nDefinition type=\n{:?}\n\nActual event metadata type=\n{:?}\n", // self.metadata_kind.debug_info(), actual_metadata_kind.debug_info())); From 4c9b14374da81ef9a2dc8362cba1eff23bc3e98f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 19 Aug 2025 13:59:34 -0400 Subject: [PATCH 19/49] chore(deps): cargo update -p vrl --- Cargo.lock | 7 ++-- Cargo.toml | 6 ++-- src/sources/opentelemetry/http.rs | 58 +++++++++++++++++-------------- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a0737ef24d5a..ca8e5f91d3742 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4831,7 +4831,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.4.10", "tokio", "tower-service", "tracing 0.1.41", @@ -7993,7 +7993,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -8039,7 +8039,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2 1.0.95", "quote 1.0.40", "syn 2.0.104", @@ -12425,6 +12425,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.26.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#77b47df5968f38204d8e3e4a58ce6da4ffe12a34" dependencies = [ "aes", "aes-siv", diff --git a/Cargo.toml b/Cargo.toml index a5d5f0fa1e72b..8cfe163687111 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -178,7 +178,7 @@ vector-lib = { path = "lib/vector-lib", default-features = false, features = ["v vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { path = "../vrl", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] cfg-if.workspace = true @@ -199,7 +199,7 @@ dnstap-parser = { path = "lib/dnstap-parser", optional = true } fakedata = { path = "lib/fakedata", optional = true } portpicker = { path = "lib/portpicker" } tracing-limit = { path = "lib/tracing-limit" } -vector-common = { path = "lib/vector-common", default-features = false} +vector-common = { path = "lib/vector-common", default-features = false } vector-lib.workspace = true vector-config.workspace = true vector-config-common.workspace = true @@ -386,7 +386,7 @@ seahash = { version = "4.1.0", default-features = false } smallvec = { version = "1", default-features = false, features = ["union", "serde"] } snap = { version = "1.1.1", default-features = false } socket2.workspace = true -sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional=true } +sqlx = { version = "0.8.6", default-features = false, features = ["derive", "postgres", "chrono", "runtime-tokio"], optional = true } stream-cancel = { version = "0.8.2", default-features = false } strip-ansi-escapes = { version = "0.2.1", default-features = false } syslog = { version = "6.1.1", default-features = false, optional = true } diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 55459b92bcea7..3c4047ba1848f 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -30,10 +30,11 @@ use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Registered, }; +use vector_lib::opentelemetry::proto::collector::trace::v1::ExportTraceServiceResponse; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ExportLogsServiceRequest, ExportLogsServiceResponse}, metrics::v1::{ExportMetricsServiceRequest, ExportMetricsServiceResponse}, - trace::v1::{ExportTraceServiceRequest, ExportTraceServiceResponse}, + trace::v1::ExportTraceServiceRequest, }; use vector_lib::tls::MaybeTlsIncomingStream; use vector_lib::{ @@ -103,7 +104,7 @@ pub(crate) fn build_warp_filter( bytes_received.clone(), events_received.clone(), headers.clone(), - deserializer, + deserializer.clone(), ); let metrics_filters = build_warp_metrics_filter( acknowledgements, @@ -116,6 +117,7 @@ pub(crate) fn build_warp_filter( out.clone(), bytes_received, events_received, + deserializer, ); log_filters .or(trace_filters) @@ -183,7 +185,7 @@ where fn build_warp_log_filter( acknowledgements: bool, log_namespace: LogNamespace, - out: SourceSender, + source_sender: SourceSender, bytes_received: Registered, events_received: Registered, headers_cfg: Vec, @@ -207,11 +209,16 @@ fn build_warp_log_filter( } }; - build_ingest_filter::(LOGS, acknowledgements, out, make_events) + build_ingest_filter::( + LOGS, + acknowledgements, + source_sender, + make_events, + ) } fn build_warp_metrics_filter( acknowledgements: bool, - out: SourceSender, + source_sender: SourceSender, bytes_received: Registered, events_received: Registered, ) -> BoxedFilter<(Response,)> { @@ -232,7 +239,7 @@ fn build_warp_metrics_filter( handle_request( events, acknowledgements, - out.clone(), + source_sender.clone(), METRICS, ExportMetricsServiceResponse::default(), ) @@ -242,33 +249,30 @@ fn build_warp_metrics_filter( fn build_warp_trace_filter( acknowledgements: bool, - out: SourceSender, + source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + deserializer: Option, ) -> BoxedFilter<(Response,)> { - warp::post() - .and(warp::path!("v1" / "traces")) - .and(warp::header::exact_ignore_case( - "content-type", - "application/x-protobuf", - )) - .and(warp::header::optional::("content-encoding")) - .and(warp::body::bytes()) - .and_then(move |encoding_header: Option, body: Bytes| { - let events = decode(encoding_header.as_deref(), body).and_then(|body| { + let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + if let Some(d) = deserializer.as_ref() { + d.parse_traces(body) + .map(|r| r.into_vec()) + .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) + } else { + decode(encoding_header.as_deref(), body).and_then(|body| { bytes_received.emit(ByteSize(body.len())); decode_trace_body(body, &events_received) - }); + }) + } + }; - handle_request( - events, - acknowledgements, - out.clone(), - TRACES, - ExportTraceServiceResponse::default(), - ) - }) - .boxed() + build_ingest_filter::( + TRACES, + acknowledgements, + source_sender, + make_events, + ) } fn decode_trace_body( From e6dab618f8dfa05e0ec998979d54384454230fa9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 19 Aug 2025 14:03:59 -0400 Subject: [PATCH 20/49] chore(dev): cargo vdev build licenses --- LICENSE-3rdparty.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 65671cc9e8574..567df49b9eba4 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -724,6 +724,7 @@ uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix +vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton From c0ac421f03e40451ed039be503889785d84b7dff Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 19 Aug 2025 17:02:23 -0400 Subject: [PATCH 21/49] e2e - wip --- scripts/e2e/opentelemetry-logs/compose.yaml | 2 +- scripts/e2e/opentelemetry-logs/test.yaml | 4 +- .../e2e/opentelemetry/logs/output/.gitignore | 0 .../logs/vector_oltp_source.yaml | 48 +++++++++++++++++++ 4 files changed, 51 insertions(+), 3 deletions(-) mode change 100644 => 100755 tests/data/e2e/opentelemetry/logs/output/.gitignore create mode 100644 tests/data/e2e/opentelemetry/logs/vector_oltp_source.yaml diff --git a/scripts/e2e/opentelemetry-logs/compose.yaml b/scripts/e2e/opentelemetry-logs/compose.yaml index c82b2e8a569d8..18fc4588a73b7 100644 --- a/scripts/e2e/opentelemetry-logs/compose.yaml +++ b/scripts/e2e/opentelemetry-logs/compose.yaml @@ -57,7 +57,7 @@ services: init: true volumes: - type: bind - source: ../../../tests/data/e2e/opentelemetry/logs/vector.yaml + source: ../../../tests/data/e2e/opentelemetry/logs/${CONFIG_VECTOR_CONFIG} target: /etc/vector/vector.yaml read_only: true - type: bind diff --git a/scripts/e2e/opentelemetry-logs/test.yaml b/scripts/e2e/opentelemetry-logs/test.yaml index 70e3862ae3347..be8a350fcfe2c 100644 --- a/scripts/e2e/opentelemetry-logs/test.yaml +++ b/scripts/e2e/opentelemetry-logs/test.yaml @@ -1,5 +1,5 @@ features: -- e2e-tests-opentelemetry + - e2e-tests-opentelemetry test: "e2e" @@ -13,7 +13,7 @@ runner: matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use - collector_version: [ 'latest' ] + vector_config: [ 'vector.yaml', 'vector_oltp_source.yaml' ] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/tests/data/e2e/opentelemetry/logs/output/.gitignore b/tests/data/e2e/opentelemetry/logs/output/.gitignore old mode 100644 new mode 100755 diff --git a/tests/data/e2e/opentelemetry/logs/vector_oltp_source.yaml b/tests/data/e2e/opentelemetry/logs/vector_oltp_source.yaml new file mode 100644 index 0000000000000..cedaa85626876 --- /dev/null +++ b/tests/data/e2e/opentelemetry/logs/vector_oltp_source.yaml @@ -0,0 +1,48 @@ +sources: + source0: + type: opentelemetry + grpc: + address: 0.0.0.0:4317 + http: + address: 0.0.0.0:4318 + keepalive: + max_connection_age_jitter_factor: 0.1 + max_connection_age_secs: 300 + use_otlp_decoding: true + + internal_metrics: + type: internal_metrics + scrape_interval_secs: 60 + +sinks: + otel_sink: + inputs: [ "source0.logs" ] + type: opentelemetry + protocol: + type: http + uri: http://otel-collector-sink:5318/v1/logs + method: post + encoding: + codec: json + framing: + method: newline_delimited + batch: + max_events: 1 + request: + headers: + content-type: application/json + + otel_file_sink: + type: file + path: "/output/vector-file-sink.log" + inputs: + - remap_otel + encoding: + codec: json + + metrics_console_sink: + type: console + inputs: + - internal_metrics + encoding: + codec: json From 1f5fa7dbfbbd14029176ac2b7b556c174d9bc759 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 Aug 2025 09:32:03 -0400 Subject: [PATCH 22/49] cleanup --- scripts/e2e/opentelemetry-logs/test.yaml | 3 ++- src/sinks/http/encoder.rs | 11 ++--------- src/sources/opentelemetry/integration_tests.rs | 18 +++++++++--------- src/sources/util/http/prelude.rs | 15 +++++++-------- .../logs/{vector.yaml => vector_default.yaml} | 0 ...ector_oltp_source.yaml => vector_oltp.yaml} | 0 6 files changed, 20 insertions(+), 27 deletions(-) rename tests/data/e2e/opentelemetry/logs/{vector.yaml => vector_default.yaml} (100%) rename tests/data/e2e/opentelemetry/logs/{vector_oltp_source.yaml => vector_oltp.yaml} (100%) diff --git a/scripts/e2e/opentelemetry-logs/test.yaml b/scripts/e2e/opentelemetry-logs/test.yaml index be8a350fcfe2c..2a430e2e616ef 100644 --- a/scripts/e2e/opentelemetry-logs/test.yaml +++ b/scripts/e2e/opentelemetry-logs/test.yaml @@ -13,7 +13,8 @@ runner: matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use - vector_config: [ 'vector.yaml', 'vector_oltp_source.yaml' ] + collector_version: [ 'latest' ] + vector_config: [ 'vector_default.yaml', 'vector_oltp.yaml' ] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/src/sinks/http/encoder.rs b/src/sinks/http/encoder.rs index 7dfc896bd26d8..a48c1c19151f2 100644 --- a/src/sinks/http/encoder.rs +++ b/src/sinks/http/encoder.rs @@ -2,18 +2,18 @@ use crate::{ event::Event, - sinks::util::encoding::{Encoder as SinkEncoder, write_all}, + sinks::util::encoding::{write_all, Encoder as SinkEncoder}, }; use bytes::{BufMut, BytesMut}; use std::io; use tokio_util::codec::Encoder as _; use vector_lib::codecs::{ - CharacterDelimitedEncoder, encoding::{ Framer, Framer::{CharacterDelimited, NewlineDelimited}, Serializer::Json, }, + CharacterDelimitedEncoder, }; use crate::sinks::prelude::*; @@ -71,10 +71,6 @@ impl SinkEncoder> for HttpEncoder { .map_err(|_| io::Error::other("unable to encode event"))?; } - println!("🚧 encoder: {:?}", self.encoder); - println!("📦 First encoded HTTP payload ({} bytes):", body.len()); - println!("{:?}", body); - match (self.encoder.serializer(), self.encoder.framer()) { (Json(_), NewlineDelimited(_)) => { if !body.is_empty() { @@ -96,9 +92,6 @@ impl SinkEncoder> for HttpEncoder { let body = body.freeze(); - println!("📦 Final encoded HTTP payload ({} bytes):", body.len()); - println!("{:?}", body); - write_all(writer, n_events, body.as_ref()).map(|()| (body.len(), byte_size)) } } diff --git a/src/sources/opentelemetry/integration_tests.rs b/src/sources/opentelemetry/integration_tests.rs index af38d52ad8e83..42ffe1e73c3f6 100644 --- a/src/sources/opentelemetry/integration_tests.rs +++ b/src/sources/opentelemetry/integration_tests.rs @@ -4,14 +4,14 @@ use itertools::Itertools; use serde_json::json; use crate::{ - config::{SourceConfig, SourceContext, log_schema}, + config::{log_schema, SourceConfig, SourceContext}, event::EventStatus, sources::opentelemetry::config::{ - GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, + GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS, TRACES, }, test_util::{ collect_n, - components::{SOURCE_TAGS, assert_source_compliance}, + components::{assert_source_compliance, SOURCE_TAGS}, retry_until, wait_for_tcp, }, }; @@ -20,10 +20,10 @@ use prost::Message; use super::tests::new_source; use vector_lib::opentelemetry::proto::{ collector::{metrics::v1::ExportMetricsServiceRequest, trace::v1::ExportTraceServiceRequest}, - common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value::Value::StringValue}, + common::v1::{any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue}, metrics::v1::{ - Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric::Data, - number_data_point::Value, + metric::Data, number_data_point::Value, Gauge, Metric, NumberDataPoint, ResourceMetrics, + ScopeMetrics, }, resource::v1::Resource, trace::v1::{ResourceSpans, ScopeSpans, Span}, @@ -63,7 +63,7 @@ async fn receive_logs_legacy_namespace() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - decoding: None, + use_oltp_decoding: false, }; let (sender, logs_output, _) = new_source(EventStatus::Delivered, LOGS.to_string()); @@ -162,7 +162,7 @@ async fn receive_trace() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - decoding: None, + use_oltp_decoding: false, }; let (sender, trace_output, _) = new_source(EventStatus::Delivered, TRACES.to_string()); @@ -267,7 +267,7 @@ async fn receive_metric() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - decoding: None, + use_oltp_decoding: false, }; let (sender, metrics_output, _) = new_source(EventStatus::Delivered, METRICS.to_string()); diff --git a/src/sources/util/http/prelude.rs b/src/sources/util/http/prelude.rs index 1e337c7dbc8d4..e9a81a6e6aea8 100644 --- a/src/sources/util/http/prelude.rs +++ b/src/sources/util/http/prelude.rs @@ -1,36 +1,36 @@ -use crate::common::http::{ErrorMessage, server_auth::HttpServerAuthConfig}; +use crate::common::http::{server_auth::HttpServerAuthConfig, ErrorMessage}; use std::{collections::HashMap, convert::Infallible, fmt, net::SocketAddr, time::Duration}; use bytes::Bytes; use futures::{FutureExt, TryFutureExt}; -use hyper::{Server, service::make_service_fn}; +use hyper::{service::make_service_fn, Server}; use tokio::net::TcpStream; use tower::ServiceBuilder; use tracing::Span; use vector_lib::{ - EstimatedJsonEncodedSizeOf, config::SourceAcknowledgementsConfig, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event}, + EstimatedJsonEncodedSizeOf, }; use warp::{ - Filter, filters::{ - BoxedFilter, path::{FullPath, Tail}, + BoxedFilter, }, http::{HeaderMap, StatusCode}, reject::Rejection, + Filter, }; use crate::{ - SourceSender, config::SourceContext, - http::{KeepaliveConfig, MaxConnectionAgeLayer, build_http_trace_layer}, + http::{build_http_trace_layer, KeepaliveConfig, MaxConnectionAgeLayer}, internal_events::{ HttpBadRequest, HttpBytesReceived, HttpEventsReceived, HttpInternalError, StreamClosedError, }, sources::util::http::HttpMethod, tls::{MaybeTlsIncomingStream, MaybeTlsSettings, TlsEnableableConfig}, + SourceSender, }; use super::encoding::decode; @@ -129,7 +129,6 @@ pub trait HttpSource: Clone + Send + Sync + 'static { addr: Option| { debug!(message = "Handling HTTP request.", headers = ?headers); let http_path = path.as_str(); - println!("📦 Raw bytes (len={}): {:02x?}", body.len(), body); let events = auth_matcher .as_ref() .map_or(Ok(()), |a| { diff --git a/tests/data/e2e/opentelemetry/logs/vector.yaml b/tests/data/e2e/opentelemetry/logs/vector_default.yaml similarity index 100% rename from tests/data/e2e/opentelemetry/logs/vector.yaml rename to tests/data/e2e/opentelemetry/logs/vector_default.yaml diff --git a/tests/data/e2e/opentelemetry/logs/vector_oltp_source.yaml b/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml similarity index 100% rename from tests/data/e2e/opentelemetry/logs/vector_oltp_source.yaml rename to tests/data/e2e/opentelemetry/logs/vector_oltp.yaml From e047de6f378bf9de2da3798d2983c3082bdcf88e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 20 Aug 2025 09:38:41 -0400 Subject: [PATCH 23/49] ran cargo fmt --- src/sinks/http/encoder.rs | 4 ++-- src/sources/opentelemetry/config.rs | 10 +++++----- src/sources/opentelemetry/grpc.rs | 14 +++++++------- src/sources/opentelemetry/http.rs | 12 ++++++------ src/sources/opentelemetry/integration_tests.rs | 12 ++++++------ src/sources/util/http/prelude.rs | 14 +++++++------- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/sinks/http/encoder.rs b/src/sinks/http/encoder.rs index a48c1c19151f2..2a033ed1fe6d0 100644 --- a/src/sinks/http/encoder.rs +++ b/src/sinks/http/encoder.rs @@ -2,18 +2,18 @@ use crate::{ event::Event, - sinks::util::encoding::{write_all, Encoder as SinkEncoder}, + sinks::util::encoding::{Encoder as SinkEncoder, write_all}, }; use bytes::{BufMut, BytesMut}; use std::io; use tokio_util::codec::Encoder as _; use vector_lib::codecs::{ + CharacterDelimitedEncoder, encoding::{ Framer, Framer::{CharacterDelimited, NewlineDelimited}, Serializer::Json, }, - CharacterDelimitedEncoder, }; use crate::sinks::prelude::*; diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 2c86e914a065b..13f296a5403a7 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,16 +1,16 @@ use std::net::SocketAddr; use futures::FutureExt; -use futures_util::{future::join, TryFutureExt}; +use futures_util::{TryFutureExt, future::join}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{log_schema, LegacyKey, LogNamespace}, + config::{LegacyKey, LogNamespace, log_schema}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{owned_value_path, OwnedTargetPath}, + lookup::{OwnedTargetPath, owned_value_path}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -34,17 +34,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ + Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, - Source, }, }; -use vrl::value::{kind::Collection, Kind}; +use vrl::value::{Kind, kind::Collection}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 2bf70db93f1fa..7ca1bd5478d64 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,31 +1,31 @@ use crate::sources::opentelemetry::config::METRICS; use crate::{ + SourceSender, internal_events::{EventsReceived, StreamClosedError}, sources::opentelemetry::config::{LOGS, TRACES}, - SourceSender, }; use futures::TryFutureExt; use prost::Message; use tonic::{Request, Response, Status}; -use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::codecs::decoding::ProtobufDeserializer; +use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ - logs_service_server::LogsService, ExportLogsServiceRequest, ExportLogsServiceResponse, + ExportLogsServiceRequest, ExportLogsServiceResponse, logs_service_server::LogsService, }, metrics::v1::{ - metrics_service_server::MetricsService, ExportMetricsServiceRequest, - ExportMetricsServiceResponse, + ExportMetricsServiceRequest, ExportMetricsServiceResponse, + metrics_service_server::MetricsService, }, trace::v1::{ - trace_service_server::TraceService, ExportTraceServiceRequest, ExportTraceServiceResponse, + ExportTraceServiceRequest, ExportTraceServiceResponse, trace_service_server::TraceService, }, }; use vector_lib::{ + EstimatedJsonEncodedSizeOf, config::LogNamespace, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event}, - EstimatedJsonEncodedSizeOf, }; #[derive(Clone)] diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 3c4047ba1848f..9e11826a018bd 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -5,28 +5,28 @@ use super::{reply::protobuf, status::Status}; use crate::common::http::ErrorMessage; use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; use crate::sources::http_server::HttpConfigParamKind; -use crate::sources::opentelemetry::config::{OpentelemetryConfig, LOGS, METRICS, TRACES}; +use crate::sources::opentelemetry::config::{LOGS, METRICS, OpentelemetryConfig, TRACES}; use crate::sources::util::add_headers; use crate::{ + SourceSender, event::Event, http::build_http_trace_layer, internal_events::{EventsReceived, StreamClosedError}, shutdown::ShutdownSignal, sources::util::decode, tls::MaybeTlsSettings, - SourceSender, }; use bytes::Bytes; use futures_util::FutureExt; use http::StatusCode; -use hyper::{service::make_service_fn, Server}; +use hyper::{Server, service::make_service_fn}; use prost::Message; use snafu::Snafu; use tokio::net::TcpStream; use tower::ServiceBuilder; use tracing::Span; -use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::codecs::decoding::ProtobufDeserializer; +use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Registered, }; @@ -38,12 +38,12 @@ use vector_lib::opentelemetry::proto::collector::{ }; use vector_lib::tls::MaybeTlsIncomingStream; use vector_lib::{ + EstimatedJsonEncodedSizeOf, config::LogNamespace, event::{BatchNotifier, BatchStatus}, - EstimatedJsonEncodedSizeOf, }; use warp::{ - filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, Filter, Reply, + Filter, Reply, filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, }; #[derive(Clone, Copy, Debug, Snafu)] diff --git a/src/sources/opentelemetry/integration_tests.rs b/src/sources/opentelemetry/integration_tests.rs index 42ffe1e73c3f6..d7cfed409d1b1 100644 --- a/src/sources/opentelemetry/integration_tests.rs +++ b/src/sources/opentelemetry/integration_tests.rs @@ -4,14 +4,14 @@ use itertools::Itertools; use serde_json::json; use crate::{ - config::{log_schema, SourceConfig, SourceContext}, + config::{SourceConfig, SourceContext, log_schema}, event::EventStatus, sources::opentelemetry::config::{ - GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS, TRACES, + GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, }, test_util::{ collect_n, - components::{assert_source_compliance, SOURCE_TAGS}, + components::{SOURCE_TAGS, assert_source_compliance}, retry_until, wait_for_tcp, }, }; @@ -20,10 +20,10 @@ use prost::Message; use super::tests::new_source; use vector_lib::opentelemetry::proto::{ collector::{metrics::v1::ExportMetricsServiceRequest, trace::v1::ExportTraceServiceRequest}, - common::v1::{any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue}, + common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value::Value::StringValue}, metrics::v1::{ - metric::Data, number_data_point::Value, Gauge, Metric, NumberDataPoint, ResourceMetrics, - ScopeMetrics, + Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric::Data, + number_data_point::Value, }, resource::v1::Resource, trace::v1::{ResourceSpans, ScopeSpans, Span}, diff --git a/src/sources/util/http/prelude.rs b/src/sources/util/http/prelude.rs index e9a81a6e6aea8..a589940b487be 100644 --- a/src/sources/util/http/prelude.rs +++ b/src/sources/util/http/prelude.rs @@ -1,36 +1,36 @@ -use crate::common::http::{server_auth::HttpServerAuthConfig, ErrorMessage}; +use crate::common::http::{ErrorMessage, server_auth::HttpServerAuthConfig}; use std::{collections::HashMap, convert::Infallible, fmt, net::SocketAddr, time::Duration}; use bytes::Bytes; use futures::{FutureExt, TryFutureExt}; -use hyper::{service::make_service_fn, Server}; +use hyper::{Server, service::make_service_fn}; use tokio::net::TcpStream; use tower::ServiceBuilder; use tracing::Span; use vector_lib::{ + EstimatedJsonEncodedSizeOf, config::SourceAcknowledgementsConfig, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event}, - EstimatedJsonEncodedSizeOf, }; use warp::{ + Filter, filters::{ - path::{FullPath, Tail}, BoxedFilter, + path::{FullPath, Tail}, }, http::{HeaderMap, StatusCode}, reject::Rejection, - Filter, }; use crate::{ + SourceSender, config::SourceContext, - http::{build_http_trace_layer, KeepaliveConfig, MaxConnectionAgeLayer}, + http::{KeepaliveConfig, MaxConnectionAgeLayer, build_http_trace_layer}, internal_events::{ HttpBadRequest, HttpBytesReceived, HttpEventsReceived, HttpInternalError, StreamClosedError, }, sources::util::http::HttpMethod, tls::{MaybeTlsIncomingStream, MaybeTlsSettings, TlsEnableableConfig}, - SourceSender, }; use super::encoding::decode; From 32f2188c1c9a55b5863a0ce1b5fb19c4b6ae4da3 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 21 Aug 2025 15:31:53 -0400 Subject: [PATCH 24/49] set default --- scripts/e2e/opentelemetry-logs/compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/e2e/opentelemetry-logs/compose.yaml b/scripts/e2e/opentelemetry-logs/compose.yaml index 18fc4588a73b7..80cb7695f45b5 100644 --- a/scripts/e2e/opentelemetry-logs/compose.yaml +++ b/scripts/e2e/opentelemetry-logs/compose.yaml @@ -57,7 +57,7 @@ services: init: true volumes: - type: bind - source: ../../../tests/data/e2e/opentelemetry/logs/${CONFIG_VECTOR_CONFIG} + source: ../../../tests/data/e2e/opentelemetry/logs/${CONFIG_VECTOR_CONFIG:-vector_default.yaml} target: /etc/vector/vector.yaml read_only: true - type: bind From f11e27fe8477aa374e20b615eaeb708ce10d9e49 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 22 Aug 2025 09:52:20 -0400 Subject: [PATCH 25/49] wip --- scripts/e2e/opentelemetry-logs/compose.yaml | 6 ++-- vdev/src/testing/build.rs | 19 ++++++----- vdev/src/testing/integration.rs | 35 ++++++++++++++++----- vdev/src/testing/runner.rs | 35 ++++++++++++--------- 4 files changed, 63 insertions(+), 32 deletions(-) diff --git a/scripts/e2e/opentelemetry-logs/compose.yaml b/scripts/e2e/opentelemetry-logs/compose.yaml index 80cb7695f45b5..d721abf6e7aa7 100644 --- a/scripts/e2e/opentelemetry-logs/compose.yaml +++ b/scripts/e2e/opentelemetry-logs/compose.yaml @@ -2,7 +2,7 @@ name: opentelemetry-vector-e2e services: otel-collector-source: container_name: otel-collector-source - image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION:-latest} + image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION} init: true volumes: - type: bind @@ -33,7 +33,7 @@ services: otel-collector-sink: container_name: otel-collector-sink - image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION:-latest} + image: otel/opentelemetry-collector-contrib:${CONFIG_COLLECTOR_VERSION} init: true volumes: - type: bind @@ -57,7 +57,7 @@ services: init: true volumes: - type: bind - source: ../../../tests/data/e2e/opentelemetry/logs/${CONFIG_VECTOR_CONFIG:-vector_default.yaml} + source: ../../../tests/data/e2e/opentelemetry/logs/${CONFIG_VECTOR_CONFIG} target: /etc/vector/vector.yaml read_only: true - type: bind diff --git a/vdev/src/testing/build.rs b/vdev/src/testing/build.rs index d3b0f213aadf7..d750f35ad48a4 100644 --- a/vdev/src/testing/build.rs +++ b/vdev/src/testing/build.rs @@ -1,7 +1,8 @@ use crate::app; use crate::app::CommandExt; -use crate::testing::config::RustToolchainConfig; +use crate::testing::config::{Environment, RustToolchainConfig}; use crate::testing::docker::docker_command; +use crate::testing::integration::append_config_environment_variables; use crate::util::IS_A_TTY; use anyhow::Result; use std::path::PathBuf; @@ -17,20 +18,21 @@ pub fn prepare_build_command( image: &str, dockerfile: &Path, features: Option<&[String]>, + config_environment_variables: &Environment, ) -> Command { // Start with `docker build` - let mut cmd = docker_command(["build"]); + let mut command = docker_command(["build"]); // Ensure we run from the repo root (so `.` context is correct) - cmd.current_dir(app::path()); + command.current_dir(app::path()); // If we're attached to a TTY, show fancy progress if *IS_A_TTY { - cmd.args(["--progress", "tty"]); + command.args(["--progress", "tty"]); } // Add all of the flags in one go - cmd.args([ + command.args([ "--pull", "--tag", image, @@ -42,10 +44,12 @@ pub fn prepare_build_command( &format!("RUST_VERSION={}", RustToolchainConfig::rust_version()), "--build-arg", &format!("FEATURES={}", features.unwrap_or(&[]).join(",")), - ".", ]); - cmd + append_config_environment_variables(&mut command, "--build-arg", config_environment_variables); + + command.args(["."]); + command } #[allow(dead_code)] @@ -59,6 +63,7 @@ pub fn build_integration_image() -> Result<()> { &image, &dockerfile, Some(&[ALL_INTEGRATIONS_FEATURE_FLAG.to_string()]), + &Environment::default(), ); waiting!("Building {image}"); cmd.check_run() diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 030bebbe251f1..55df6901d3207 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,11 +1,11 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use tempfile::{Builder, NamedTempFile}; use super::config::{ - ComposeConfig, ComposeTestConfig, E2E_TESTS_DIR, Environment, INTEGRATION_TESTS_DIR, - RustToolchainConfig, + ComposeConfig, ComposeTestConfig, Environment, RustToolchainConfig, E2E_TESTS_DIR, + INTEGRATION_TESTS_DIR, }; use super::runner::{ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _}; use super::state::EnvsDir; @@ -121,7 +121,7 @@ impl ComposeTest { let mut env_vars = self.config.env.clone(); // Make sure the test runner has the same config environment vars as the services do. - for (key, value) in config_env(&self.env_config) { + for (key, value) in adapt_environment_variables(&self.env_config) { env_vars.insert(key, Some(value)); } @@ -181,8 +181,11 @@ impl ComposeTest { // image for the runner. So we must build that image before starting the // compose so that it is available. if self.local_config.kind == ComposeTestKind::E2E { - self.runner - .build(Some(&self.config.features), self.local_config.directory)?; + self.runner.build( + Some(&self.config.features), + self.local_config.directory, + &self.env_config, + )?; } self.config.check_required()?; @@ -321,7 +324,7 @@ impl Compose { } } if let Some(config) = config { - command.envs(config_env(config)); + command.envs(adapt_environment_variables(config)); } waiting!("{action} service environment"); @@ -335,7 +338,9 @@ impl Compose { } } -fn config_env(config: &Environment) -> impl Iterator + '_ { +pub(crate) fn adapt_environment_variables( + config: &Environment, +) -> impl Iterator + '_ { config.iter().filter_map(|(var, value)| { value.as_ref().map(|value| { ( @@ -346,6 +351,20 @@ fn config_env(config: &Environment) -> impl Iterator + }) } +pub(crate) fn append_config_environment_variables( + command: &mut Command, + arg_type: &str, + config_environment_variables: &Environment, +) { + for (key, value) in config_environment_variables { + command.arg(arg_type); + match value { + Some(value) => command.arg(format!("{key}={value}")), + None => command.arg(key), + }; + } +} + #[cfg(unix)] mod unix { use std::fs::{self, Metadata, Permissions}; diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index c1cead3a47bd1..f0cc0bd1aefb2 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -7,7 +7,8 @@ use anyhow::Result; use super::config::{Environment, IntegrationRunnerConfig, RustToolchainConfig}; use crate::app::{self, CommandExt as _}; use crate::testing::build::prepare_build_command; -use crate::testing::docker::{DOCKER_SOCKET, docker_command}; +use crate::testing::docker::{docker_command, DOCKER_SOCKET}; +use crate::testing::integration::append_config_environment_variables; use crate::util::{ChainArgs as _, IS_A_TTY}; const MOUNT_PATH: &str = "/home/vector"; @@ -97,7 +98,12 @@ pub trait ContainerTestRunner: TestRunner { Ok(RunnerState::Missing) } - fn ensure_running(&self, features: Option<&[String]>, directory: &str) -> Result<()> { + fn ensure_running( + &self, + features: Option<&[String]>, + directory: &str, + config_environment_variables: &Environment, + ) -> Result<()> { match self.state()? { RunnerState::Running | RunnerState::Restarting => (), RunnerState::Created | RunnerState::Exited => self.start()?, @@ -108,7 +114,7 @@ pub trait ContainerTestRunner: TestRunner { self.start()?; } RunnerState::Missing => { - self.build(features, directory)?; + self.build(features, directory, config_environment_variables)?; self.ensure_volumes()?; self.create()?; self.start()?; @@ -137,11 +143,18 @@ pub trait ContainerTestRunner: TestRunner { Ok(()) } - fn build(&self, features: Option<&[String]>, directory: &str) -> Result<()> { + fn build( + &self, + features: Option<&[String]>, + directory: &str, + config_env_vars: &Environment, + ) -> Result<()> { let dockerfile: PathBuf = [app::path(), "scripts", directory, "Dockerfile"] .iter() .collect(); - let mut command = prepare_build_command(&self.image_name(), &dockerfile, features); + + let mut command = + prepare_build_command(&self.image_name(), &dockerfile, features, config_env_vars); waiting!("Building image {}", self.image_name()); command.check_run() } @@ -216,12 +229,12 @@ where fn test( &self, outer_env: &Environment, - inner_env: &Environment, + config_environment_variables: &Environment, features: Option<&[String]>, args: &[String], directory: &str, ) -> Result<()> { - self.ensure_running(features, directory)?; + self.ensure_running(features, directory, config_environment_variables)?; let mut command = docker_command(["exec"]); if *IS_A_TTY { @@ -236,13 +249,7 @@ where } command.args(["--env", key]); } - for (key, value) in inner_env { - command.arg("--env"); - match value { - Some(value) => command.arg(format!("{key}={value}")), - None => command.arg(key), - }; - } + append_config_environment_variables(&mut command, "--env", config_environment_variables); command.arg(self.container_name()); command.args(TEST_COMMAND); From 5066d92a30b1c0f5a13a1fd906d2a4e3cb20d990 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 22 Aug 2025 10:40:36 -0400 Subject: [PATCH 26/49] vdev cruft --- vdev/src/commands/compose_tests/show.rs | 4 +- vdev/src/env_vars.rs | 64 +++++++++++++++++++++ vdev/src/main.rs | 1 + vdev/src/testing/build.rs | 7 ++- vdev/src/testing/config.rs | 5 +- vdev/src/testing/integration.rs | 75 +++++++++---------------- vdev/src/testing/runner.rs | 6 +- vdev/src/testing/state.rs | 4 +- 8 files changed, 105 insertions(+), 61 deletions(-) create mode 100644 vdev/src/env_vars.rs diff --git a/vdev/src/commands/compose_tests/show.rs b/vdev/src/commands/compose_tests/show.rs index dc9ef5012157a..b25f3472219e3 100644 --- a/vdev/src/commands/compose_tests/show.rs +++ b/vdev/src/commands/compose_tests/show.rs @@ -1,7 +1,7 @@ +use crate::env_vars::Environment; +use crate::testing::{config::ComposeTestConfig, state}; use anyhow::Result; -use crate::testing::{config::ComposeTestConfig, config::Environment, state}; - pub fn exec(integration: Option<&String>, path: &str) -> Result<()> { match integration { None => show_all(path), diff --git a/vdev/src/env_vars.rs b/vdev/src/env_vars.rs new file mode 100644 index 0000000000000..224168c4660e0 --- /dev/null +++ b/vdev/src/env_vars.rs @@ -0,0 +1,64 @@ +use regex::Regex; +use std::collections::BTreeMap; +use std::process::Command; +use std::sync::OnceLock; + +pub type Environment = BTreeMap>; + +pub(crate) fn rename_environment_keys(environment: &Environment) -> Environment { + environment + .iter() + .map(|(var, value)| { + ( + format!("CONFIG_{}", var.replace('-', "_").to_uppercase()), + value.clone(), + ) + }) + .collect() +} + +pub(crate) fn extract_present(environment: &Environment) -> BTreeMap { + environment + .iter() + .filter_map(|(k, v)| v.as_ref().map(|s| (k.clone(), s.clone()))) + .collect() +} + +pub(crate) fn append_environment_variables(command: &mut Command, environment: &Environment) { + for (key, value) in environment { + command.arg("--env"); + match value { + Some(value) => command.arg(format!("{key}={value}")), + None => command.arg(key), + }; + } +} + +/// If the variable is not found or is `None`, it is left unchanged. +pub fn resolve_placeholders(input: &str, environment: &Environment) -> String { + // static regexes + static BRACED: OnceLock = OnceLock::new(); + static BARE: OnceLock = OnceLock::new(); + + let braced = BRACED.get_or_init(|| Regex::new(r"\$\{([A-Za-z0-9_]+)\}").unwrap()); + let bare = BARE.get_or_init(|| Regex::new(r"\$([A-Za-z0-9_]+)").unwrap()); + + // First replace ${VAR} + let step1 = braced.replace_all(input, |caps: ®ex::Captures| { + let name = &caps[1]; + lookup(name, environment).unwrap_or_else(|| caps[0].to_string()) + }); + + // Then replace $VAR + bare.replace_all(&step1, |caps: ®ex::Captures| { + let name = &caps[1]; + lookup(name, environment).unwrap_or_else(|| caps[0].to_string()) + }) + .into_owned() +} + +fn lookup(name: &str, env: &Environment) -> Option { + env.get(name) + .and_then(|opt| opt.clone()) + .or_else(|| std::env::var(name).ok()) +} diff --git a/vdev/src/main.rs b/vdev/src/main.rs index fc87b158f5c30..7036139c59ef9 100644 --- a/vdev/src/main.rs +++ b/vdev/src/main.rs @@ -11,6 +11,7 @@ mod macros; mod app; mod commands; mod config; +mod env_vars; mod features; mod git; mod platform; diff --git a/vdev/src/testing/build.rs b/vdev/src/testing/build.rs index d750f35ad48a4..658d8ce771e2e 100644 --- a/vdev/src/testing/build.rs +++ b/vdev/src/testing/build.rs @@ -1,8 +1,8 @@ use crate::app; use crate::app::CommandExt; -use crate::testing::config::{Environment, RustToolchainConfig}; +use crate::env_vars::{extract_present, rename_environment_keys, Environment}; +use crate::testing::config::RustToolchainConfig; use crate::testing::docker::docker_command; -use crate::testing::integration::append_config_environment_variables; use crate::util::IS_A_TTY; use anyhow::Result; use std::path::PathBuf; @@ -46,7 +46,8 @@ pub fn prepare_build_command( &format!("FEATURES={}", features.unwrap_or(&[]).join(",")), ]); - append_config_environment_variables(&mut command, "--build-arg", config_environment_variables); + let env_vars = extract_present(&rename_environment_keys(config_environment_variables)); + command.envs(env_vars); command.args(["."]); command diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 53a971ffa7abe..5928f524e4807 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -2,12 +2,13 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::{env, fs}; -use anyhow::{Context, Result, bail}; +use anyhow::{bail, Context, Result}; use indexmap::IndexMap; use itertools::{self, Itertools}; use serde::{Deserialize, Serialize}; use serde_yaml::Value; +use crate::env_vars::Environment; use crate::{app, util}; const FILE_NAME: &str = "test.yaml"; @@ -15,8 +16,6 @@ const FILE_NAME: &str = "test.yaml"; pub const INTEGRATION_TESTS_DIR: &str = "integration"; pub const E2E_TESTS_DIR: &str = "e2e"; -pub type Environment = BTreeMap>; - #[derive(Deserialize, Debug)] pub struct RustToolchainRootConfig { pub toolchain: RustToolchainConfig, diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index 55df6901d3207..c1f4bf125306d 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -4,12 +4,12 @@ use anyhow::{bail, Context, Result}; use tempfile::{Builder, NamedTempFile}; use super::config::{ - ComposeConfig, ComposeTestConfig, Environment, RustToolchainConfig, E2E_TESTS_DIR, - INTEGRATION_TESTS_DIR, + ComposeConfig, ComposeTestConfig, RustToolchainConfig, E2E_TESTS_DIR, INTEGRATION_TESTS_DIR, }; use super::runner::{ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _}; use super::state::EnvsDir; use crate::app::CommandExt as _; +use crate::env_vars::{extract_present, rename_environment_keys, Environment}; use crate::testing::build::ALL_INTEGRATIONS_FEATURE_FLAG; use crate::testing::docker::{CONTAINER_TOOL, DOCKER_SOCKET}; @@ -121,8 +121,8 @@ impl ComposeTest { let mut env_vars = self.config.env.clone(); // Make sure the test runner has the same config environment vars as the services do. - for (key, value) in adapt_environment_variables(&self.env_config) { - env_vars.insert(key, Some(value)); + for (key, value) in rename_environment_keys(&self.env_config) { + env_vars.insert(key, value); } env_vars.insert("TEST_LOG".to_string(), Some("info".into())); @@ -196,7 +196,7 @@ impl ComposeTest { bail!("environment is already up"); } - compose.start(&self.env_config)?; + compose.start(&rename_environment_keys(&self.env_config))?; self.envs_dir.save(&self.environment, &self.env_config) } else { @@ -278,9 +278,11 @@ impl Compose { } } - fn start(&self, config: &Environment) -> Result<()> { - self.prepare()?; - self.run("Starting", &["up", "--detach"], Some(config)) + fn start(&self, environment: &Environment) -> Result<()> { + #[cfg(unix)] + unix::prepare_compose_volumes(&self.config, &self.test_dir, &environment)?; + + self.run("Starting", &["up", "--detach"], Some(environment)) } fn stop(&self) -> Result<()> { @@ -292,7 +294,12 @@ impl Compose { ) } - fn run(&self, action: &str, args: &[&'static str], config: Option<&Environment>) -> Result<()> { + fn run( + &self, + action: &str, + args: &[&'static str], + environment: Option<&Environment>, + ) -> Result<()> { let mut command = Command::new(CONTAINER_TOOL.clone()); command.arg("compose"); // When the integration test environment is already active, the tempfile path does not @@ -323,46 +330,13 @@ impl Compose { command.env(key, value); } } - if let Some(config) = config { - command.envs(adapt_environment_variables(config)); + if let Some(config) = environment { + command.envs(extract_present(&config)); } waiting!("{action} service environment"); command.check_run() } - - fn prepare(&self) -> Result<()> { - #[cfg(unix)] - unix::prepare_compose_volumes(&self.config, &self.test_dir)?; - Ok(()) - } -} - -pub(crate) fn adapt_environment_variables( - config: &Environment, -) -> impl Iterator + '_ { - config.iter().filter_map(|(var, value)| { - value.as_ref().map(|value| { - ( - format!("CONFIG_{}", var.replace('-', "_").to_uppercase()), - value.to_string(), - ) - }) - }) -} - -pub(crate) fn append_config_environment_variables( - command: &mut Command, - arg_type: &str, - config_environment_variables: &Environment, -) { - for (key, value) in config_environment_variables { - command.arg(arg_type); - match value { - Some(value) => command.arg(format!("{key}={value}")), - None => command.arg(key), - }; - } } #[cfg(unix)] @@ -372,6 +346,7 @@ mod unix { use std::path::{Path, PathBuf}; use super::super::config::ComposeConfig; + use crate::env_vars::{resolve_placeholders, Environment}; use crate::testing::config::VolumeMount; use anyhow::{Context, Result}; @@ -381,7 +356,11 @@ mod unix { const ALL_READ_DIR: u32 = 0o555; /// Fix up potential issues before starting a compose container - pub fn prepare_compose_volumes(config: &ComposeConfig, test_dir: &Path) -> Result<()> { + pub fn prepare_compose_volumes( + config: &ComposeConfig, + test_dir: &Path, + environment: &Environment, + ) -> Result<()> { for service in config.services.values() { if let Some(volumes) = &service.volumes { for volume in volumes { @@ -393,12 +372,12 @@ mod unix { } VolumeMount::Long { source, .. } => source, }; - - if !config.volumes.contains_key(source) + let source = resolve_placeholders(&source, environment); + if !config.volumes.contains_key(&source) && !source.starts_with('/') && !source.starts_with('$') { - let path: PathBuf = [test_dir, Path::new(source)].iter().collect(); + let path: PathBuf = [test_dir, Path::new(&source)].iter().collect(); add_read_permission(&path)?; } } diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index f0cc0bd1aefb2..e6d6cea845875 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -4,11 +4,11 @@ use std::{env, path::PathBuf}; use anyhow::Result; -use super::config::{Environment, IntegrationRunnerConfig, RustToolchainConfig}; +use super::config::{IntegrationRunnerConfig, RustToolchainConfig}; use crate::app::{self, CommandExt as _}; +use crate::env_vars::{append_environment_variables, Environment}; use crate::testing::build::prepare_build_command; use crate::testing::docker::{docker_command, DOCKER_SOCKET}; -use crate::testing::integration::append_config_environment_variables; use crate::util::{ChainArgs as _, IS_A_TTY}; const MOUNT_PATH: &str = "/home/vector"; @@ -249,7 +249,7 @@ where } command.args(["--env", key]); } - append_config_environment_variables(&mut command, "--env", config_environment_variables); + append_environment_variables(&mut command, config_environment_variables); command.arg(self.container_name()); command.args(TEST_COMMAND); diff --git a/vdev/src/testing/state.rs b/vdev/src/testing/state.rs index 2f9125a9b181a..69a3fc62ce165 100644 --- a/vdev/src/testing/state.rs +++ b/vdev/src/testing/state.rs @@ -1,10 +1,10 @@ use std::path::{Path, PathBuf}; use std::{fs, io::ErrorKind, sync::LazyLock}; -use anyhow::{Context, Result, anyhow}; +use anyhow::{anyhow, Context, Result}; use serde::{Deserialize, Serialize}; -use super::config::Environment; +use crate::env_vars::Environment; use crate::{platform, util}; static DATA_DIR: LazyLock = LazyLock::new(|| { From 6d4a14f687ada7f6d2dd9e656a4ea7033cd696c1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Fri, 22 Aug 2025 14:04:57 -0400 Subject: [PATCH 27/49] ran cargo fmt --- vdev/src/testing/build.rs | 2 +- vdev/src/testing/config.rs | 2 +- vdev/src/testing/integration.rs | 8 ++++---- vdev/src/testing/runner.rs | 4 ++-- vdev/src/testing/state.rs | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/vdev/src/testing/build.rs b/vdev/src/testing/build.rs index 658d8ce771e2e..5f96e85ed3a00 100644 --- a/vdev/src/testing/build.rs +++ b/vdev/src/testing/build.rs @@ -1,6 +1,6 @@ use crate::app; use crate::app::CommandExt; -use crate::env_vars::{extract_present, rename_environment_keys, Environment}; +use crate::env_vars::{Environment, extract_present, rename_environment_keys}; use crate::testing::config::RustToolchainConfig; use crate::testing::docker::docker_command; use crate::util::IS_A_TTY; diff --git a/vdev/src/testing/config.rs b/vdev/src/testing/config.rs index 5928f524e4807..08ce18f3779e1 100644 --- a/vdev/src/testing/config.rs +++ b/vdev/src/testing/config.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; use std::{env, fs}; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use indexmap::IndexMap; use itertools::{self, Itertools}; use serde::{Deserialize, Serialize}; diff --git a/vdev/src/testing/integration.rs b/vdev/src/testing/integration.rs index c1f4bf125306d..468e2a0278bf1 100644 --- a/vdev/src/testing/integration.rs +++ b/vdev/src/testing/integration.rs @@ -1,15 +1,15 @@ use std::{collections::BTreeMap, fs, path::Path, path::PathBuf, process::Command}; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result, bail}; use tempfile::{Builder, NamedTempFile}; use super::config::{ - ComposeConfig, ComposeTestConfig, RustToolchainConfig, E2E_TESTS_DIR, INTEGRATION_TESTS_DIR, + ComposeConfig, ComposeTestConfig, E2E_TESTS_DIR, INTEGRATION_TESTS_DIR, RustToolchainConfig, }; use super::runner::{ContainerTestRunner as _, IntegrationTestRunner, TestRunner as _}; use super::state::EnvsDir; use crate::app::CommandExt as _; -use crate::env_vars::{extract_present, rename_environment_keys, Environment}; +use crate::env_vars::{Environment, extract_present, rename_environment_keys}; use crate::testing::build::ALL_INTEGRATIONS_FEATURE_FLAG; use crate::testing::docker::{CONTAINER_TOOL, DOCKER_SOCKET}; @@ -346,7 +346,7 @@ mod unix { use std::path::{Path, PathBuf}; use super::super::config::ComposeConfig; - use crate::env_vars::{resolve_placeholders, Environment}; + use crate::env_vars::{Environment, resolve_placeholders}; use crate::testing::config::VolumeMount; use anyhow::{Context, Result}; diff --git a/vdev/src/testing/runner.rs b/vdev/src/testing/runner.rs index e6d6cea845875..0c5a4bb726079 100644 --- a/vdev/src/testing/runner.rs +++ b/vdev/src/testing/runner.rs @@ -6,9 +6,9 @@ use anyhow::Result; use super::config::{IntegrationRunnerConfig, RustToolchainConfig}; use crate::app::{self, CommandExt as _}; -use crate::env_vars::{append_environment_variables, Environment}; +use crate::env_vars::{Environment, append_environment_variables}; use crate::testing::build::prepare_build_command; -use crate::testing::docker::{docker_command, DOCKER_SOCKET}; +use crate::testing::docker::{DOCKER_SOCKET, docker_command}; use crate::util::{ChainArgs as _, IS_A_TTY}; const MOUNT_PATH: &str = "/home/vector"; diff --git a/vdev/src/testing/state.rs b/vdev/src/testing/state.rs index 69a3fc62ce165..e37912aadfa19 100644 --- a/vdev/src/testing/state.rs +++ b/vdev/src/testing/state.rs @@ -1,7 +1,7 @@ use std::path::{Path, PathBuf}; use std::{fs, io::ErrorKind, sync::LazyLock}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{Context, Result, anyhow}; use serde::{Deserialize, Serialize}; use crate::env_vars::Environment; From 43f3f9d3a85f07360646b668c5e3eb19aeea6313 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 27 Aug 2025 11:33:32 -0400 Subject: [PATCH 28/49] cleanup --- src/sinks/http/config.rs | 1 - vdev/src/env_vars.rs | 64 ---------------------------------------- 2 files changed, 65 deletions(-) delete mode 100644 vdev/src/env_vars.rs diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index 695751ea140ae..8927d1373cc37 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -241,7 +241,6 @@ impl SinkConfig for HttpSinkConfig { let batch_settings = self.batch.validate()?.into_batcher_settings()?; let encoder = self.build_encoder()?; - println!("🚧 encoder: {encoder:?}"); let transformer = self.encoding.transformer(); let mut request = self.request.clone(); diff --git a/vdev/src/env_vars.rs b/vdev/src/env_vars.rs deleted file mode 100644 index 224168c4660e0..0000000000000 --- a/vdev/src/env_vars.rs +++ /dev/null @@ -1,64 +0,0 @@ -use regex::Regex; -use std::collections::BTreeMap; -use std::process::Command; -use std::sync::OnceLock; - -pub type Environment = BTreeMap>; - -pub(crate) fn rename_environment_keys(environment: &Environment) -> Environment { - environment - .iter() - .map(|(var, value)| { - ( - format!("CONFIG_{}", var.replace('-', "_").to_uppercase()), - value.clone(), - ) - }) - .collect() -} - -pub(crate) fn extract_present(environment: &Environment) -> BTreeMap { - environment - .iter() - .filter_map(|(k, v)| v.as_ref().map(|s| (k.clone(), s.clone()))) - .collect() -} - -pub(crate) fn append_environment_variables(command: &mut Command, environment: &Environment) { - for (key, value) in environment { - command.arg("--env"); - match value { - Some(value) => command.arg(format!("{key}={value}")), - None => command.arg(key), - }; - } -} - -/// If the variable is not found or is `None`, it is left unchanged. -pub fn resolve_placeholders(input: &str, environment: &Environment) -> String { - // static regexes - static BRACED: OnceLock = OnceLock::new(); - static BARE: OnceLock = OnceLock::new(); - - let braced = BRACED.get_or_init(|| Regex::new(r"\$\{([A-Za-z0-9_]+)\}").unwrap()); - let bare = BARE.get_or_init(|| Regex::new(r"\$([A-Za-z0-9_]+)").unwrap()); - - // First replace ${VAR} - let step1 = braced.replace_all(input, |caps: ®ex::Captures| { - let name = &caps[1]; - lookup(name, environment).unwrap_or_else(|| caps[0].to_string()) - }); - - // Then replace $VAR - bare.replace_all(&step1, |caps: ®ex::Captures| { - let name = &caps[1]; - lookup(name, environment).unwrap_or_else(|| caps[0].to_string()) - }) - .into_owned() -} - -fn lookup(name: &str, env: &Environment) -> Option { - env.get(name) - .and_then(|opt| opt.clone()) - .or_else(|| std::env::var(name).ok()) -} From a619059f6f71b9a03700129417f4e50b1c4b2cac Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 27 Aug 2025 12:33:17 -0400 Subject: [PATCH 29/49] fix new config --- scripts/run-integration-test.sh | 24 +++++++++++-------- .../e2e/opentelemetry/logs/vector_oltp.yaml | 7 +++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/scripts/run-integration-test.sh b/scripts/run-integration-test.sh index cab8458adba33..36c254834ce93 100755 --- a/scripts/run-integration-test.sh +++ b/scripts/run-integration-test.sh @@ -32,7 +32,8 @@ Options: -h Show this help and exit -r Number of retries for the "test" phase (default: 2) -v Increase verbosity; repeat for more (e.g. -vv or -vvv) - -e TEST_ENVIRONMENT to export as TEST_ENVIRONMENT (default: not set) + -e One or more environments to run (repeatable or comma-separated). + If provided, these are used as TEST_ENVIRONMENTS instead of auto-discovery. Notes: - All existing two-argument invocations remain compatible: @@ -43,7 +44,8 @@ USAGE # Parse options # Note: options must come before positional args (standard getopts behavior) -while getopts ":hr:v:" opt; do +TEST_ENV="" +while getopts ":hr:v:e:" opt; do case "$opt" in h) usage @@ -59,6 +61,9 @@ while getopts ":hr:v:" opt; do v) VERBOSITY+="v" ;; + e) + TEST_ENV="$OPTARG" + ;; \?) echo "error: unknown option: -$OPTARG" >&2 usage @@ -95,14 +100,13 @@ case "$TEST_TYPE" in ;; esac -# Collect all available environments -mapfile -t TEST_ENVIRONMENTS < <(cargo vdev "${VERBOSITY}" "${TEST_TYPE}" show -e "${TEST_NAME}") - -if [[ "${ACTIONS_RUNNER_DEBUG:-}" == "true" ]]; then - echo "Environments found: ${#TEST_ENVIRONMENTS[@]}" - for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do - echo "${TEST_ENV}" - done +# Determine environments to run +if [[ ${#TEST_ENV} -gt 0 ]]; then + # Use the environments supplied via -e + TEST_ENVIRONMENTS="${TEST_ENV}" +else + # Collect all available environments via auto-discovery + mapfile -t TEST_ENVIRONMENTS < <(cargo vdev "${VERBOSITY}" "${TEST_TYPE}" show -e "${TEST_NAME}") fi for TEST_ENV in "${TEST_ENVIRONMENTS[@]}"; do diff --git a/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml b/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml index cedaa85626876..6a1bcfb596904 100644 --- a/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml +++ b/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml @@ -8,7 +8,7 @@ sources: keepalive: max_connection_age_jitter_factor: 0.1 max_connection_age_secs: 300 - use_otlp_decoding: true + use_oltp_decoding: true internal_metrics: type: internal_metrics @@ -16,7 +16,8 @@ sources: sinks: otel_sink: - inputs: [ "source0.logs" ] + inputs: + - source0.logs type: opentelemetry protocol: type: http @@ -36,7 +37,7 @@ sinks: type: file path: "/output/vector-file-sink.log" inputs: - - remap_otel + - source0.logs encoding: codec: json From 1c483612372a1fb9e779de22f31052caa283bdd6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Wed, 27 Aug 2025 17:29:15 -0400 Subject: [PATCH 30/49] use JSON names - wip --- Cargo.lock | 143 +++++++++++++++++---- Cargo.toml | 2 +- lib/codecs/src/decoding/format/protobuf.rs | 18 ++- lib/vector-core/src/tls/settings.rs | 8 +- src/sources/opentelemetry/config.rs | 11 +- 5 files changed, 141 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0f0977881a231..b297d5bfcb832 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2731,7 +2731,7 @@ dependencies = [ "bitflags 2.9.0", "crossterm_winapi", "mio", - "parking_lot", + "parking_lot 0.12.4", "rustix 0.38.40", "signal-hook", "signal-hook-mio", @@ -2749,7 +2749,7 @@ dependencies = [ "document-features", "futures-core", "mio", - "parking_lot", + "parking_lot 0.12.4", "rustix 1.0.1", "signal-hook", "signal-hook-mio", @@ -2968,7 +2968,7 @@ dependencies = [ "hashbrown 0.14.5", "lock_api", "once_cell", - "parking_lot_core", + "parking_lot_core 0.9.11", ] [[package]] @@ -2992,7 +2992,7 @@ dependencies = [ "cookie", "log", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "percent-encoding", "reqwest 0.12.9", "semver 1.0.26", @@ -4007,7 +4007,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" dependencies = [ "futures-core", "lock_api", - "parking_lot", + "parking_lot 0.12.4", ] [[package]] @@ -4224,7 +4224,7 @@ dependencies = [ "getrandom 0.3.1", "hashbrown 0.15.2", "nonzero_ext", - "parking_lot", + "parking_lot 0.12.4", "portable-atomic", "rand 0.9.2", "smallvec", @@ -4315,7 +4315,7 @@ dependencies = [ "futures 0.3.31", "futures-util", "greptime-proto", - "parking_lot", + "parking_lot 0.12.4", "prost 0.12.6", "rand 0.9.2", "snafu 0.8.6", @@ -4850,7 +4850,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", @@ -4905,7 +4905,7 @@ dependencies = [ "once_cell", "openssl", "openssl-sys", - "parking_lot", + "parking_lot 0.12.4", "tokio", "tokio-openssl", "tower-layer", @@ -4924,7 +4924,7 @@ dependencies = [ "once_cell", "openssl", "openssl-sys", - "parking_lot", + "parking_lot 0.12.4", "pin-project", "tower-layer", "tower-service", @@ -4974,6 +4974,7 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "rustls 0.23.23", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", "tokio-rustls 0.26.2", @@ -5356,6 +5357,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -5773,7 +5777,7 @@ dependencies = [ "jsonptr", "k8s-openapi 0.22.0", "kube-client", - "parking_lot", + "parking_lot 0.12.4", "pin-project", "serde", "serde_json", @@ -5828,7 +5832,7 @@ dependencies = [ "flume 0.11.0", "futures-core", "futures-io", - "parking_lot", + "parking_lot 0.12.4", "pinky-swear", "reactor-trait", "serde", @@ -6345,7 +6349,7 @@ dependencies = [ "mlua-sys", "mlua_derive", "num-traits", - "parking_lot", + "parking_lot 0.12.4", "rustc-hash", "rustversion", ] @@ -6398,7 +6402,7 @@ dependencies = [ "event-listener 5.3.1", "futures-util", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "quanta", "rustc_version 0.4.1", "smallvec", @@ -7316,6 +7320,17 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb813b8af86854136c6922af0598d719255ecb2179515e6e7730d468f05c9cae" +[[package]] +name = "parking_lot" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" +dependencies = [ + "instant", + "lock_api", + "parking_lot_core 0.8.6", +] + [[package]] name = "parking_lot" version = "0.12.4" @@ -7323,7 +7338,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" dependencies = [ "lock_api", - "parking_lot_core", + "parking_lot_core 0.9.11", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc" +dependencies = [ + "cfg-if", + "instant", + "libc", + "redox_syscall 0.2.16", + "smallvec", + "winapi", ] [[package]] @@ -7537,7 +7566,7 @@ checksum = "d894b67aa7a4bf295db5e85349078c604edaa6fa5c8721e8eca3c7729a27f2ac" dependencies = [ "doc-comment", "flume 0.10.14", - "parking_lot", + "parking_lot 0.12.4", "tracing 0.1.41", ] @@ -8012,7 +8041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8058,7 +8087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2 1.0.95", "quote 1.0.40", "syn 2.0.104", @@ -8696,7 +8725,7 @@ dependencies = [ "ahash 0.8.11", "fluent-uri 0.3.2", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "percent-encoding", "serde_json", ] @@ -8839,6 +8868,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.4.11", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -8857,6 +8887,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.23", + "rustls-native-certs 0.8.1", "rustls-pemfile 2.1.0", "rustls-pki-types", "serde", @@ -8877,6 +8908,42 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "reqwest-middleware" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" +dependencies = [ + "anyhow", + "async-trait", + "http 1.1.0", + "reqwest 0.12.9", + "serde", + "thiserror 1.0.68", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178" +dependencies = [ + "anyhow", + "async-trait", + "futures 0.3.31", + "getrandom 0.2.15", + "http 1.1.0", + "hyper 1.4.1", + "parking_lot 0.11.2", + "reqwest 0.12.9", + "reqwest-middleware", + "retry-policies", + "thiserror 1.0.68", + "tokio", + "wasm-timer", +] + [[package]] name = "resolv-conf" version = "0.7.0" @@ -8887,6 +8954,15 @@ dependencies = [ "quick-error", ] +[[package]] +name = "retry-policies" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c" +dependencies = [ + "rand 0.8.5", +] + [[package]] name = "rfc6979" version = "0.4.0" @@ -10291,7 +10367,7 @@ checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" dependencies = [ "new_debug_unreachable", "once_cell", - "parking_lot", + "parking_lot 0.12.4", "phf_shared 0.10.0", "precomputed-hash", ] @@ -10766,7 +10842,7 @@ dependencies = [ "io-uring", "libc", "mio", - "parking_lot", + "parking_lot 0.12.4", "pin-project-lite", "signal-hook-registry", "slab", @@ -10842,7 +10918,7 @@ dependencies = [ "futures-channel", "futures-util", "log", - "parking_lot", + "parking_lot 0.12.4", "percent-encoding", "phf 0.11.2", "pin-project-lite", @@ -11480,7 +11556,7 @@ dependencies = [ "lazy_static", "log", "lru-cache", - "parking_lot", + "parking_lot 0.12.4", "resolv-conf", "smallvec", "thiserror 1.0.68", @@ -12262,7 +12338,7 @@ dependencies = [ "noisy_float", "openssl", "ordered-float 4.6.0", - "parking_lot", + "parking_lot 0.12.4", "pin-project", "proptest", "prost 0.12.6", @@ -12442,7 +12518,6 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.26.0" -source = "git+https://github.com/vectordotdev/vrl.git?branch=main#b0a4de2555453f794d9f841a7700bb4bed081fb3" dependencies = [ "aes", "aes-siv", @@ -12516,6 +12591,9 @@ dependencies = [ "quoted_printable", "rand 0.8.5", "regex", + "reqwest 0.12.9", + "reqwest-middleware", + "reqwest-retry", "roxmltree", "rust_decimal", "rustyline", @@ -12733,6 +12811,21 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm-timer" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f" +dependencies = [ + "futures 0.3.31", + "js-sys", + "parking_lot 0.11.2", + "pin-utils", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.77" diff --git a/Cargo.toml b/Cargo.toml index ebbe292270a8b..a0c34c0f68a3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -193,7 +193,7 @@ vector-lib = { path = "lib/vector-lib", default-features = false, features = ["v vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { path = "../vrl", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] cfg-if.workspace = true diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 353ab340a023e..3e9b8674a3d98 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,14 +4,15 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{SmallVec, smallvec}; +use smallvec::{smallvec, SmallVec}; use vector_config::configurable_component; use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{DataType, LogNamespace, log_schema}, + config::{log_schema, DataType, LogNamespace}, event::Event, schema, }; +use vrl::protobuf::Options; use vrl::value::{Kind, Value}; use super::Deserializer; @@ -82,25 +83,27 @@ pub struct ProtobufDeserializerOptions { #[derive(Debug, Clone)] pub struct ProtobufDeserializer { message_descriptor: MessageDescriptor, + options: Options, } impl ProtobufDeserializer { /// Creates a new `ProtobufDeserializer`. pub fn new(message_descriptor: MessageDescriptor) -> Self { - Self { message_descriptor } + Self { message_descriptor, options: Default::default() } } /// Creates a new deserializer instance using the descriptor bytes directly. - pub fn new_from_bytes(desc_bytes: &[u8], message_type: &str) -> vector_common::Result { + pub fn new_from_bytes(desc_bytes: &[u8], message_type: &str, options: Options) -> vector_common::Result { let message_descriptor = vrl::protobuf::get_message_descriptor_from_bytes(desc_bytes, message_type)?; - Ok(Self { message_descriptor }) + Ok(Self { message_descriptor, options }) } } fn extract_vrl_value( bytes: Bytes, message_descriptor: &MessageDescriptor, + options: &Options ) -> vector_common::Result { let dynamic_message = DynamicMessage::decode(message_descriptor.clone(), bytes) .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; @@ -108,6 +111,7 @@ fn extract_vrl_value( Ok(vrl::protobuf::proto_to_value( &prost_reflect::Value::Message(dynamic_message), None, + options, )?) } @@ -117,7 +121,7 @@ impl Deserializer for ProtobufDeserializer { bytes: Bytes, log_namespace: LogNamespace, ) -> vector_common::Result> { - let vrl_value = extract_vrl_value(bytes, &self.message_descriptor)?; + let vrl_value = extract_vrl_value(bytes, &self.message_descriptor, &self.options)?; let mut event = Event::Log(LogEvent::from(vrl_value)); let event = match log_namespace { @@ -138,7 +142,7 @@ impl Deserializer for ProtobufDeserializer { } fn parse_traces(&self, bytes: Bytes) -> vector_common::Result> { - let vrl_value = extract_vrl_value(bytes, &self.message_descriptor)?; + let vrl_value = extract_vrl_value(bytes, &self.message_descriptor, &self.options)?; let trace_event = Event::Trace(TraceEvent::from(vrl_value)); Ok(smallvec![trace_event]) } diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index 8db20cea3c1d9..0ad17d3248bc9 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -9,9 +9,9 @@ use lookup::lookup_v2::OptionalValuePath; use openssl::{ pkcs12::{ParsedPkcs12_2, Pkcs12}, pkey::{PKey, Private}, - ssl::{AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode, select_next_proto}, + ssl::{select_next_proto, AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode}, stack::Stack, - x509::{X509, store::X509StoreBuilder}, + x509::{store::X509StoreBuilder, X509}, }; use snafu::ResultExt; use vector_config::configurable_component; @@ -317,7 +317,9 @@ impl TlsSettings { load_windows_certs(context).unwrap(); #[cfg(target_os = "macos")] - load_mac_certs(context).unwrap(); + if let Err(e) = load_mac_certs(context) { + warn!("{e:?}"); // TODO remove me before merging + } } else { let mut store = X509StoreBuilder::new().context(NewStoreBuilderSnafu)?; for authority in &self.authorities { diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 13f296a5403a7..4bf94f4bdd7b5 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,16 +1,16 @@ use std::net::SocketAddr; use futures::FutureExt; -use futures_util::{TryFutureExt, future::join}; +use futures_util::{future::join, TryFutureExt}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{LegacyKey, LogNamespace, log_schema}, + config::{log_schema, LegacyKey, LogNamespace}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{OwnedTargetPath, owned_value_path}, + lookup::{owned_value_path, OwnedTargetPath}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -34,17 +34,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ - Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, + Source, }, }; -use vrl::value::{Kind, kind::Collection}; +use vrl::value::{kind::Collection, Kind}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -173,6 +173,7 @@ impl OpentelemetryConfig { let deserializer = ProtobufDeserializer::new_from_bytes( vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, message_type, + vrl::protobuf::Options { use_json_names: true } )?; Ok(Some(deserializer)) } else { From 425186f42cade012a01a47b18f11b1eb25e1e1c5 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 10:32:04 -0400 Subject: [PATCH 31/49] ran cargo fmt --- lib/codecs/src/decoding/format/protobuf.rs | 22 ++++++++++++++++------ lib/vector-core/src/tls/settings.rs | 4 ++-- src/sources/opentelemetry/config.rs | 14 ++++++++------ 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 3e9b8674a3d98..b4b6ea517b8ff 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,11 +4,11 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{log_schema, DataType, LogNamespace}, + config::{DataType, LogNamespace, log_schema}, event::Event, schema, }; @@ -89,21 +89,31 @@ pub struct ProtobufDeserializer { impl ProtobufDeserializer { /// Creates a new `ProtobufDeserializer`. pub fn new(message_descriptor: MessageDescriptor) -> Self { - Self { message_descriptor, options: Default::default() } + Self { + message_descriptor, + options: Default::default(), + } } /// Creates a new deserializer instance using the descriptor bytes directly. - pub fn new_from_bytes(desc_bytes: &[u8], message_type: &str, options: Options) -> vector_common::Result { + pub fn new_from_bytes( + desc_bytes: &[u8], + message_type: &str, + options: Options, + ) -> vector_common::Result { let message_descriptor = vrl::protobuf::get_message_descriptor_from_bytes(desc_bytes, message_type)?; - Ok(Self { message_descriptor, options }) + Ok(Self { + message_descriptor, + options, + }) } } fn extract_vrl_value( bytes: Bytes, message_descriptor: &MessageDescriptor, - options: &Options + options: &Options, ) -> vector_common::Result { let dynamic_message = DynamicMessage::decode(message_descriptor.clone(), bytes) .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; diff --git a/lib/vector-core/src/tls/settings.rs b/lib/vector-core/src/tls/settings.rs index 538dca98f931f..2ed50a90257cd 100644 --- a/lib/vector-core/src/tls/settings.rs +++ b/lib/vector-core/src/tls/settings.rs @@ -9,9 +9,9 @@ use lookup::lookup_v2::OptionalValuePath; use openssl::{ pkcs12::{ParsedPkcs12_2, Pkcs12}, pkey::{PKey, Private}, - ssl::{select_next_proto, AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode}, + ssl::{AlpnError, ConnectConfiguration, SslContextBuilder, SslVerifyMode, select_next_proto}, stack::Stack, - x509::{store::X509StoreBuilder, X509}, + x509::{X509, store::X509StoreBuilder}, }; use snafu::ResultExt; use vector_config::configurable_component; diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 4bf94f4bdd7b5..41b8ee16ad3e5 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,16 +1,16 @@ use std::net::SocketAddr; use futures::FutureExt; -use futures_util::{future::join, TryFutureExt}; +use futures_util::{TryFutureExt, future::join}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{log_schema, LegacyKey, LogNamespace}, + config::{LegacyKey, LogNamespace, log_schema}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{owned_value_path, OwnedTargetPath}, + lookup::{OwnedTargetPath, owned_value_path}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -34,17 +34,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ + Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, - Source, }, }; -use vrl::value::{kind::Collection, Kind}; +use vrl::value::{Kind, kind::Collection}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -173,7 +173,9 @@ impl OpentelemetryConfig { let deserializer = ProtobufDeserializer::new_from_bytes( vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, message_type, - vrl::protobuf::Options { use_json_names: true } + vrl::protobuf::Options { + use_json_names: true, + }, )?; Ok(Some(deserializer)) } else { From 6d626583226c757bc5c786a3e0cd271adcbbf416 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 10:32:14 -0400 Subject: [PATCH 32/49] chore(dev): cargo vdev build licenses --- LICENSE-3rdparty.csv | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 1ecc775681200..41f5db1425a6c 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -464,6 +464,7 @@ p256,https://github.com/RustCrypto/elliptic-curves/tree/master/p256,Apache-2.0 O p384,https://github.com/RustCrypto/elliptic-curves/tree/master/p384,Apache-2.0 OR MIT,"RustCrypto Developers, Frank Denis " pad,https://github.com/ogham/rust-pad,MIT,Ben S parking,https://github.com/smol-rs/parking,Apache-2.0 OR MIT,"Stjepan Glavina , The Rust Project Developers" +parking_lot,https://github.com/Amanieu/parking_lot,Apache-2.0 OR MIT,Amanieu d'Antras parking_lot,https://github.com/Amanieu/parking_lot,MIT OR Apache-2.0,Amanieu d'Antras parse-size,https://github.com/kennytm/parse-size,MIT,kennytm passt,https://github.com/kevingimbel/passt,MIT OR Apache-2.0,Kevin Gimbel @@ -543,7 +544,9 @@ regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Proje regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur +reqwest-middleware,https://github.com/TrueLayer/reqwest-middleware,MIT OR Apache-2.0,Rodrigo Gryzinski resolv-conf,http://github.com/tailhook/resolv-conf,MIT OR Apache-2.0,paul@colomiets.name +retry-policies,https://github.com/TrueLayer/retry-policies,MIT OR Apache-2.0,Luca Palmieri rfc6979,https://github.com/RustCrypto/signatures/tree/master/rfc6979,Apache-2.0 OR MIT,RustCrypto Developers ring,https://github.com/briansmith/ring,Apache-2.0 AND ISC,The ring Authors rkyv,https://github.com/rkyv/rkyv,MIT,David Koloski @@ -724,7 +727,6 @@ uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix -vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton @@ -742,6 +744,7 @@ wasm-bindgen-macro,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/m wasm-bindgen-macro-support,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/macro-support,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-bindgen-shared,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/shared,MIT OR Apache-2.0,The wasm-bindgen Developers wasm-streams,https://github.com/MattiasBuelens/wasm-streams,MIT OR Apache-2.0,Mattias Buelens +wasm-timer,https://github.com/tomaka/wasm-timer,MIT,Pierre Krieger web-sys,https://github.com/rustwasm/wasm-bindgen/tree/master/crates/web-sys,MIT OR Apache-2.0,The wasm-bindgen Developers web-time,https://github.com/daxpedda/web-time,MIT OR Apache-2.0,The web-time Authors webbrowser,https://github.com/amodm/webbrowser-rs,MIT OR Apache-2.0,Amod Malviya @amodm From 2b6f1c8e771b80eadaa5710989064c8cdcec5ed6 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 13:01:56 -0400 Subject: [PATCH 33/49] update VRL --- Cargo.lock | 7 ++++--- Cargo.toml | 2 +- lib/codecs/src/decoding/format/protobuf.rs | 13 +++++++------ lib/codecs/src/encoding/format/protobuf.rs | 9 +++++---- src/sources/opentelemetry/config.rs | 14 +++++++------- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b297d5bfcb832..e7820e6b3d842 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4850,7 +4850,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.4.10", "tokio", "tower-service", "tracing 0.1.41", @@ -8041,7 +8041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.14.0", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -8087,7 +8087,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.11.0", "proc-macro2 1.0.95", "quote 1.0.40", "syn 2.0.104", @@ -12518,6 +12518,7 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "vrl" version = "0.26.0" +source = "git+https://github.com/vectordotdev/vrl.git?branch=main#2703fad09e3f88e50d086c2571100b267f27539b" dependencies = [ "aes", "aes-siv", diff --git a/Cargo.toml b/Cargo.toml index 46e391c0a7452..e8fd1983e747d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -193,7 +193,7 @@ vector-lib = { path = "lib/vector-lib", default-features = false, features = ["v vector-config = { path = "lib/vector-config" } vector-config-common = { path = "lib/vector-config-common" } vector-config-macros = { path = "lib/vector-config-macros" } -vrl = { path = "../vrl", features = ["arbitrary", "cli", "test", "test_framework"] } +vrl = { git = "https://github.com/vectordotdev/vrl.git", branch = "main", features = ["arbitrary", "cli", "test", "test_framework"] } [dependencies] cfg-if.workspace = true diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index b4b6ea517b8ff..9f8ecc6e07459 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,15 +4,16 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{SmallVec, smallvec}; +use smallvec::{smallvec, SmallVec}; use vector_config::configurable_component; use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{DataType, LogNamespace, log_schema}, + config::{log_schema, DataType, LogNamespace}, event::Event, schema, }; -use vrl::protobuf::Options; +use vrl::protobuf::descriptor::{get_message_descriptor, get_message_descriptor_from_bytes}; +use vrl::protobuf::parse::{proto_to_value, Options}; use vrl::value::{Kind, Value}; use super::Deserializer; @@ -102,7 +103,7 @@ impl ProtobufDeserializer { options: Options, ) -> vector_common::Result { let message_descriptor = - vrl::protobuf::get_message_descriptor_from_bytes(desc_bytes, message_type)?; + get_message_descriptor_from_bytes(desc_bytes, message_type)?; Ok(Self { message_descriptor, options, @@ -118,7 +119,7 @@ fn extract_vrl_value( let dynamic_message = DynamicMessage::decode(message_descriptor.clone(), bytes) .map_err(|error| format!("Error parsing protobuf: {error:?}"))?; - Ok(vrl::protobuf::proto_to_value( + Ok(proto_to_value( &prost_reflect::Value::Message(dynamic_message), None, options, @@ -161,7 +162,7 @@ impl Deserializer for ProtobufDeserializer { impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer { type Error = vector_common::Error; fn try_from(config: &ProtobufDeserializerConfig) -> vector_common::Result { - let message_descriptor = vrl::protobuf::get_message_descriptor( + let message_descriptor = get_message_descriptor( &config.protobuf.desc_file, &config.protobuf.message_type, )?; diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index 523c63e38b548..6db3c7b1df7af 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -1,6 +1,6 @@ use crate::encoding::BuildError; use bytes::BytesMut; -use prost_reflect::{MessageDescriptor, prost::Message as _}; +use prost_reflect::{prost::Message as _, MessageDescriptor}; use std::path::PathBuf; use tokio_util::codec::Encoder; use vector_config_macros::configurable_component; @@ -9,6 +9,7 @@ use vector_core::{ event::{Event, Value}, schema, }; +use vrl::protobuf::{descriptor::get_message_descriptor, encode::encode_message}; /// Config used to build a `ProtobufSerializer`. #[configurable_component] @@ -21,7 +22,7 @@ pub struct ProtobufSerializerConfig { impl ProtobufSerializerConfig { /// Build the `ProtobufSerializer` from this configuration. pub fn build(&self) -> Result { - let message_descriptor = vrl::protobuf::get_message_descriptor( + let message_descriptor = get_message_descriptor( &self.protobuf.desc_file, &self.protobuf.message_type, )?; @@ -83,10 +84,10 @@ impl Encoder for ProtobufSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let message = match event { Event::Log(log) => { - vrl::protobuf::encode_message(&self.message_descriptor, log.into_parts().0) + encode_message(&self.message_descriptor, log.into_parts().0) } Event::Metric(_) => unimplemented!(), - Event::Trace(trace) => vrl::protobuf::encode_message( + Event::Trace(trace) => encode_message( &self.message_descriptor, Value::Object(trace.into_parts().0), ), diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 41b8ee16ad3e5..983f043f18768 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,16 +1,15 @@ use std::net::SocketAddr; use futures::FutureExt; -use futures_util::{TryFutureExt, future::join}; +use futures_util::{future::join, TryFutureExt}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; - use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{LegacyKey, LogNamespace, log_schema}, + config::{log_schema, LegacyKey, LogNamespace}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{OwnedTargetPath, owned_value_path}, + lookup::{owned_value_path, OwnedTargetPath}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -25,6 +24,7 @@ use vector_lib::{ schema::Definition, tls::{MaybeTlsSettings, TlsEnableableConfig}, }; +use vrl::protobuf::parse::Options; use crate::{ config::{ @@ -34,17 +34,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ - Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, + Source, }, }; -use vrl::value::{Kind, kind::Collection}; +use vrl::value::{kind::Collection, Kind}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -173,7 +173,7 @@ impl OpentelemetryConfig { let deserializer = ProtobufDeserializer::new_from_bytes( vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, message_type, - vrl::protobuf::Options { + Options { use_json_names: true, }, )?; From a830fdf9db6d301e685ee3d79504a7c76e138060 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 13:23:13 -0400 Subject: [PATCH 34/49] ran cargo fmt --- lib/codecs/src/decoding/format/protobuf.rs | 15 ++++++--------- lib/codecs/src/encoding/format/protobuf.rs | 12 ++++-------- src/sources/opentelemetry/config.rs | 10 +++++----- 3 files changed, 15 insertions(+), 22 deletions(-) diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 9f8ecc6e07459..4c7643c78b5e4 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,16 +4,16 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{log_schema, DataType, LogNamespace}, + config::{DataType, LogNamespace, log_schema}, event::Event, schema, }; use vrl::protobuf::descriptor::{get_message_descriptor, get_message_descriptor_from_bytes}; -use vrl::protobuf::parse::{proto_to_value, Options}; +use vrl::protobuf::parse::{Options, proto_to_value}; use vrl::value::{Kind, Value}; use super::Deserializer; @@ -102,8 +102,7 @@ impl ProtobufDeserializer { message_type: &str, options: Options, ) -> vector_common::Result { - let message_descriptor = - get_message_descriptor_from_bytes(desc_bytes, message_type)?; + let message_descriptor = get_message_descriptor_from_bytes(desc_bytes, message_type)?; Ok(Self { message_descriptor, options, @@ -162,10 +161,8 @@ impl Deserializer for ProtobufDeserializer { impl TryFrom<&ProtobufDeserializerConfig> for ProtobufDeserializer { type Error = vector_common::Error; fn try_from(config: &ProtobufDeserializerConfig) -> vector_common::Result { - let message_descriptor = get_message_descriptor( - &config.protobuf.desc_file, - &config.protobuf.message_type, - )?; + let message_descriptor = + get_message_descriptor(&config.protobuf.desc_file, &config.protobuf.message_type)?; Ok(Self::new(message_descriptor)) } } diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index 6db3c7b1df7af..613c46acd6569 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -1,6 +1,6 @@ use crate::encoding::BuildError; use bytes::BytesMut; -use prost_reflect::{prost::Message as _, MessageDescriptor}; +use prost_reflect::{MessageDescriptor, prost::Message as _}; use std::path::PathBuf; use tokio_util::codec::Encoder; use vector_config_macros::configurable_component; @@ -22,10 +22,8 @@ pub struct ProtobufSerializerConfig { impl ProtobufSerializerConfig { /// Build the `ProtobufSerializer` from this configuration. pub fn build(&self) -> Result { - let message_descriptor = get_message_descriptor( - &self.protobuf.desc_file, - &self.protobuf.message_type, - )?; + let message_descriptor = + get_message_descriptor(&self.protobuf.desc_file, &self.protobuf.message_type)?; Ok(ProtobufSerializer { message_descriptor }) } @@ -83,9 +81,7 @@ impl Encoder for ProtobufSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let message = match event { - Event::Log(log) => { - encode_message(&self.message_descriptor, log.into_parts().0) - } + Event::Log(log) => encode_message(&self.message_descriptor, log.into_parts().0), Event::Metric(_) => unimplemented!(), Event::Trace(trace) => encode_message( &self.message_descriptor, diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 983f043f18768..b5eb9757352c8 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,15 +1,15 @@ use std::net::SocketAddr; use futures::FutureExt; -use futures_util::{future::join, TryFutureExt}; +use futures_util::{TryFutureExt, future::join}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{log_schema, LegacyKey, LogNamespace}, + config::{LegacyKey, LogNamespace, log_schema}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{owned_value_path, OwnedTargetPath}, + lookup::{OwnedTargetPath, owned_value_path}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -34,17 +34,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ + Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, - Source, }, }; -use vrl::value::{kind::Collection, Kind}; +use vrl::value::{Kind, kind::Collection}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; From 4ec6f8a867bc5b055bd80c704bab723d4a70f77f Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 13:23:20 -0400 Subject: [PATCH 35/49] chore(dev): cargo vdev build licenses --- LICENSE-3rdparty.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index 41f5db1425a6c..efbb8d68d4b70 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -727,6 +727,7 @@ uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix +vrl,https://github.com/vectordotdev/vrl,MPL-2.0,Vector Contributors vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm , Christian Duerr " wait-timeout,https://github.com/alexcrichton/wait-timeout,MIT OR Apache-2.0,Alex Crichton From de3ac9a675dd6eb124d40612c305d43b9c860bf8 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 13:25:56 -0400 Subject: [PATCH 36/49] handle metrics too --- src/sources/opentelemetry/config.rs | 14 +++-- src/sources/opentelemetry/grpc.rs | 30 +++++++---- src/sources/opentelemetry/http.rs | 52 +++++++++---------- .../sources/generated/opentelemetry.cue | 11 ++++ 4 files changed, 65 insertions(+), 42 deletions(-) diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index b5eb9757352c8..4b2d0bfff7f18 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,15 +1,15 @@ use std::net::SocketAddr; use futures::FutureExt; -use futures_util::{TryFutureExt, future::join}; +use futures_util::{future::join, TryFutureExt}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{LegacyKey, LogNamespace, log_schema}, + config::{log_schema, LegacyKey, LogNamespace}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{OwnedTargetPath, owned_value_path}, + lookup::{owned_value_path, OwnedTargetPath}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -34,17 +34,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ - Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, + Source, }, }; -use vrl::value::{Kind, kind::Collection}; +use vrl::value::{kind::Collection, Kind}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -78,6 +78,10 @@ pub struct OpentelemetryConfig { pub log_namespace: Option, /// Setting this field will override the legacy mapping of OTEL protos to Vector events and use the proto directly. + /// + /// One major caveat here is that the incoming metrics will be parsed as logs but they will preserve the OLTP format. + /// This means that components that work on metrics, will not be compatible with this output. + /// However, these events can be forwarded directly to a downstream OTEL collector. #[configurable(derived)] #[serde(default)] pub use_oltp_decoding: bool, diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 7ca1bd5478d64..3cd2c348b8b2b 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,31 +1,31 @@ use crate::sources::opentelemetry::config::METRICS; use crate::{ - SourceSender, internal_events::{EventsReceived, StreamClosedError}, sources::opentelemetry::config::{LOGS, TRACES}, + SourceSender, }; use futures::TryFutureExt; use prost::Message; use tonic::{Request, Response, Status}; -use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::codecs::decoding::format::Deserializer; +use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ - ExportLogsServiceRequest, ExportLogsServiceResponse, logs_service_server::LogsService, + logs_service_server::LogsService, ExportLogsServiceRequest, ExportLogsServiceResponse, }, metrics::v1::{ - ExportMetricsServiceRequest, ExportMetricsServiceResponse, - metrics_service_server::MetricsService, + metrics_service_server::MetricsService, ExportMetricsServiceRequest, + ExportMetricsServiceResponse, }, trace::v1::{ - ExportTraceServiceRequest, ExportTraceServiceResponse, trace_service_server::TraceService, + trace_service_server::TraceService, ExportTraceServiceRequest, ExportTraceServiceResponse, }, }; use vector_lib::{ - EstimatedJsonEncodedSizeOf, config::LogNamespace, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event}, + EstimatedJsonEncodedSizeOf, }; #[derive(Clone)] @@ -101,13 +101,23 @@ impl MetricsService for Service { &self, request: Request, ) -> Result, Status> { - // Protobuf deserializer doesn't support metrics. - let events: Vec = request + let events = if let Some(deserializer) = self.deserializer.as_ref() { + let raw_bytes = request.get_ref().encode_to_vec(); + // Major caveat here, the output event will be logs. + let bytes = bytes::Bytes::from(raw_bytes); + deserializer + .parse(bytes, self.log_namespace) + .map_err(|e| Status::invalid_argument(e.to_string())) + .map(|buf| buf.into_vec())? + } else { + request .into_inner() .resource_metrics .into_iter() .flat_map(|v| v.into_event_iter()) - .collect(); + .collect() + }; + self.handle_events(events, METRICS).await?; Ok(Response::new(ExportMetricsServiceResponse { diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 9e11826a018bd..4c4f56e964bec 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -5,28 +5,28 @@ use super::{reply::protobuf, status::Status}; use crate::common::http::ErrorMessage; use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; use crate::sources::http_server::HttpConfigParamKind; -use crate::sources::opentelemetry::config::{LOGS, METRICS, OpentelemetryConfig, TRACES}; +use crate::sources::opentelemetry::config::{OpentelemetryConfig, LOGS, METRICS, TRACES}; use crate::sources::util::add_headers; use crate::{ - SourceSender, event::Event, http::build_http_trace_layer, internal_events::{EventsReceived, StreamClosedError}, shutdown::ShutdownSignal, sources::util::decode, tls::MaybeTlsSettings, + SourceSender, }; use bytes::Bytes; use futures_util::FutureExt; use http::StatusCode; -use hyper::{Server, service::make_service_fn}; +use hyper::{service::make_service_fn, Server}; use prost::Message; use snafu::Snafu; use tokio::net::TcpStream; use tower::ServiceBuilder; use tracing::Span; -use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::codecs::decoding::format::Deserializer; +use vector_lib::codecs::decoding::ProtobufDeserializer; use vector_lib::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Registered, }; @@ -38,12 +38,12 @@ use vector_lib::opentelemetry::proto::collector::{ }; use vector_lib::tls::MaybeTlsIncomingStream; use vector_lib::{ - EstimatedJsonEncodedSizeOf, config::LogNamespace, event::{BatchNotifier, BatchStatus}, + EstimatedJsonEncodedSizeOf, }; use warp::{ - Filter, Reply, filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, + filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, Filter, Reply, }; #[derive(Clone, Copy, Debug, Snafu)] @@ -111,6 +111,7 @@ pub(crate) fn build_warp_filter( out.clone(), bytes_received.clone(), events_received.clone(), + deserializer.clone(), ); let trace_filters = build_warp_trace_filter( acknowledgements, @@ -221,30 +222,27 @@ fn build_warp_metrics_filter( source_sender: SourceSender, bytes_received: Registered, events_received: Registered, + deserializer: Option, ) -> BoxedFilter<(Response,)> { - warp::post() - .and(warp::path!("v1" / "metrics")) - .and(warp::header::exact_ignore_case( - "content-type", - "application/x-protobuf", - )) - .and(warp::header::optional::("content-encoding")) - .and(warp::body::bytes()) - .and_then(move |encoding_header: Option, body: Bytes| { - let events = decode(encoding_header.as_deref(), body).and_then(|body| { + let make_events = move |encoding_header: Option, _headers: HeaderMap, body: Bytes| { + if let Some(d) = deserializer.as_ref() { + d.parse(body, LogNamespace::default()) + .map(|r| r.into_vec()) + .map_err(|e| ErrorMessage::new(StatusCode::BAD_REQUEST, e.to_string())) + } else { + decode(encoding_header.as_deref(), body).and_then(|body| { bytes_received.emit(ByteSize(body.len())); decode_metrics_body(body, &events_received) - }); - - handle_request( - events, - acknowledgements, - source_sender.clone(), - METRICS, - ExportMetricsServiceResponse::default(), - ) - }) - .boxed() + }) + } + }; + + build_ingest_filter::( + METRICS, + acknowledgements, + source_sender, + make_events, + ) } fn build_warp_trace_filter( diff --git a/website/cue/reference/components/sources/generated/opentelemetry.cue b/website/cue/reference/components/sources/generated/opentelemetry.cue index 2ab191d1d6fdb..3e2392f1b8811 100644 --- a/website/cue/reference/components/sources/generated/opentelemetry.cue +++ b/website/cue/reference/components/sources/generated/opentelemetry.cue @@ -325,4 +325,15 @@ generated: components: sources: opentelemetry: configuration: { } } } + use_oltp_decoding: { + description: """ + Setting this field will override the legacy mapping of OTEL protos to Vector events and use the proto directly. + + One major caveat here is that the incoming metrics will be parsed as logs but they will preserve the OLTP format. + This means that components that work on metrics, will not be compatible with this output. + However, these events can be forwarded directly to a downstream OTEL collector. + """ + required: false + type: bool: default: false + } } From 910108804c4901e0c02af9516591c4d19d572b53 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 13:41:20 -0400 Subject: [PATCH 37/49] ran cargo fmt --- src/sources/opentelemetry/config.rs | 20 ++++++++++++-------- src/sources/opentelemetry/grpc.rs | 22 +++++++++++----------- src/sources/opentelemetry/http.rs | 12 ++++++------ 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 4b2d0bfff7f18..06209739f35f1 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,15 +1,14 @@ -use std::net::SocketAddr; - use futures::FutureExt; -use futures_util::{future::join, TryFutureExt}; +use futures_util::{TryFutureExt, future::join}; +use std::net::SocketAddr; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{log_schema, LegacyKey, LogNamespace}, + config::{LegacyKey, LogNamespace, log_schema}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{owned_value_path, OwnedTargetPath}, + lookup::{OwnedTargetPath, owned_value_path}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -34,17 +33,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ + Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, - Source, }, }; -use vrl::value::{kind::Collection, Kind}; +use vrl::value::{Kind, kind::Collection}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -364,9 +363,14 @@ impl SourceConfig for OpentelemetryConfig { } }; + let metrics_output = if self.use_oltp_decoding { + SourceOutput::new_maybe_logs(DataType::Log, Definition::any()).with_port(METRICS) + } else { + SourceOutput::new_metrics().with_port(METRICS) + }; vec![ SourceOutput::new_maybe_logs(DataType::Log, schema_definition).with_port(LOGS), - SourceOutput::new_metrics().with_port(METRICS), + metrics_output, SourceOutput::new_traces().with_port(TRACES), ] } diff --git a/src/sources/opentelemetry/grpc.rs b/src/sources/opentelemetry/grpc.rs index 3cd2c348b8b2b..4d0554ba271d1 100644 --- a/src/sources/opentelemetry/grpc.rs +++ b/src/sources/opentelemetry/grpc.rs @@ -1,31 +1,31 @@ use crate::sources::opentelemetry::config::METRICS; use crate::{ + SourceSender, internal_events::{EventsReceived, StreamClosedError}, sources::opentelemetry::config::{LOGS, TRACES}, - SourceSender, }; use futures::TryFutureExt; use prost::Message; use tonic::{Request, Response, Status}; -use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::codecs::decoding::ProtobufDeserializer; +use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::internal_event::{CountByteSize, InternalEventHandle as _, Registered}; use vector_lib::opentelemetry::proto::collector::{ logs::v1::{ - logs_service_server::LogsService, ExportLogsServiceRequest, ExportLogsServiceResponse, + ExportLogsServiceRequest, ExportLogsServiceResponse, logs_service_server::LogsService, }, metrics::v1::{ - metrics_service_server::MetricsService, ExportMetricsServiceRequest, - ExportMetricsServiceResponse, + ExportMetricsServiceRequest, ExportMetricsServiceResponse, + metrics_service_server::MetricsService, }, trace::v1::{ - trace_service_server::TraceService, ExportTraceServiceRequest, ExportTraceServiceResponse, + ExportTraceServiceRequest, ExportTraceServiceResponse, trace_service_server::TraceService, }, }; use vector_lib::{ + EstimatedJsonEncodedSizeOf, config::LogNamespace, event::{BatchNotifier, BatchStatus, BatchStatusReceiver, Event}, - EstimatedJsonEncodedSizeOf, }; #[derive(Clone)] @@ -111,10 +111,10 @@ impl MetricsService for Service { .map(|buf| buf.into_vec())? } else { request - .into_inner() - .resource_metrics - .into_iter() - .flat_map(|v| v.into_event_iter()) + .into_inner() + .resource_metrics + .into_iter() + .flat_map(|v| v.into_event_iter()) .collect() }; diff --git a/src/sources/opentelemetry/http.rs b/src/sources/opentelemetry/http.rs index 4c4f56e964bec..db5fc9752b241 100644 --- a/src/sources/opentelemetry/http.rs +++ b/src/sources/opentelemetry/http.rs @@ -5,28 +5,28 @@ use super::{reply::protobuf, status::Status}; use crate::common::http::ErrorMessage; use crate::http::{KeepaliveConfig, MaxConnectionAgeLayer}; use crate::sources::http_server::HttpConfigParamKind; -use crate::sources::opentelemetry::config::{OpentelemetryConfig, LOGS, METRICS, TRACES}; +use crate::sources::opentelemetry::config::{LOGS, METRICS, OpentelemetryConfig, TRACES}; use crate::sources::util::add_headers; use crate::{ + SourceSender, event::Event, http::build_http_trace_layer, internal_events::{EventsReceived, StreamClosedError}, shutdown::ShutdownSignal, sources::util::decode, tls::MaybeTlsSettings, - SourceSender, }; use bytes::Bytes; use futures_util::FutureExt; use http::StatusCode; -use hyper::{service::make_service_fn, Server}; +use hyper::{Server, service::make_service_fn}; use prost::Message; use snafu::Snafu; use tokio::net::TcpStream; use tower::ServiceBuilder; use tracing::Span; -use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::codecs::decoding::ProtobufDeserializer; +use vector_lib::codecs::decoding::format::Deserializer; use vector_lib::internal_event::{ ByteSize, BytesReceived, CountByteSize, InternalEventHandle as _, Registered, }; @@ -38,12 +38,12 @@ use vector_lib::opentelemetry::proto::collector::{ }; use vector_lib::tls::MaybeTlsIncomingStream; use vector_lib::{ + EstimatedJsonEncodedSizeOf, config::LogNamespace, event::{BatchNotifier, BatchStatus}, - EstimatedJsonEncodedSizeOf, }; use warp::{ - filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, Filter, Reply, + Filter, Reply, filters::BoxedFilter, http::HeaderMap, reject::Rejection, reply::Response, }; #[derive(Clone, Copy, Debug, Snafu)] From b537cd3597a3685a56b5009863fb31b08b6e5dd9 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 14:25:28 -0400 Subject: [PATCH 38/49] vrl update fixes --- lib/codecs/src/decoding/format/protobuf.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index 4c7643c78b5e4..dc44ff0f83fa0 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,16 +4,16 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{SmallVec, smallvec}; +use smallvec::{smallvec, SmallVec}; use vector_config::configurable_component; use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{DataType, LogNamespace, log_schema}, + config::{log_schema, DataType, LogNamespace}, event::Event, schema, }; use vrl::protobuf::descriptor::{get_message_descriptor, get_message_descriptor_from_bytes}; -use vrl::protobuf::parse::{Options, proto_to_value}; +use vrl::protobuf::parse::{proto_to_value, Options}; use vrl::value::{Kind, Value}; use super::Deserializer; @@ -189,7 +189,7 @@ mod tests { ) { let input = Bytes::from(protobuf_bin_message); let message_descriptor = - vrl::protobuf::get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); + get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); let deserializer = ProtobufDeserializer::new(message_descriptor); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { @@ -285,7 +285,7 @@ mod tests { #[test] fn deserialize_error_invalid_protobuf() { let input = Bytes::from("{ foo"); - let message_descriptor = vrl::protobuf::get_message_descriptor( + let message_descriptor = get_message_descriptor( &test_data_dir().join("protos/test_protobuf.desc"), "test_protobuf.Person", ) From db3b3248abc7493c92a21ebe1a25c4eeb8ee433e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Thu, 28 Aug 2025 16:47:16 -0400 Subject: [PATCH 39/49] ran cargo fmt --- lib/codecs/src/decoding/format/protobuf.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/codecs/src/decoding/format/protobuf.rs b/lib/codecs/src/decoding/format/protobuf.rs index dc44ff0f83fa0..beece2053a795 100644 --- a/lib/codecs/src/decoding/format/protobuf.rs +++ b/lib/codecs/src/decoding/format/protobuf.rs @@ -4,16 +4,16 @@ use bytes::Bytes; use chrono::Utc; use derivative::Derivative; use prost_reflect::{DynamicMessage, MessageDescriptor}; -use smallvec::{smallvec, SmallVec}; +use smallvec::{SmallVec, smallvec}; use vector_config::configurable_component; use vector_core::event::{LogEvent, TraceEvent}; use vector_core::{ - config::{log_schema, DataType, LogNamespace}, + config::{DataType, LogNamespace, log_schema}, event::Event, schema, }; use vrl::protobuf::descriptor::{get_message_descriptor, get_message_descriptor_from_bytes}; -use vrl::protobuf::parse::{proto_to_value, Options}; +use vrl::protobuf::parse::{Options, proto_to_value}; use vrl::value::{Kind, Value}; use super::Deserializer; @@ -188,8 +188,7 @@ mod tests { validate_log: fn(&LogEvent), ) { let input = Bytes::from(protobuf_bin_message); - let message_descriptor = - get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); + let message_descriptor = get_message_descriptor(&protobuf_desc_path, message_type).unwrap(); let deserializer = ProtobufDeserializer::new(message_descriptor); for namespace in [LogNamespace::Legacy, LogNamespace::Vector] { From 860808f07944c24fd5aedfefd7bda5e760d85e42 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 09:46:02 -0400 Subject: [PATCH 40/49] wip --- Cargo.lock | 56 +++++++++++++++++++----------------------------------- 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7ee182ff9031e..442faebe9ccf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3719,7 +3719,7 @@ dependencies = [ "bit-set", "derivative", "regex-automata 0.4.8", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -3730,7 +3730,7 @@ checksum = "bf04c5ec15464ace8355a7b440a33aece288993475556d461154d7a62ad9947c" dependencies = [ "bit-set", "regex-automata 0.4.8", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -4850,7 +4850,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.10", "tokio", "tower-service", "tracing 0.1.41", @@ -5596,7 +5596,7 @@ dependencies = [ "percent-encoding", "referencing", "regex", - "regex-syntax 0.8.5", + "regex-syntax", "serde", "serde_json", "uuid-simd", @@ -5800,7 +5800,7 @@ dependencies = [ "lalrpop-util", "petgraph", "regex", - "regex-syntax 0.8.5", + "regex-syntax", "sha3", "string_cache", "term 1.0.1", @@ -6145,11 +6145,11 @@ checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata 0.4.8", ] [[package]] @@ -6765,12 +6765,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" dependencies = [ - "overload", - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -7265,12 +7264,6 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "4.2.2" @@ -7944,7 +7937,7 @@ dependencies = [ "rand 0.9.2", "rand_chacha 0.9.0", "rand_xorshift", - "regex-syntax 0.8.5", + "regex-syntax", "rusty-fork", "tempfile", "unarray", @@ -8041,7 +8034,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.11.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -8087,7 +8080,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2 1.0.95", "quote 1.0.40", "syn 2.0.104", @@ -8739,7 +8732,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata 0.4.8", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -8747,9 +8740,6 @@ name = "regex-automata" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] [[package]] name = "regex-automata" @@ -8759,7 +8749,7 @@ checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -8773,7 +8763,7 @@ dependencies = [ "itertools 0.13.0", "nohash", "regex", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -8782,12 +8772,6 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30b661b2f27137bdbc16f00eda72866a92bb28af1753ffbd56744fb6e2e9cd8e" -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.5" @@ -11462,14 +11446,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata 0.4.8", "serde", "serde_json", "sharded-slab", From bcea37168e7c8c45610c784dfa62535483e94be1 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 09:50:31 -0400 Subject: [PATCH 41/49] chore(dev): cargo vdev build licenses --- LICENSE-3rdparty.csv | 3 --- 1 file changed, 3 deletions(-) diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index efbb8d68d4b70..6f7e83daaa9ee 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -458,7 +458,6 @@ openssl-probe,https://github.com/alexcrichton/openssl-probe,MIT OR Apache-2.0,Al openssl-sys,https://github.com/sfackler/rust-openssl,MIT,"Alex Crichton , Steven Fackler " ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem , Matt Brubeck " outref,https://github.com/Nugine/outref,MIT,The outref Authors -overload,https://github.com/danaugrs/overload,MIT,Daniel Salvadori owo-colors,https://github.com/owo-colors/owo-colors,MIT,jam1garner <8260240+jam1garner@users.noreply.github.com> p256,https://github.com/RustCrypto/elliptic-curves/tree/master/p256,Apache-2.0 OR MIT,RustCrypto Developers p384,https://github.com/RustCrypto/elliptic-curves/tree/master/p384,Apache-2.0 OR MIT,"RustCrypto Developers, Frank Denis " @@ -536,11 +535,9 @@ redox_syscall,https://gitlab.redox-os.org/redox-os/syscall,MIT,Jeremy Soller , Wesley Hershberger " ref-cast,https://github.com/dtolnay/ref-cast,MIT OR Apache-2.0,David Tolnay regex,https://github.com/rust-lang/regex,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-automata,https://github.com/BurntSushi/regex-automata,Unlicense OR MIT,Andrew Gallant regex-automata,https://github.com/rust-lang/regex/tree/master/regex-automata,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " regex-filtered,https://github.com/ua-parser/uap-rust,BSD-3-Clause,The regex-filtered Authors regex-lite,https://github.com/rust-lang/regex/tree/master/regex-lite,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " -regex-syntax,https://github.com/rust-lang/regex,MIT OR Apache-2.0,The Rust Project Developers regex-syntax,https://github.com/rust-lang/regex/tree/master/regex-syntax,MIT OR Apache-2.0,"The Rust Project Developers, Andrew Gallant " rend,https://github.com/djkoloski/rend,MIT,David Koloski reqwest,https://github.com/seanmonstar/reqwest,MIT OR Apache-2.0,Sean McArthur From af06922ce17c44722fec2a2d399dc66a35b15dae Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 10:16:07 -0400 Subject: [PATCH 42/49] fix typo and add first version of the changelog --- changelog.d/otlp_decoding.enhancement.md | 6 +++++ scripts/e2e/opentelemetry-logs/test.yaml | 2 +- src/sources/opentelemetry/config.rs | 20 +++++++------- .../opentelemetry/integration_tests.rs | 18 ++++++------- src/sources/opentelemetry/tests.rs | 26 +++++++++---------- .../e2e/opentelemetry/logs/vector_oltp.yaml | 2 +- .../sources/generated/opentelemetry.cue | 2 +- 7 files changed, 41 insertions(+), 35 deletions(-) create mode 100644 changelog.d/otlp_decoding.enhancement.md diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md new file mode 100644 index 0000000000000..43bda0f8b24d5 --- /dev/null +++ b/changelog.d/otlp_decoding.enhancement.md @@ -0,0 +1,6 @@ +The `opentelemetry` source now supports a new decoding mode which can be enabled by setting `use_otlp_decoding` to `true`. In this mode, +all events will preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to +the `opentelemetry` sink without modifications. + +A caveat here is that OTLP metrics and Vector metric format differ and thus we treat as logs as they come out the source. These events +cannot be used with existing metrics transforms. However, these can be ingested by the OTEL collectors as metrics. diff --git a/scripts/e2e/opentelemetry-logs/test.yaml b/scripts/e2e/opentelemetry-logs/test.yaml index 2a430e2e616ef..4063e7cc1214a 100644 --- a/scripts/e2e/opentelemetry-logs/test.yaml +++ b/scripts/e2e/opentelemetry-logs/test.yaml @@ -14,7 +14,7 @@ runner: matrix: # Determines which `otel/opentelemetry-collector-contrib` version to use collector_version: [ 'latest' ] - vector_config: [ 'vector_default.yaml', 'vector_oltp.yaml' ] + vector_config: [ 'vector_default.yaml', 'vector_otlp.yaml' ] # Only trigger this integration test if relevant OTEL source/sink files change paths: diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 06209739f35f1..4331a3410ed66 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,14 +1,14 @@ use futures::FutureExt; -use futures_util::{TryFutureExt, future::join}; +use futures_util::{future::join, TryFutureExt}; use std::net::SocketAddr; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{LegacyKey, LogNamespace, log_schema}, + config::{log_schema, LegacyKey, LogNamespace}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{OwnedTargetPath, owned_value_path}, + lookup::{owned_value_path, OwnedTargetPath}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -33,17 +33,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ - Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, + Source, }, }; -use vrl::value::{Kind, kind::Collection}; +use vrl::value::{kind::Collection, Kind}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; @@ -78,12 +78,12 @@ pub struct OpentelemetryConfig { /// Setting this field will override the legacy mapping of OTEL protos to Vector events and use the proto directly. /// - /// One major caveat here is that the incoming metrics will be parsed as logs but they will preserve the OLTP format. + /// One major caveat here is that the incoming metrics will be parsed as logs but they will preserve the OTLP format. /// This means that components that work on metrics, will not be compatible with this output. /// However, these events can be forwarded directly to a downstream OTEL collector. #[configurable(derived)] #[serde(default)] - pub use_oltp_decoding: bool, + pub use_otlp_decoding: bool, } /// Configuration for the `opentelemetry` gRPC server. @@ -161,7 +161,7 @@ impl GenerateConfig for OpentelemetryConfig { http: example_http_config(), acknowledgements: Default::default(), log_namespace: None, - use_oltp_decoding: false, + use_otlp_decoding: false, }) .unwrap() } @@ -172,7 +172,7 @@ impl OpentelemetryConfig { &self, message_type: &str, ) -> vector_common::Result> { - if self.use_oltp_decoding { + if self.use_otlp_decoding { let deserializer = ProtobufDeserializer::new_from_bytes( vector_lib::opentelemetry::proto::DESCRIPTOR_BYTES, message_type, @@ -363,7 +363,7 @@ impl SourceConfig for OpentelemetryConfig { } }; - let metrics_output = if self.use_oltp_decoding { + let metrics_output = if self.use_otlp_decoding { SourceOutput::new_maybe_logs(DataType::Log, Definition::any()).with_port(METRICS) } else { SourceOutput::new_metrics().with_port(METRICS) diff --git a/src/sources/opentelemetry/integration_tests.rs b/src/sources/opentelemetry/integration_tests.rs index d7cfed409d1b1..55bafbcc5dcbd 100644 --- a/src/sources/opentelemetry/integration_tests.rs +++ b/src/sources/opentelemetry/integration_tests.rs @@ -4,14 +4,14 @@ use itertools::Itertools; use serde_json::json; use crate::{ - config::{SourceConfig, SourceContext, log_schema}, + config::{log_schema, SourceConfig, SourceContext}, event::EventStatus, sources::opentelemetry::config::{ - GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, + GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS, TRACES, }, test_util::{ collect_n, - components::{SOURCE_TAGS, assert_source_compliance}, + components::{assert_source_compliance, SOURCE_TAGS}, retry_until, wait_for_tcp, }, }; @@ -20,10 +20,10 @@ use prost::Message; use super::tests::new_source; use vector_lib::opentelemetry::proto::{ collector::{metrics::v1::ExportMetricsServiceRequest, trace::v1::ExportTraceServiceRequest}, - common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value::Value::StringValue}, + common::v1::{any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue}, metrics::v1::{ - Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric::Data, - number_data_point::Value, + metric::Data, number_data_point::Value, Gauge, Metric, NumberDataPoint, ResourceMetrics, + ScopeMetrics, }, resource::v1::Resource, trace::v1::{ResourceSpans, ScopeSpans, Span}, @@ -63,7 +63,7 @@ async fn receive_logs_legacy_namespace() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - use_oltp_decoding: false, + use_otlp_decoding: false, }; let (sender, logs_output, _) = new_source(EventStatus::Delivered, LOGS.to_string()); @@ -162,7 +162,7 @@ async fn receive_trace() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - use_oltp_decoding: false, + use_otlp_decoding: false, }; let (sender, trace_output, _) = new_source(EventStatus::Delivered, TRACES.to_string()); @@ -267,7 +267,7 @@ async fn receive_metric() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - use_oltp_decoding: false, + use_otlp_decoding: false, }; let (sender, metrics_output, _) = new_source(EventStatus::Delivered, METRICS.to_string()); diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 12d29796480e8..24f68762840d1 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -2,17 +2,17 @@ use crate::config::OutputId; use crate::event::metric::{Bucket, Quantile}; use crate::event::{MetricKind, MetricTags, MetricValue}; use crate::{ - SourceSender, config::{SourceConfig, SourceContext}, event::{ - Event, EventStatus, LogEvent, Metric as MetricEvent, ObjectMap, Value, into_event_stream, + into_event_stream, Event, EventStatus, LogEvent, Metric as MetricEvent, ObjectMap, Value, }, - sources::opentelemetry::config::{GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig}, + sources::opentelemetry::config::{GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS}, test_util::{ self, - components::{SOURCE_TAGS, assert_source_compliance}, + components::{assert_source_compliance, SOURCE_TAGS}, next_addr, }, + SourceSender, }; use chrono::{DateTime, TimeZone, Utc}; use futures::Stream; @@ -26,18 +26,18 @@ use vector_lib::config::LogNamespace; use vector_lib::lookup::path; use vector_lib::opentelemetry::proto::{ collector::{ - logs::v1::{ExportLogsServiceRequest, logs_service_client::LogsServiceClient}, - metrics::v1::{ExportMetricsServiceRequest, metrics_service_client::MetricsServiceClient}, + logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, + metrics::v1::{metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest}, }, common::v1::{ - AnyValue, InstrumentationScope, KeyValue, any_value, any_value::Value::StringValue, + any_value, any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue, }, logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, metrics::v1::{ - AggregationTemporality, ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, - Histogram, HistogramDataPoint, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, - Summary, SummaryDataPoint, exponential_histogram_data_point::Buckets, metric::Data, - summary_data_point::ValueAtQuantile, + exponential_histogram_data_point::Buckets, metric::Data, summary_data_point::ValueAtQuantile, AggregationTemporality, + ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, Histogram, HistogramDataPoint, Metric, NumberDataPoint, + ResourceMetrics, ScopeMetrics, Sum, Summary, + SummaryDataPoint, }, resource::v1::{Resource, Resource as OtelResource}, }; @@ -1088,7 +1088,7 @@ async fn http_headers() { }, acknowledgements: Default::default(), log_namespace: Default::default(), - use_oltp_decoding: false, + use_otlp_decoding: false, }; let schema_definitions = source .outputs(LogNamespace::Legacy) @@ -1194,7 +1194,7 @@ pub async fn build_otlp_test_env( }, acknowledgements: Default::default(), log_namespace, - use_oltp_decoding: false, + use_otlp_decoding: false, }; let (sender, output, _) = new_source(EventStatus::Delivered, event_name.to_string()); diff --git a/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml b/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml index 6a1bcfb596904..a3b98647059c5 100644 --- a/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml +++ b/tests/data/e2e/opentelemetry/logs/vector_oltp.yaml @@ -8,7 +8,7 @@ sources: keepalive: max_connection_age_jitter_factor: 0.1 max_connection_age_secs: 300 - use_oltp_decoding: true + use_otlp_decoding: true internal_metrics: type: internal_metrics diff --git a/website/cue/reference/components/sources/generated/opentelemetry.cue b/website/cue/reference/components/sources/generated/opentelemetry.cue index 3e2392f1b8811..e8e4dc655dcc1 100644 --- a/website/cue/reference/components/sources/generated/opentelemetry.cue +++ b/website/cue/reference/components/sources/generated/opentelemetry.cue @@ -325,7 +325,7 @@ generated: components: sources: opentelemetry: configuration: { } } } - use_oltp_decoding: { + use_otlp_decoding: { description: """ Setting this field will override the legacy mapping of OTEL protos to Vector events and use the proto directly. From d4f723ff5114b2911783878e6d8fc5b4c1af8532 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 10:16:21 -0400 Subject: [PATCH 43/49] ran cargo fmt --- src/sources/opentelemetry/config.rs | 10 ++++----- .../opentelemetry/integration_tests.rs | 12 +++++----- src/sources/opentelemetry/tests.rs | 22 +++++++++---------- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index 4331a3410ed66..3c58bcdc33e51 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,14 +1,14 @@ use futures::FutureExt; -use futures_util::{future::join, TryFutureExt}; +use futures_util::{TryFutureExt, future::join}; use std::net::SocketAddr; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; use vector_lib::{ codecs::decoding::ProtobufDeserializer, - config::{log_schema, LegacyKey, LogNamespace}, + config::{LegacyKey, LogNamespace, log_schema}, configurable::configurable_component, internal_event::{BytesReceived, EventsReceived, Protocol}, - lookup::{owned_value_path, OwnedTargetPath}, + lookup::{OwnedTargetPath, owned_value_path}, opentelemetry::{ logs::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, @@ -33,17 +33,17 @@ use crate::{ http::KeepaliveConfig, serde::bool_or_struct, sources::{ + Source, http_server::{build_param_matcher, remove_duplicates}, opentelemetry::{ grpc::Service, http::{build_warp_filter, run_http_server}, }, util::grpc::run_grpc_server_with_routes, - Source, }, }; -use vrl::value::{kind::Collection, Kind}; +use vrl::value::{Kind, kind::Collection}; pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; diff --git a/src/sources/opentelemetry/integration_tests.rs b/src/sources/opentelemetry/integration_tests.rs index 55bafbcc5dcbd..d0af9142ad733 100644 --- a/src/sources/opentelemetry/integration_tests.rs +++ b/src/sources/opentelemetry/integration_tests.rs @@ -4,14 +4,14 @@ use itertools::Itertools; use serde_json::json; use crate::{ - config::{log_schema, SourceConfig, SourceContext}, + config::{SourceConfig, SourceContext, log_schema}, event::EventStatus, sources::opentelemetry::config::{ - GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS, TRACES, + GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig, TRACES, }, test_util::{ collect_n, - components::{assert_source_compliance, SOURCE_TAGS}, + components::{SOURCE_TAGS, assert_source_compliance}, retry_until, wait_for_tcp, }, }; @@ -20,10 +20,10 @@ use prost::Message; use super::tests::new_source; use vector_lib::opentelemetry::proto::{ collector::{metrics::v1::ExportMetricsServiceRequest, trace::v1::ExportTraceServiceRequest}, - common::v1::{any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue}, + common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value::Value::StringValue}, metrics::v1::{ - metric::Data, number_data_point::Value, Gauge, Metric, NumberDataPoint, ResourceMetrics, - ScopeMetrics, + Gauge, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, metric::Data, + number_data_point::Value, }, resource::v1::Resource, trace::v1::{ResourceSpans, ScopeSpans, Span}, diff --git a/src/sources/opentelemetry/tests.rs b/src/sources/opentelemetry/tests.rs index 24f68762840d1..69180c683a802 100644 --- a/src/sources/opentelemetry/tests.rs +++ b/src/sources/opentelemetry/tests.rs @@ -2,17 +2,17 @@ use crate::config::OutputId; use crate::event::metric::{Bucket, Quantile}; use crate::event::{MetricKind, MetricTags, MetricValue}; use crate::{ + SourceSender, config::{SourceConfig, SourceContext}, event::{ - into_event_stream, Event, EventStatus, LogEvent, Metric as MetricEvent, ObjectMap, Value, + Event, EventStatus, LogEvent, Metric as MetricEvent, ObjectMap, Value, into_event_stream, }, - sources::opentelemetry::config::{GrpcConfig, HttpConfig, OpentelemetryConfig, LOGS, METRICS}, + sources::opentelemetry::config::{GrpcConfig, HttpConfig, LOGS, METRICS, OpentelemetryConfig}, test_util::{ self, - components::{assert_source_compliance, SOURCE_TAGS}, + components::{SOURCE_TAGS, assert_source_compliance}, next_addr, }, - SourceSender, }; use chrono::{DateTime, TimeZone, Utc}; use futures::Stream; @@ -26,18 +26,18 @@ use vector_lib::config::LogNamespace; use vector_lib::lookup::path; use vector_lib::opentelemetry::proto::{ collector::{ - logs::v1::{logs_service_client::LogsServiceClient, ExportLogsServiceRequest}, - metrics::v1::{metrics_service_client::MetricsServiceClient, ExportMetricsServiceRequest}, + logs::v1::{ExportLogsServiceRequest, logs_service_client::LogsServiceClient}, + metrics::v1::{ExportMetricsServiceRequest, metrics_service_client::MetricsServiceClient}, }, common::v1::{ - any_value, any_value::Value::StringValue, AnyValue, InstrumentationScope, KeyValue, + AnyValue, InstrumentationScope, KeyValue, any_value, any_value::Value::StringValue, }, logs::v1::{LogRecord, ResourceLogs, ScopeLogs}, metrics::v1::{ - exponential_histogram_data_point::Buckets, metric::Data, summary_data_point::ValueAtQuantile, AggregationTemporality, - ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, Histogram, HistogramDataPoint, Metric, NumberDataPoint, - ResourceMetrics, ScopeMetrics, Sum, Summary, - SummaryDataPoint, + AggregationTemporality, ExponentialHistogram, ExponentialHistogramDataPoint, Gauge, + Histogram, HistogramDataPoint, Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, + Summary, SummaryDataPoint, exponential_histogram_data_point::Buckets, metric::Data, + summary_data_point::ValueAtQuantile, }, resource::v1::{Resource, Resource as OtelResource}, }; From 43a2bcd137a2b3e56f1687f57c9734de88620518 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 10:17:27 -0400 Subject: [PATCH 44/49] fix typo and add first version of the changelog --- changelog.d/otlp_decoding.enhancement.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md index 43bda0f8b24d5..e6b7b644d6b7b 100644 --- a/changelog.d/otlp_decoding.enhancement.md +++ b/changelog.d/otlp_decoding.enhancement.md @@ -1,6 +1,6 @@ The `opentelemetry` source now supports a new decoding mode which can be enabled by setting `use_otlp_decoding` to `true`. In this mode, -all events will preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to -the `opentelemetry` sink without modifications. +all events will preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to +the `opentelemetry` sink without modifications. -A caveat here is that OTLP metrics and Vector metric format differ and thus we treat as logs as they come out the source. These events +A caveat here is that OTLP metrics and Vector metric format differ and thus we treat as logs as they come out the source. These events cannot be used with existing metrics transforms. However, these can be ingested by the OTEL collectors as metrics. From afc48ae08f553d29140d6a54f920ad150afe248a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 10:22:06 -0400 Subject: [PATCH 45/49] add author --- changelog.d/otlp_decoding.enhancement.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md index e6b7b644d6b7b..4b1c0fb19bf76 100644 --- a/changelog.d/otlp_decoding.enhancement.md +++ b/changelog.d/otlp_decoding.enhancement.md @@ -4,3 +4,5 @@ the `opentelemetry` sink without modifications. A caveat here is that OTLP metrics and Vector metric format differ and thus we treat as logs as they come out the source. These events cannot be used with existing metrics transforms. However, these can be ingested by the OTEL collectors as metrics. + +authors: pront From 2ac056adb1237fc1b8f9dcf50621fbf07d713404 Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 12:21:04 -0400 Subject: [PATCH 46/49] Apply suggestions from code review Co-authored-by: Ursula Chen <58821586+urseberry@users.noreply.github.com> --- changelog.d/otlp_decoding.enhancement.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md index 4b1c0fb19bf76..eaf986eb69264 100644 --- a/changelog.d/otlp_decoding.enhancement.md +++ b/changelog.d/otlp_decoding.enhancement.md @@ -1,8 +1,8 @@ The `opentelemetry` source now supports a new decoding mode which can be enabled by setting `use_otlp_decoding` to `true`. In this mode, -all events will preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to +all events preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to the `opentelemetry` sink without modifications. -A caveat here is that OTLP metrics and Vector metric format differ and thus we treat as logs as they come out the source. These events -cannot be used with existing metrics transforms. However, these can be ingested by the OTEL collectors as metrics. +The OTLP metric format and the Vector metric format differ, so Vector treats [OTLP formatted events??] as logs as they come out the source. These events +cannot be used with existing metrics transforms. However, they can be ingested by the OTEL collectors as metrics. authors: pront From 40a76ee194d083ad1c00e253fd34ad99622a918d Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 12:25:34 -0400 Subject: [PATCH 47/49] tweaks to changelog --- changelog.d/otlp_decoding.enhancement.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md index eaf986eb69264..1eb836956f3fe 100644 --- a/changelog.d/otlp_decoding.enhancement.md +++ b/changelog.d/otlp_decoding.enhancement.md @@ -2,7 +2,7 @@ The `opentelemetry` source now supports a new decoding mode which can be enabled all events preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to the `opentelemetry` sink without modifications. -The OTLP metric format and the Vector metric format differ, so Vector treats [OTLP formatted events??] as logs as they come out the source. These events -cannot be used with existing metrics transforms. However, they can be ingested by the OTEL collectors as metrics. +**Note:** The OTLP metric format and the Vector metric format differ, so the `opentelemetry` source emits treats OTLP formatted metrics as Vector log +events. These events cannot be used with existing metrics transforms. However, they can be ingested by the OTEL collectors as metrics. authors: pront From 55eec575334a8021cc1f6229be9634a4fb8e817a Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 12:52:15 -0400 Subject: [PATCH 48/49] gen comp docs --- .../reference/components/sources/generated/opentelemetry.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/cue/reference/components/sources/generated/opentelemetry.cue b/website/cue/reference/components/sources/generated/opentelemetry.cue index e8e4dc655dcc1..4791cc458c8cf 100644 --- a/website/cue/reference/components/sources/generated/opentelemetry.cue +++ b/website/cue/reference/components/sources/generated/opentelemetry.cue @@ -329,7 +329,7 @@ generated: components: sources: opentelemetry: configuration: { description: """ Setting this field will override the legacy mapping of OTEL protos to Vector events and use the proto directly. - One major caveat here is that the incoming metrics will be parsed as logs but they will preserve the OLTP format. + One major caveat here is that the incoming metrics will be parsed as logs but they will preserve the OTLP format. This means that components that work on metrics, will not be compatible with this output. However, these events can be forwarded directly to a downstream OTEL collector. """ From 0980161e5b241699b462fbe5f70dc78b2b389a9e Mon Sep 17 00:00:00 2001 From: Pavlos Rontidis Date: Tue, 2 Sep 2025 15:23:18 -0400 Subject: [PATCH 49/49] fix changelog typo --- changelog.d/otlp_decoding.enhancement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/otlp_decoding.enhancement.md b/changelog.d/otlp_decoding.enhancement.md index 1eb836956f3fe..337671779505d 100644 --- a/changelog.d/otlp_decoding.enhancement.md +++ b/changelog.d/otlp_decoding.enhancement.md @@ -2,7 +2,7 @@ The `opentelemetry` source now supports a new decoding mode which can be enabled all events preserve the [OTLP](https://opentelemetry.io/docs/specs/otel/protocol/) format. These events can be forwarded directly to the `opentelemetry` sink without modifications. -**Note:** The OTLP metric format and the Vector metric format differ, so the `opentelemetry` source emits treats OTLP formatted metrics as Vector log +**Note:** The OTLP metric format and the Vector metric format differ, so the `opentelemetry` source emits OTLP formatted metrics as Vector log events. These events cannot be used with existing metrics transforms. However, they can be ingested by the OTEL collectors as metrics. authors: pront