diff --git a/Cargo.lock b/Cargo.lock index 9d2a9e0fab8fc..f5b9f0e938dd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2308,6 +2308,7 @@ dependencies = [ "indoc", "influxdb-line-protocol", "memchr", + "opentelemetry-proto", "ordered-float 4.6.0", "prost 0.12.6", "prost-reflect", @@ -8061,7 +8062,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck 0.5.0", - "itertools 0.10.5", + "itertools 0.11.0", "log", "multimap", "once_cell", @@ -8107,7 +8108,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.11.0", "proc-macro2 1.0.101", "quote 1.0.40", "syn 2.0.106", diff --git a/Cargo.toml b/Cargo.toml index 22ba045541fcc..99eb681d45fa1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -577,6 +577,7 @@ enrichment-tables-memory = ["dep:evmap", "dep:evmap-derive", "dep:thread_local"] # Codecs codecs-syslog = ["vector-lib/syslog"] +codecs-opentelemetry = ["vector-lib/opentelemetry"] # Secrets secrets = ["secrets-aws-secrets-manager"] @@ -668,7 +669,17 @@ sources-mqtt = ["dep:rumqttc"] sources-nats = ["dep:async-nats", "dep:nkeys"] sources-nginx_metrics = ["dep:nom"] sources-okta = ["sources-utils-http-client"] -sources-opentelemetry = ["dep:hex", "vector-lib/opentelemetry", "dep:prost", "dep:prost-types", "sources-http_server", "sources-utils-http", "sources-utils-http-headers", "sources-vector"] +sources-opentelemetry = [ + "dep:hex", + "codecs-opentelemetry", + "vector-lib/opentelemetry", + "dep:prost", + "dep:prost-types", + "sources-http_server", + "sources-utils-http", + "sources-utils-http-headers", + "sources-vector", +] sources-postgresql_metrics = ["dep:postgres-openssl", "dep:tokio-postgres"] sources-prometheus = ["sources-prometheus-scrape", "sources-prometheus-remote-write", "sources-prometheus-pushgateway"] sources-prometheus-scrape = ["sinks-prometheus", "sources-utils-http-client", "vector-lib/prometheus"] @@ -855,7 +866,7 @@ sinks-mqtt = ["dep:rumqttc"] sinks-nats = ["dep:async-nats", "dep:nkeys"] sinks-new_relic_logs = ["sinks-http"] sinks-new_relic = [] -sinks-opentelemetry = ["sinks-http"] +sinks-opentelemetry = ["sinks-http", "codecs-opentelemetry"] sinks-papertrail = ["dep:syslog"] sinks-prometheus = ["dep:base64", "dep:prost", "vector-lib/prometheus"] sinks-postgres = ["dep:sqlx"] @@ -1004,7 +1015,8 @@ e2e-tests-opentelemetry = [ "sources-internal_metrics", "transforms-remap", "sinks-console", - "sinks-file" + "sinks-file", + "codecs-opentelemetry", ] vector-api-tests = [ diff --git a/changelog.d/otlp_encoding.feature.md b/changelog.d/otlp_encoding.feature.md new file mode 100644 index 0000000000000..513bcf4379d93 --- /dev/null +++ b/changelog.d/otlp_encoding.feature.md @@ -0,0 +1,4 @@ +Added `otlp` codec for encoding Vector events to OTLP format. +The codec can be used with sinks that support encoding configuration. + +authors: pront diff --git a/lib/codecs/Cargo.toml b/lib/codecs/Cargo.toml index 28f5411ef3cd0..8e4ce532c54b6 100644 --- a/lib/codecs/Cargo.toml +++ b/lib/codecs/Cargo.toml @@ -20,6 +20,7 @@ flate2.workspace = true influxdb-line-protocol = { version = "2", default-features = false } lookup = { package = "vector-lookup", path = "../vector-lookup", default-features = false, features = ["test"] } memchr = { version = "2", default-features = false } +opentelemetry-proto = { path = "../opentelemetry-proto", optional = true } ordered-float.workspace = true prost.workspace = true prost-reflect.workspace = true @@ -53,3 +54,4 @@ vrl.workspace = true [features] syslog = ["dep:syslog_loose"] +opentelemetry = ["dep:opentelemetry-proto"] diff --git a/lib/codecs/src/encoding/format/mod.rs b/lib/codecs/src/encoding/format/mod.rs index d699e3f9942c4..9377cdca5d906 100644 --- a/lib/codecs/src/encoding/format/mod.rs +++ b/lib/codecs/src/encoding/format/mod.rs @@ -12,6 +12,8 @@ mod json; mod logfmt; mod native; mod native_json; +#[cfg(feature = "opentelemetry")] +mod otlp; mod protobuf; mod raw_message; mod text; @@ -26,6 +28,8 @@ pub use json::{JsonSerializer, JsonSerializerConfig, JsonSerializerOptions}; pub use logfmt::{LogfmtSerializer, LogfmtSerializerConfig}; pub use native::{NativeSerializer, NativeSerializerConfig}; pub use native_json::{NativeJsonSerializer, NativeJsonSerializerConfig}; +#[cfg(feature = "opentelemetry")] +pub use otlp::{OtlpSerializer, OtlpSerializerConfig}; pub use protobuf::{ProtobufSerializer, ProtobufSerializerConfig, ProtobufSerializerOptions}; pub use raw_message::{RawMessageSerializer, RawMessageSerializerConfig}; pub use text::{TextSerializer, TextSerializerConfig}; diff --git a/lib/codecs/src/encoding/format/otlp.rs b/lib/codecs/src/encoding/format/otlp.rs new file mode 100644 index 0000000000000..fd0ffffe47b04 --- /dev/null +++ b/lib/codecs/src/encoding/format/otlp.rs @@ -0,0 +1,132 @@ +use crate::encoding::ProtobufSerializer; +use bytes::BytesMut; +use opentelemetry_proto::proto::{ + DESCRIPTOR_BYTES, LOGS_REQUEST_MESSAGE_TYPE, METRICS_REQUEST_MESSAGE_TYPE, + RESOURCE_LOGS_JSON_FIELD, RESOURCE_METRICS_JSON_FIELD, RESOURCE_SPANS_JSON_FIELD, + TRACES_REQUEST_MESSAGE_TYPE, +}; +use tokio_util::codec::Encoder; +use vector_config_macros::configurable_component; +use vector_core::{config::DataType, event::Event, schema}; +use vrl::protobuf::encode::Options; + +/// Config used to build an `OtlpSerializer`. +#[configurable_component] +#[derive(Debug, Clone, Default)] +pub struct OtlpSerializerConfig { + // No configuration options needed - OTLP serialization is opinionated +} + +impl OtlpSerializerConfig { + /// Build the `OtlpSerializer` from this configuration. + pub fn build(&self) -> Result { + OtlpSerializer::new() + } + + /// The data type of events that are accepted by `OtlpSerializer`. + pub fn input_type(&self) -> DataType { + DataType::Log | DataType::Trace + } + + /// The schema required by the serializer. + pub fn schema_requirement(&self) -> schema::Requirement { + schema::Requirement::empty() + } +} + +/// Serializer that converts an `Event` to bytes using the OTLP (OpenTelemetry Protocol) protobuf format. +/// +/// This serializer encodes events using the OTLP protobuf specification, which is the recommended +/// encoding format for OpenTelemetry data. The output is suitable for sending to OTLP-compatible +/// endpoints with `content-type: application/x-protobuf`. +/// +/// # Implementation approach +/// +/// This serializer converts Vector's internal event representation to the appropriate OTLP message type +/// based on the top-level field in the event: +/// - `resourceLogs` → `ExportLogsServiceRequest` +/// - `resourceMetrics` → `ExportMetricsServiceRequest` +/// - `resourceSpans` → `ExportTraceServiceRequest` +/// +/// The implementation is the inverse of what the `opentelemetry` source does when decoding, +/// ensuring round-trip compatibility. +#[derive(Debug, Clone)] +#[allow(dead_code)] // Fields will be used once encoding is implemented +pub struct OtlpSerializer { + logs_descriptor: ProtobufSerializer, + metrics_descriptor: ProtobufSerializer, + traces_descriptor: ProtobufSerializer, + options: Options, +} + +impl OtlpSerializer { + /// Creates a new OTLP serializer with the appropriate message descriptors. + pub fn new() -> vector_common::Result { + let options = Options { + use_json_names: true, + }; + + let logs_descriptor = ProtobufSerializer::new_from_bytes( + DESCRIPTOR_BYTES, + LOGS_REQUEST_MESSAGE_TYPE, + &options, + )?; + + let metrics_descriptor = ProtobufSerializer::new_from_bytes( + DESCRIPTOR_BYTES, + METRICS_REQUEST_MESSAGE_TYPE, + &options, + )?; + + let traces_descriptor = ProtobufSerializer::new_from_bytes( + DESCRIPTOR_BYTES, + TRACES_REQUEST_MESSAGE_TYPE, + &options, + )?; + + Ok(Self { + logs_descriptor, + metrics_descriptor, + traces_descriptor, + options, + }) + } +} + +impl Encoder for OtlpSerializer { + type Error = vector_common::Error; + + fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { + // Determine which descriptor to use based on top-level OTLP fields + // This handles events that were decoded with use_otlp_decoding enabled + // The deserializer uses use_json_names: true, so fields are in camelCase + match &event { + Event::Log(log) => { + if log.contains(RESOURCE_LOGS_JSON_FIELD) { + self.logs_descriptor.encode(event, buffer) + } else if log.contains(RESOURCE_METRICS_JSON_FIELD) { + // Currently the OTLP metrics are Vector logs (not metrics). + self.metrics_descriptor.encode(event, buffer) + } else { + Err(format!( + "Log event does not contain OTLP top-level fields ({RESOURCE_LOGS_JSON_FIELD} or {RESOURCE_METRICS_JSON_FIELD})", + ) + .into()) + } + } + Event::Trace(trace) => { + if trace.contains(RESOURCE_SPANS_JSON_FIELD) { + self.traces_descriptor.encode(event, buffer) + } else { + Err(format!( + "Trace event does not contain OTLP top-level field ({RESOURCE_SPANS_JSON_FIELD})", + ) + .into()) + } + } + Event::Metric(_) => { + Err("OTLP serializer does not support native Vector metrics yet.".into()) + } + } + } +} diff --git a/lib/codecs/src/encoding/format/protobuf.rs b/lib/codecs/src/encoding/format/protobuf.rs index 4c0ba5a9090e1..1f3d3b7a64939 100644 --- a/lib/codecs/src/encoding/format/protobuf.rs +++ b/lib/codecs/src/encoding/format/protobuf.rs @@ -10,8 +10,10 @@ use vector_core::{ event::{Event, Value}, schema, }; -use vrl::protobuf::encode::Options; -use vrl::protobuf::{descriptor::get_message_descriptor, encode::encode_message}; +use vrl::protobuf::{ + descriptor::{get_message_descriptor, get_message_descriptor_from_bytes}, + encode::{Options, encode_message}, +}; /// Config used to build a `ProtobufSerializer`. #[configurable_component] @@ -26,7 +28,10 @@ impl ProtobufSerializerConfig { pub fn build(&self) -> Result { let message_descriptor = get_message_descriptor(&self.protobuf.desc_file, &self.protobuf.message_type)?; - Ok(ProtobufSerializer { message_descriptor }) + Ok(ProtobufSerializer { + message_descriptor, + options: Options::default(), + }) } /// The data type of events that are accepted by `ProtobufSerializer`. @@ -64,12 +69,29 @@ pub struct ProtobufSerializerOptions { pub struct ProtobufSerializer { /// The protobuf message definition to use for serialization. message_descriptor: MessageDescriptor, + options: Options, } impl ProtobufSerializer { /// Creates a new `ProtobufSerializer`. pub fn new(message_descriptor: MessageDescriptor) -> Self { - Self { message_descriptor } + Self { + message_descriptor, + options: Options::default(), + } + } + + /// Creates a new serializer instance using the descriptor bytes directly. + pub fn new_from_bytes( + desc_bytes: &[u8], + message_type: &str, + options: &Options, + ) -> vector_common::Result { + let message_descriptor = get_message_descriptor_from_bytes(desc_bytes, message_type)?; + Ok(Self { + message_descriptor, + options: options.clone(), + }) } /// Get a description of the message type used in serialization. @@ -83,16 +105,14 @@ impl Encoder for ProtobufSerializer { fn encode(&mut self, event: Event, buffer: &mut BytesMut) -> Result<(), Self::Error> { let message = match event { - Event::Log(log) => encode_message( - &self.message_descriptor, - log.into_parts().0, - &Options::default(), - ), + Event::Log(log) => { + encode_message(&self.message_descriptor, log.into_parts().0, &self.options) + } Event::Metric(_) => unimplemented!(), Event::Trace(trace) => encode_message( &self.message_descriptor, Value::Object(trace.into_parts().0), - &Options::default(), + &self.options, ), }?; message.encode(buffer).map_err(Into::into) diff --git a/lib/codecs/src/encoding/mod.rs b/lib/codecs/src/encoding/mod.rs index 91e45ffc6d1c7..21b634602d302 100644 --- a/lib/codecs/src/encoding/mod.rs +++ b/lib/codecs/src/encoding/mod.rs @@ -18,6 +18,8 @@ pub use format::{ ProtobufSerializerOptions, RawMessageSerializer, RawMessageSerializerConfig, TextSerializer, TextSerializerConfig, }; +#[cfg(feature = "opentelemetry")] +pub use format::{OtlpSerializer, OtlpSerializerConfig}; pub use framing::{ BoxedFramer, BoxedFramingError, BytesEncoder, BytesEncoderConfig, CharacterDelimitedEncoder, CharacterDelimitedEncoderConfig, CharacterDelimitedEncoderOptions, LengthDelimitedEncoder, @@ -268,6 +270,16 @@ pub enum SerializerConfig { /// [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs NativeJson, + /// Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + /// + /// This codec uses protobuf encoding, which is the recommended format for OTLP. + /// The output is suitable for sending to OTLP-compatible endpoints with + /// `content-type: application/x-protobuf`. + /// + /// [otlp]: https://opentelemetry.io/docs/specs/otlp/ + #[cfg(feature = "opentelemetry")] + Otlp, + /// Encodes an event as a [Protobuf][protobuf] message. /// /// [protobuf]: https://protobuf.dev/ @@ -293,6 +305,12 @@ pub enum SerializerConfig { Text(TextSerializerConfig), } +impl Default for SerializerConfig { + fn default() -> Self { + Self::Json(JsonSerializerConfig::default()) + } +} + impl From for SerializerConfig { fn from(config: AvroSerializerConfig) -> Self { Self::Avro { avro: config.avro } @@ -341,6 +359,13 @@ impl From for SerializerConfig { } } +#[cfg(feature = "opentelemetry")] +impl From for SerializerConfig { + fn from(_: OtlpSerializerConfig) -> Self { + Self::Otlp + } +} + impl From for SerializerConfig { fn from(config: ProtobufSerializerConfig) -> Self { Self::Protobuf(config) @@ -375,6 +400,10 @@ impl SerializerConfig { SerializerConfig::NativeJson => { Ok(Serializer::NativeJson(NativeJsonSerializerConfig.build())) } + #[cfg(feature = "opentelemetry")] + SerializerConfig::Otlp => { + Ok(Serializer::Otlp(OtlpSerializerConfig::default().build()?)) + } SerializerConfig::Protobuf(config) => Ok(Serializer::Protobuf(config.build()?)), SerializerConfig::RawMessage => { Ok(Serializer::RawMessage(RawMessageSerializerConfig.build())) @@ -400,6 +429,8 @@ impl SerializerConfig { SerializerConfig::Avro { .. } | SerializerConfig::Native => { FramingConfig::LengthDelimited(LengthDelimitedEncoderConfig::default()) } + #[cfg(feature = "opentelemetry")] + SerializerConfig::Otlp => FramingConfig::Bytes, SerializerConfig::Protobuf(_) => { FramingConfig::VarintLengthDelimited(VarintLengthDelimitedEncoderConfig::default()) } @@ -429,6 +460,8 @@ impl SerializerConfig { SerializerConfig::Logfmt => LogfmtSerializerConfig.input_type(), SerializerConfig::Native => NativeSerializerConfig.input_type(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.input_type(), + #[cfg(feature = "opentelemetry")] + SerializerConfig::Otlp => OtlpSerializerConfig::default().input_type(), SerializerConfig::Protobuf(config) => config.input_type(), SerializerConfig::RawMessage => RawMessageSerializerConfig.input_type(), SerializerConfig::Text(config) => config.input_type(), @@ -448,6 +481,8 @@ impl SerializerConfig { SerializerConfig::Logfmt => LogfmtSerializerConfig.schema_requirement(), SerializerConfig::Native => NativeSerializerConfig.schema_requirement(), SerializerConfig::NativeJson => NativeJsonSerializerConfig.schema_requirement(), + #[cfg(feature = "opentelemetry")] + SerializerConfig::Otlp => OtlpSerializerConfig::default().schema_requirement(), SerializerConfig::Protobuf(config) => config.schema_requirement(), SerializerConfig::RawMessage => RawMessageSerializerConfig.schema_requirement(), SerializerConfig::Text(config) => config.schema_requirement(), @@ -474,6 +509,9 @@ pub enum Serializer { Native(NativeSerializer), /// Uses a `NativeJsonSerializer` for serialization. NativeJson(NativeJsonSerializer), + /// Uses an `OtlpSerializer` for serialization. + #[cfg(feature = "opentelemetry")] + Otlp(OtlpSerializer), /// Uses a `ProtobufSerializer` for serialization. Protobuf(ProtobufSerializer), /// Uses a `RawMessageSerializer` for serialization. @@ -495,6 +533,8 @@ impl Serializer { | Serializer::Native(_) | Serializer::Protobuf(_) | Serializer::RawMessage(_) => false, + #[cfg(feature = "opentelemetry")] + Serializer::Otlp(_) => false, } } @@ -519,6 +559,10 @@ impl Serializer { | Serializer::RawMessage(_) => { panic!("Serializer does not support JSON") } + #[cfg(feature = "opentelemetry")] + Serializer::Otlp(_) => { + panic!("Serializer does not support JSON") + } } } @@ -579,6 +623,13 @@ impl From for Serializer { } } +#[cfg(feature = "opentelemetry")] +impl From for Serializer { + fn from(serializer: OtlpSerializer) -> Self { + Self::Otlp(serializer) + } +} + impl From for Serializer { fn from(serializer: ProtobufSerializer) -> Self { Self::Protobuf(serializer) @@ -610,6 +661,8 @@ impl tokio_util::codec::Encoder for Serializer { Serializer::Logfmt(serializer) => serializer.encode(event, buffer), Serializer::Native(serializer) => serializer.encode(event, buffer), Serializer::NativeJson(serializer) => serializer.encode(event, buffer), + #[cfg(feature = "opentelemetry")] + Serializer::Otlp(serializer) => serializer.encode(event, buffer), Serializer::Protobuf(serializer) => serializer.encode(event, buffer), Serializer::RawMessage(serializer) => serializer.encode(event, buffer), Serializer::Text(serializer) => serializer.encode(event, buffer), diff --git a/lib/opentelemetry-proto/src/proto.rs b/lib/opentelemetry-proto/src/proto.rs index 5559113bd14db..8b9fd9f460c90 100644 --- a/lib/opentelemetry-proto/src/proto.rs +++ b/lib/opentelemetry-proto/src/proto.rs @@ -1,3 +1,15 @@ +pub const LOGS_REQUEST_MESSAGE_TYPE: &str = + "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest"; +pub const TRACES_REQUEST_MESSAGE_TYPE: &str = + "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest"; +pub const METRICS_REQUEST_MESSAGE_TYPE: &str = + "opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest"; + +// JSON names (camelCase) for the same fields, used when use_json_names is enabled +pub const RESOURCE_LOGS_JSON_FIELD: &str = "resourceLogs"; +pub const RESOURCE_METRICS_JSON_FIELD: &str = "resourceMetrics"; +pub const RESOURCE_SPANS_JSON_FIELD: &str = "resourceSpans"; + /// Service stub and clients. pub mod collector { pub mod trace { diff --git a/lib/vector-lib/Cargo.toml b/lib/vector-lib/Cargo.toml index 89ae3bed7547d..394af42114d71 100644 --- a/lib/vector-lib/Cargo.toml +++ b/lib/vector-lib/Cargo.toml @@ -27,7 +27,7 @@ api = ["vector-tap/api"] api-client = ["dep:vector-api-client"] lua = ["vector-core/lua"] file-source = ["dep:file-source", "dep:file-source-common"] -opentelemetry = ["dep:opentelemetry-proto"] +opentelemetry = ["dep:opentelemetry-proto", "codecs/opentelemetry"] prometheus = ["dep:prometheus-parser"] proptest = ["vector-lookup/proptest", "vrl/proptest"] syslog = ["codecs/syslog"] diff --git a/src/codecs/encoding/config.rs b/src/codecs/encoding/config.rs index 756bc8e2406f2..a04f44315047a 100644 --- a/src/codecs/encoding/config.rs +++ b/src/codecs/encoding/config.rs @@ -1,3 +1,4 @@ +use crate::codecs::Transformer; use vector_lib::{ codecs::{ CharacterDelimitedEncoder, LengthDelimitedEncoder, NewlineDelimitedEncoder, @@ -6,7 +7,8 @@ use vector_lib::{ configurable::configurable_component, }; -use crate::codecs::Transformer; +#[cfg(feature = "codecs-opentelemetry")] +use vector_lib::codecs::BytesEncoder; /// Encoding configuration. #[configurable_component] @@ -130,6 +132,8 @@ impl EncodingConfigWithFraming { | Serializer::RawMessage(_) | Serializer::Text(_), ) => NewlineDelimitedEncoder::default().into(), + #[cfg(feature = "codecs-opentelemetry")] + (None, Serializer::Otlp(_)) => BytesEncoder.into(), }; Ok((framer, serializer)) diff --git a/src/codecs/encoding/encoder.rs b/src/codecs/encoding/encoder.rs index 14642bb462f82..f1d0741bb669c 100644 --- a/src/codecs/encoding/encoder.rs +++ b/src/codecs/encoding/encoder.rs @@ -128,6 +128,8 @@ impl Encoder { | Serializer::Text(_), _, ) => "text/plain", + #[cfg(feature = "codecs-opentelemetry")] + (Serializer::Otlp(_), _) => "application/x-protobuf", } } } diff --git a/src/components/validation/resources/mod.rs b/src/components/validation/resources/mod.rs index a0dc1cecad0c5..a79100791f6c1 100644 --- a/src/components/validation/resources/mod.rs +++ b/src/components/validation/resources/mod.rs @@ -233,6 +233,8 @@ fn serializer_config_to_deserializer( }) } SerializerConfig::RawMessage | SerializerConfig::Text(_) => DeserializerConfig::Bytes, + #[cfg(feature = "codecs-opentelemetry")] + SerializerConfig::Otlp => todo!(), }; deserializer_config.build() diff --git a/src/sinks/http/config.rs b/src/sinks/http/config.rs index ab72afb9c4cab..2c9f257fae5e1 100644 --- a/src/sinks/http/config.rs +++ b/src/sinks/http/config.rs @@ -270,6 +270,8 @@ impl SinkConfig for HttpSinkConfig { (Json(_), CharacterDelimited(CharacterDelimitedEncoder { delimiter: b',' })) => { Some(CONTENT_TYPE_JSON.to_owned()) } + #[cfg(feature = "codecs-opentelemetry")] + (Otlp(_), _) => Some("application/x-protobuf".to_owned()), _ => None, } }; diff --git a/src/sinks/websocket/sink.rs b/src/sinks/websocket/sink.rs index 1a378803e4fd5..4127ec886f3dd 100644 --- a/src/sinks/websocket/sink.rs +++ b/src/sinks/websocket/sink.rs @@ -4,6 +4,15 @@ use std::{ time::{Duration, Instant}, }; +use crate::{ + codecs::{Encoder, Transformer}, + common::websocket::{PingInterval, WebSocketConnector, is_closed}, + event::{Event, EventStatus, Finalizable}, + internal_events::{ + ConnectionOpen, OpenGauge, WebSocketConnectionError, WebSocketConnectionShutdown, + }, + sinks::{util::StreamSink, websocket::config::WebSocketSinkConfig}, +}; use async_trait::async_trait; use bytes::BytesMut; use futures::{Sink, Stream, StreamExt, pin_mut, sink::SinkExt, stream::BoxStream}; @@ -16,15 +25,8 @@ use vector_lib::{ }, }; -use crate::{ - codecs::{Encoder, Transformer}, - common::websocket::{PingInterval, WebSocketConnector, is_closed}, - event::{Event, EventStatus, Finalizable}, - internal_events::{ - ConnectionOpen, OpenGauge, WebSocketConnectionError, WebSocketConnectionShutdown, - }, - sinks::{util::StreamSink, websocket::config::WebSocketSinkConfig}, -}; +#[cfg(feature = "codecs-opentelemetry")] +use vector_lib::codecs::encoding::Serializer::Otlp; pub struct WebSocketSink { transformer: Transformer, @@ -82,6 +84,8 @@ impl WebSocketSink { match self.encoder.serializer() { RawMessage(_) | Avro(_) | Native(_) | Protobuf(_) => true, + #[cfg(feature = "codecs-opentelemetry")] + Otlp(_) => true, Cef(_) | Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | Text(_) | NativeJson(_) => false, } } diff --git a/src/sinks/websocket_server/sink.rs b/src/sinks/websocket_server/sink.rs index 033248d573aa7..978e407be2e5e 100644 --- a/src/sinks/websocket_server/sink.rs +++ b/src/sinks/websocket_server/sink.rs @@ -4,6 +4,24 @@ use std::{ sync::{Arc, Mutex}, }; +use super::{ + WebSocketListenerSinkConfig, + buffering::MessageBufferingConfig, + config::{ExtraMetricTagsConfig, SubProtocolConfig}, +}; +use crate::{ + codecs::{Encoder, Transformer}, + common::http::server_auth::HttpServerAuthMatcher, + internal_events::{ + ConnectionOpen, OpenGauge, WebSocketListenerConnectionEstablished, + WebSocketListenerConnectionFailedError, WebSocketListenerConnectionShutdown, + WebSocketListenerMessageSent, WebSocketListenerSendError, + }, + sinks::{ + prelude::*, + websocket_server::buffering::{BufferReplayRequest, WsMessageBufferConfig}, + }, +}; use async_trait::async_trait; use bytes::BytesMut; use futures::{ @@ -22,6 +40,8 @@ use tokio_util::codec::Encoder as _; use tracing::Instrument; use url::Url; use uuid::Uuid; +#[cfg(feature = "codecs-opentelemetry")] +use vector_lib::codecs::encoding::Serializer::Otlp; use vector_lib::{ EstimatedJsonEncodedSizeOf, event::{Event, EventStatus}, @@ -33,25 +53,6 @@ use vector_lib::{ tls::{MaybeTlsIncomingStream, MaybeTlsListener, MaybeTlsSettings}, }; -use super::{ - WebSocketListenerSinkConfig, - buffering::MessageBufferingConfig, - config::{ExtraMetricTagsConfig, SubProtocolConfig}, -}; -use crate::{ - codecs::{Encoder, Transformer}, - common::http::server_auth::HttpServerAuthMatcher, - internal_events::{ - ConnectionOpen, OpenGauge, WebSocketListenerConnectionEstablished, - WebSocketListenerConnectionFailedError, WebSocketListenerConnectionShutdown, - WebSocketListenerMessageSent, WebSocketListenerSendError, - }, - sinks::{ - prelude::*, - websocket_server::buffering::{BufferReplayRequest, WsMessageBufferConfig}, - }, -}; - pub struct WebSocketListenerSink { tls: MaybeTlsSettings, transformer: Transformer, @@ -93,6 +94,8 @@ impl WebSocketListenerSink { match self.encoder.serializer() { RawMessage(_) | Avro(_) | Native(_) | Protobuf(_) => true, + #[cfg(feature = "codecs-opentelemetry")] + Otlp(_) => true, Cef(_) | Csv(_) | Logfmt(_) | Gelf(_) | Json(_) | Text(_) | NativeJson(_) => false, } } diff --git a/src/sources/opentelemetry/config.rs b/src/sources/opentelemetry/config.rs index b2457f8f70113..1c6b30bc23a82 100644 --- a/src/sources/opentelemetry/config.rs +++ b/src/sources/opentelemetry/config.rs @@ -1,5 +1,22 @@ use std::net::SocketAddr; +use crate::{ + config::{ + DataType, GenerateConfig, Resource, SourceAcknowledgementsConfig, SourceConfig, + SourceContext, SourceOutput, + }, + http::KeepaliveConfig, + serde::bool_or_struct, + sources::{ + Source, + http_server::{build_param_matcher, remove_duplicates}, + opentelemetry::{ + grpc::Service, + http::{build_warp_filter, run_http_server}, + }, + util::grpc::run_grpc_server_with_routes, + }, +}; use futures::FutureExt; use futures_util::{TryFutureExt, future::join}; use tonic::{codec::CompressionEncoding, transport::server::RoutesBuilder}; @@ -14,10 +31,13 @@ use vector_lib::{ ATTRIBUTES_KEY, DROPPED_ATTRIBUTES_COUNT_KEY, FLAGS_KEY, OBSERVED_TIMESTAMP_KEY, RESOURCE_KEY, SEVERITY_NUMBER_KEY, SEVERITY_TEXT_KEY, SPAN_ID_KEY, TRACE_ID_KEY, }, - proto::collector::{ - logs::v1::logs_service_server::LogsServiceServer, - metrics::v1::metrics_service_server::MetricsServiceServer, - trace::v1::trace_service_server::TraceServiceServer, + proto::{ + LOGS_REQUEST_MESSAGE_TYPE, METRICS_REQUEST_MESSAGE_TYPE, TRACES_REQUEST_MESSAGE_TYPE, + collector::{ + logs::v1::logs_service_server::LogsServiceServer, + metrics::v1::metrics_service_server::MetricsServiceServer, + trace::v1::trace_service_server::TraceServiceServer, + }, }, }, schema::Definition, @@ -28,35 +48,10 @@ use vrl::{ value::{Kind, kind::Collection}, }; -use crate::{ - config::{ - DataType, GenerateConfig, Resource, SourceAcknowledgementsConfig, SourceConfig, - SourceContext, SourceOutput, - }, - http::KeepaliveConfig, - serde::bool_or_struct, - sources::{ - Source, - http_server::{build_param_matcher, remove_duplicates}, - opentelemetry::{ - grpc::Service, - http::{build_warp_filter, run_http_server}, - }, - util::grpc::run_grpc_server_with_routes, - }, -}; - pub const LOGS: &str = "logs"; pub const METRICS: &str = "metrics"; pub const TRACES: &str = "traces"; -pub const OTEL_PROTO_LOGS_REQUEST: &str = - "opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest"; -pub const OTEL_PROTO_TRACES_REQUEST: &str = - "opentelemetry.proto.collector.trace.v1.ExportTraceServiceRequest"; -pub const OTEL_PROTO_METRICS_REQUEST: &str = - "opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest"; - /// Configuration for the `opentelemetry` source. #[configurable_component(source("opentelemetry", "Receive OTLP data through gRPC or HTTP."))] #[derive(Clone, Debug)] @@ -198,7 +193,7 @@ impl SourceConfig for OpentelemetryConfig { let grpc_tls_settings = MaybeTlsSettings::from_config(self.grpc.tls.as_ref(), true)?; - let log_deserializer = self.get_deserializer(OTEL_PROTO_LOGS_REQUEST)?; + let log_deserializer = self.get_deserializer(LOGS_REQUEST_MESSAGE_TYPE)?; let log_service = LogsServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, @@ -209,7 +204,7 @@ impl SourceConfig for OpentelemetryConfig { .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); - let metric_deserializer = self.get_deserializer(OTEL_PROTO_METRICS_REQUEST)?; + let metric_deserializer = self.get_deserializer(METRICS_REQUEST_MESSAGE_TYPE)?; let metrics_service = MetricsServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, @@ -220,7 +215,7 @@ impl SourceConfig for OpentelemetryConfig { .accept_compressed(CompressionEncoding::Gzip) .max_decoding_message_size(usize::MAX); - let trace_deserializer = self.get_deserializer(OTEL_PROTO_TRACES_REQUEST)?; + let trace_deserializer = self.get_deserializer(TRACES_REQUEST_MESSAGE_TYPE)?; let trace_service = TraceServiceServer::new(Service { pipeline: cx.out.clone(), acknowledgements, diff --git a/tests/data/e2e/opentelemetry/logs/vector_otlp.yaml b/tests/data/e2e/opentelemetry/logs/vector_otlp.yaml index c2d3e55549ae1..2c4be656844fb 100644 --- a/tests/data/e2e/opentelemetry/logs/vector_otlp.yaml +++ b/tests/data/e2e/opentelemetry/logs/vector_otlp.yaml @@ -22,16 +22,8 @@ sinks: protocol: type: http uri: http://otel-collector-sink:5318/v1/logs - method: post encoding: - codec: json - framing: - method: newline_delimited - batch: - max_events: 1 - request: - headers: - content-type: application/json + codec: otlp otel_file_sink: type: file diff --git a/tests/e2e/opentelemetry/logs/mod.rs b/tests/e2e/opentelemetry/logs/mod.rs index d3ade1c6d2a15..f20433912744f 100644 --- a/tests/e2e/opentelemetry/logs/mod.rs +++ b/tests/e2e/opentelemetry/logs/mod.rs @@ -1,8 +1,5 @@ use serde_json::Value; -use std::collections::BTreeMap; -use std::io; -use std::path::Path; -use std::process::Command; +use std::{collections::BTreeMap, io, path::Path, process::Command}; const EXPECTED_LOG_COUNT: usize = 100; diff --git a/website/cue/reference/components/sinks/generated/amqp.cue b/website/cue/reference/components/sinks/generated/amqp.cue index 0d24b39aa8e93..5887fb620c615 100644 --- a/website/cue/reference/components/sinks/generated/amqp.cue +++ b/website/cue/reference/components/sinks/generated/amqp.cue @@ -204,6 +204,15 @@ generated: components: sinks: amqp: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/generated/aws_cloudwatch_logs.cue index 3857573f5996e..c1b0e1f494998 100644 --- a/website/cue/reference/components/sinks/generated/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/generated/aws_cloudwatch_logs.cue @@ -400,6 +400,15 @@ generated: components: sinks: aws_cloudwatch_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/aws_kinesis_firehose.cue b/website/cue/reference/components/sinks/generated/aws_kinesis_firehose.cue index 93ddcadd6e25d..ff1d1cdec8c85 100644 --- a/website/cue/reference/components/sinks/generated/aws_kinesis_firehose.cue +++ b/website/cue/reference/components/sinks/generated/aws_kinesis_firehose.cue @@ -379,6 +379,15 @@ generated: components: sinks: aws_kinesis_firehose: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/aws_kinesis_streams.cue b/website/cue/reference/components/sinks/generated/aws_kinesis_streams.cue index 7dbe25f9021ea..150b66142bbec 100644 --- a/website/cue/reference/components/sinks/generated/aws_kinesis_streams.cue +++ b/website/cue/reference/components/sinks/generated/aws_kinesis_streams.cue @@ -379,6 +379,15 @@ generated: components: sinks: aws_kinesis_streams: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/aws_s3.cue b/website/cue/reference/components/sinks/generated/aws_s3.cue index a0e9b3c76a67d..0a50a9da454e6 100644 --- a/website/cue/reference/components/sinks/generated/aws_s3.cue +++ b/website/cue/reference/components/sinks/generated/aws_s3.cue @@ -488,6 +488,15 @@ generated: components: sinks: aws_s3: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/aws_sns.cue b/website/cue/reference/components/sinks/generated/aws_sns.cue index 70ce7e9a48d5a..67e018c83f76e 100644 --- a/website/cue/reference/components/sinks/generated/aws_sns.cue +++ b/website/cue/reference/components/sinks/generated/aws_sns.cue @@ -310,6 +310,15 @@ generated: components: sinks: aws_sns: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/aws_sqs.cue b/website/cue/reference/components/sinks/generated/aws_sqs.cue index 181730563df9e..626413075a3ef 100644 --- a/website/cue/reference/components/sinks/generated/aws_sqs.cue +++ b/website/cue/reference/components/sinks/generated/aws_sqs.cue @@ -310,6 +310,15 @@ generated: components: sinks: aws_sqs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/azure_blob.cue b/website/cue/reference/components/sinks/generated/azure_blob.cue index f63fd171a75d5..1099e8e686f9f 100644 --- a/website/cue/reference/components/sinks/generated/azure_blob.cue +++ b/website/cue/reference/components/sinks/generated/azure_blob.cue @@ -334,6 +334,15 @@ generated: components: sinks: azure_blob: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/console.cue b/website/cue/reference/components/sinks/generated/console.cue index c0a52286f6af2..01a8b3764f6c7 100644 --- a/website/cue/reference/components/sinks/generated/console.cue +++ b/website/cue/reference/components/sinks/generated/console.cue @@ -188,6 +188,15 @@ generated: components: sinks: console: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/file.cue b/website/cue/reference/components/sinks/generated/file.cue index df1022a2f2797..472622b5a4f55 100644 --- a/website/cue/reference/components/sinks/generated/file.cue +++ b/website/cue/reference/components/sinks/generated/file.cue @@ -208,6 +208,15 @@ generated: components: sinks: file: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/gcp_chronicle_unstructured.cue b/website/cue/reference/components/sinks/generated/gcp_chronicle_unstructured.cue index 69727212e5235..e4633659e88ce 100644 --- a/website/cue/reference/components/sinks/generated/gcp_chronicle_unstructured.cue +++ b/website/cue/reference/components/sinks/generated/gcp_chronicle_unstructured.cue @@ -276,6 +276,15 @@ generated: components: sinks: gcp_chronicle_unstructured: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/gcp_cloud_storage.cue b/website/cue/reference/components/sinks/generated/gcp_cloud_storage.cue index 524669bbaa830..0a96d8a53eacf 100644 --- a/website/cue/reference/components/sinks/generated/gcp_cloud_storage.cue +++ b/website/cue/reference/components/sinks/generated/gcp_cloud_storage.cue @@ -349,6 +349,15 @@ generated: components: sinks: gcp_cloud_storage: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/gcp_pubsub.cue b/website/cue/reference/components/sinks/generated/gcp_pubsub.cue index a56f9979bd540..fd972d8bd933c 100644 --- a/website/cue/reference/components/sinks/generated/gcp_pubsub.cue +++ b/website/cue/reference/components/sinks/generated/gcp_pubsub.cue @@ -255,6 +255,15 @@ generated: components: sinks: gcp_pubsub: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/http.cue b/website/cue/reference/components/sinks/generated/http.cue index f71f8411b630d..97d6d17178bd5 100644 --- a/website/cue/reference/components/sinks/generated/http.cue +++ b/website/cue/reference/components/sinks/generated/http.cue @@ -431,6 +431,15 @@ generated: components: sinks: http: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/humio_logs.cue b/website/cue/reference/components/sinks/generated/humio_logs.cue index cbe42c0dad7f0..afe28006824d8 100644 --- a/website/cue/reference/components/sinks/generated/humio_logs.cue +++ b/website/cue/reference/components/sinks/generated/humio_logs.cue @@ -254,6 +254,15 @@ generated: components: sinks: humio_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/kafka.cue b/website/cue/reference/components/sinks/generated/kafka.cue index a8f8b1b0baacd..a90c3e1bf0857 100644 --- a/website/cue/reference/components/sinks/generated/kafka.cue +++ b/website/cue/reference/components/sinks/generated/kafka.cue @@ -243,6 +243,15 @@ generated: components: sinks: kafka: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/loki.cue b/website/cue/reference/components/sinks/generated/loki.cue index 13eb339ff98cd..b83cd54280f7f 100644 --- a/website/cue/reference/components/sinks/generated/loki.cue +++ b/website/cue/reference/components/sinks/generated/loki.cue @@ -433,6 +433,15 @@ generated: components: sinks: loki: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/mqtt.cue b/website/cue/reference/components/sinks/generated/mqtt.cue index ada012a020233..c2d192d35ac3e 100644 --- a/website/cue/reference/components/sinks/generated/mqtt.cue +++ b/website/cue/reference/components/sinks/generated/mqtt.cue @@ -198,6 +198,15 @@ generated: components: sinks: mqtt: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/nats.cue b/website/cue/reference/components/sinks/generated/nats.cue index 3c802edad2783..84cba5611b2af 100644 --- a/website/cue/reference/components/sinks/generated/nats.cue +++ b/website/cue/reference/components/sinks/generated/nats.cue @@ -288,6 +288,15 @@ generated: components: sinks: nats: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/opentelemetry.cue b/website/cue/reference/components/sinks/generated/opentelemetry.cue index 29c2008b0f327..e0b4e672fef2e 100644 --- a/website/cue/reference/components/sinks/generated/opentelemetry.cue +++ b/website/cue/reference/components/sinks/generated/opentelemetry.cue @@ -434,6 +434,15 @@ generated: components: sinks: opentelemetry: configuration: protocol: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/papertrail.cue b/website/cue/reference/components/sinks/generated/papertrail.cue index b5f1728e984de..8704ca7a46d57 100644 --- a/website/cue/reference/components/sinks/generated/papertrail.cue +++ b/website/cue/reference/components/sinks/generated/papertrail.cue @@ -188,6 +188,15 @@ generated: components: sinks: papertrail: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/pulsar.cue b/website/cue/reference/components/sinks/generated/pulsar.cue index 4e2034a0c81a5..c5545a0f17082 100644 --- a/website/cue/reference/components/sinks/generated/pulsar.cue +++ b/website/cue/reference/components/sinks/generated/pulsar.cue @@ -322,6 +322,15 @@ generated: components: sinks: pulsar: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/redis.cue b/website/cue/reference/components/sinks/generated/redis.cue index 6d365d7decdeb..2f5ac1cc0890b 100644 --- a/website/cue/reference/components/sinks/generated/redis.cue +++ b/website/cue/reference/components/sinks/generated/redis.cue @@ -247,6 +247,15 @@ generated: components: sinks: redis: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/socket.cue b/website/cue/reference/components/sinks/generated/socket.cue index 753f6a4a02f05..17531379e6155 100644 --- a/website/cue/reference/components/sinks/generated/socket.cue +++ b/website/cue/reference/components/sinks/generated/socket.cue @@ -200,6 +200,15 @@ generated: components: sinks: socket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/splunk_hec_logs.cue b/website/cue/reference/components/sinks/generated/splunk_hec_logs.cue index 860ce6a0fb6e5..1a68a339a166d 100644 --- a/website/cue/reference/components/sinks/generated/splunk_hec_logs.cue +++ b/website/cue/reference/components/sinks/generated/splunk_hec_logs.cue @@ -304,6 +304,15 @@ generated: components: sinks: splunk_hec_logs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/webhdfs.cue b/website/cue/reference/components/sinks/generated/webhdfs.cue index a491ddb52c64a..3a6b5703f5c7f 100644 --- a/website/cue/reference/components/sinks/generated/webhdfs.cue +++ b/website/cue/reference/components/sinks/generated/webhdfs.cue @@ -254,6 +254,15 @@ generated: components: sinks: webhdfs: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/websocket.cue b/website/cue/reference/components/sinks/generated/websocket.cue index 817e8247ff15e..a6f86e05d4ff2 100644 --- a/website/cue/reference/components/sinks/generated/websocket.cue +++ b/website/cue/reference/components/sinks/generated/websocket.cue @@ -360,6 +360,15 @@ generated: components: sinks: websocket: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sinks/generated/websocket_server.cue b/website/cue/reference/components/sinks/generated/websocket_server.cue index 0817002c06acf..ca04a07cfee13 100644 --- a/website/cue/reference/components/sinks/generated/websocket_server.cue +++ b/website/cue/reference/components/sinks/generated/websocket_server.cue @@ -244,6 +244,15 @@ generated: components: sinks: websocket_server: configuration: { [vector_native_json]: https://github.com/vectordotdev/vector/blob/master/lib/codecs/tests/data/native_encoding/schema.cue [experimental]: https://vector.dev/highlights/2022-03-31-native-event-codecs """ + otlp: """ + Encodes an event in the [OTLP (OpenTelemetry Protocol)][otlp] format. + + This codec uses protobuf encoding, which is the recommended format for OTLP. + The output is suitable for sending to OTLP-compatible endpoints with + `content-type: application/x-protobuf`. + + [otlp]: https://opentelemetry.io/docs/specs/otlp/ + """ protobuf: """ Encodes an event as a [Protobuf][protobuf] message. diff --git a/website/cue/reference/components/sources/opentelemetry.cue b/website/cue/reference/components/sources/opentelemetry.cue index 622e4e8c684f3..b18a3474e382b 100644 --- a/website/cue/reference/components/sources/opentelemetry.cue +++ b/website/cue/reference/components/sources/opentelemetry.cue @@ -298,26 +298,8 @@ components: sources: opentelemetry: { protocol: type: http uri: http://localhost:5318/v1/logs - method: post - encoding: - codec: protobuf - protobuf: - desc_file: path/to/opentelemetry-proto.desc - message_type: opentelemetry.proto.collector.logs.v1.ExportLogsServiceRequest - framing: - method: "bytes" - request: - headers: - content-type: "application/x-protobuf" - ``` - - The `desc` file was generated with the following command: - ```bash - protoc -I=/path/to/vector/lib/opentelemetry-proto/src/proto/opentelemetry-proto \\ - --include_imports \\ - --include_source_info \\ - --descriptor_set_out=opentelemetry-proto.desc \\ - $(find /path/to/vector/lib/opentelemetry-proto/src/proto/opentelemetry-proto -name '*.proto') + encoding: + codec: otlp ``` """ }