From 31a860039c8584b434b87b4a8fcc1e405b438fc0 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 9 Dec 2021 14:51:35 -0500 Subject: [PATCH 1/4] ARROW-15044: [C++] Enable ostream span exporter for debugging --- cpp/CMakeLists.txt | 10 ++++++++-- cpp/src/arrow/util/tracing_internal.cc | 11 ++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 0262357d6c7..0e7b7b79a9f 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -766,9 +766,15 @@ if(ARROW_S3) endif() if(ARROW_WITH_OPENTELEMETRY) - list(APPEND ARROW_LINK_LIBS opentelemetry-cpp::trace + list(APPEND + ARROW_LINK_LIBS + opentelemetry-cpp::trace + opentelemetry-cpp::ostream_span_exporter opentelemetry-cpp::otlp_http_exporter) - list(APPEND ARROW_STATIC_LINK_LIBS opentelemetry-cpp::trace + list(APPEND + ARROW_STATIC_LINK_LIBS + opentelemetry-cpp::trace + opentelemetry-cpp::ostream_span_exporter opentelemetry-cpp::otlp_http_exporter) endif() diff --git a/cpp/src/arrow/util/tracing_internal.cc b/cpp/src/arrow/util/tracing_internal.cc index fe6422eaa35..2b76a8b3bc3 100644 --- a/cpp/src/arrow/util/tracing_internal.cc +++ b/cpp/src/arrow/util/tracing_internal.cc @@ -25,12 +25,12 @@ #pragma warning(push) #pragma warning(disable : 4522) #endif +#include #include #include #include #include #include -#include #include #ifdef _MSC_VER #pragma warning(pop) @@ -68,15 +68,12 @@ std::unique_ptr InitializeExporter() { auto maybe_env_var = arrow::internal::GetEnvVar(kTracingBackendEnvVar); if (maybe_env_var.ok()) { auto env_var = maybe_env_var.ValueOrDie(); - if (env_var == "otlp_http") { -#ifdef ARROW_WITH_OPENTELEMETRY + if (env_var == "ostream") { + return arrow::internal::make_unique(); + } else if (env_var == "otlp_http") { namespace otlp = opentelemetry::exporter::otlp; otlp::OtlpHttpExporterOptions opts; return arrow::internal::make_unique(opts); -#else - ARROW_LOG(WARNING) << "Requested " << kTracingBackendEnvVar << "=" < < < < env_var - " but Arrow was not built with ARROW_WITH_OPENTELEMETRY"; -#endif } else if (!env_var.empty()) { ARROW_LOG(WARNING) << "Requested unknown backend " << kTracingBackendEnvVar << "=" << env_var; From 623703be16f85d127b5bf12c223fc78ed7c72506 Mon Sep 17 00:00:00 2001 From: David Li Date: Thu, 9 Dec 2021 17:01:32 -0500 Subject: [PATCH 2/4] ARROW-15044: [C++] Enable OTLP stdout exporter for debugging --- cpp/src/arrow/util/tracing_internal.cc | 46 ++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/cpp/src/arrow/util/tracing_internal.cc b/cpp/src/arrow/util/tracing_internal.cc index 2b76a8b3bc3..628662fe5ba 100644 --- a/cpp/src/arrow/util/tracing_internal.cc +++ b/cpp/src/arrow/util/tracing_internal.cc @@ -25,13 +25,20 @@ #pragma warning(push) #pragma warning(disable : 4522) #endif +#include + #include #include +#include #include #include #include #include #include + +#include +#include +#include #ifdef _MSC_VER #pragma warning(pop) #endif @@ -53,6 +60,43 @@ namespace { namespace sdktrace = opentelemetry::sdk::trace; +// Custom JSON stdout exporter. Leverages the OTLP HTTP exporter's +// utilities to log the same format that would be sent to OTLP. +class OtlpStdoutExporter final : public sdktrace::SpanExporter { + public: + OtlpStdoutExporter() { protobuf_json_options_.add_whitespace = false; } + + std::unique_ptr MakeRecordable() noexcept override { + // The header for the Recordable definition is not installed, work around that + return exporter_.MakeRecordable(); + } + otel::sdk::common::ExportResult Export( + const nostd::span>& spans) noexcept override { + opentelemetry::proto::collector::trace::v1::ExportTraceServiceRequest request; + otel::exporter::otlp::OtlpRecordableUtils::PopulateRequest(spans, &request); + + for (const auto& spans : request.resource_spans()) { + std::string output; + auto status = google::protobuf::util::MessageToJsonString(spans, &output, + protobuf_json_options_); + if (ARROW_PREDICT_FALSE(!status.ok())) { + return otel::sdk::common::ExportResult::kFailure; + } + std::cout << output << std::endl; + } + + return otel::sdk::common::ExportResult::kSuccess; + } + bool Shutdown(std::chrono::microseconds timeout = + std::chrono::microseconds(0)) noexcept override { + return exporter_.Shutdown(timeout); + } + + private: + opentelemetry::exporter::otlp::OtlpHttpExporter exporter_; + google::protobuf::util::JsonPrintOptions protobuf_json_options_; +}; + class ThreadIdSpanProcessor : public sdktrace::BatchSpanProcessor { public: using sdktrace::BatchSpanProcessor::BatchSpanProcessor; @@ -74,6 +118,8 @@ std::unique_ptr InitializeExporter() { namespace otlp = opentelemetry::exporter::otlp; otlp::OtlpHttpExporterOptions opts; return arrow::internal::make_unique(opts); + } else if (env_var == "arrow_otlp_stdout") { + return arrow::internal::make_unique(); } else if (!env_var.empty()) { ARROW_LOG(WARNING) << "Requested unknown backend " << kTracingBackendEnvVar << "=" << env_var; From 9961764792592f721465e12edcea573bdfce266b Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 15 Dec 2021 15:46:01 -0500 Subject: [PATCH 3/4] ARROW-15044: [C++] Add stderr exporter --- cpp/src/arrow/util/tracing_internal.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cpp/src/arrow/util/tracing_internal.cc b/cpp/src/arrow/util/tracing_internal.cc index 628662fe5ba..2e6a71a9621 100644 --- a/cpp/src/arrow/util/tracing_internal.cc +++ b/cpp/src/arrow/util/tracing_internal.cc @@ -60,11 +60,13 @@ namespace { namespace sdktrace = opentelemetry::sdk::trace; -// Custom JSON stdout exporter. Leverages the OTLP HTTP exporter's -// utilities to log the same format that would be sent to OTLP. -class OtlpStdoutExporter final : public sdktrace::SpanExporter { +// Custom JSON exporter. Leverages the OTLP HTTP exporter's utilities +// to log the same format that would be sent to OTLP. +class OtlpOStreamExporter final : public sdktrace::SpanExporter { public: - OtlpStdoutExporter() { protobuf_json_options_.add_whitespace = false; } + OtlpOStreamExporter(std::basic_ostream* out) : out_(out) { + protobuf_json_options_.add_whitespace = false; + } std::unique_ptr MakeRecordable() noexcept override { // The header for the Recordable definition is not installed, work around that @@ -82,7 +84,7 @@ class OtlpStdoutExporter final : public sdktrace::SpanExporter { if (ARROW_PREDICT_FALSE(!status.ok())) { return otel::sdk::common::ExportResult::kFailure; } - std::cout << output << std::endl; + (*out_) << output << std::endl; } return otel::sdk::common::ExportResult::kSuccess; @@ -93,6 +95,7 @@ class OtlpStdoutExporter final : public sdktrace::SpanExporter { } private: + std::basic_ostream* out_; opentelemetry::exporter::otlp::OtlpHttpExporter exporter_; google::protobuf::util::JsonPrintOptions protobuf_json_options_; }; @@ -119,7 +122,9 @@ std::unique_ptr InitializeExporter() { otlp::OtlpHttpExporterOptions opts; return arrow::internal::make_unique(opts); } else if (env_var == "arrow_otlp_stdout") { - return arrow::internal::make_unique(); + return arrow::internal::make_unique(&std::cout); + } else if (env_var == "arrow_otlp_stderr") { + return arrow::internal::make_unique(&std::cerr); } else if (!env_var.empty()) { ARROW_LOG(WARNING) << "Requested unknown backend " << kTracingBackendEnvVar << "=" << env_var; From f7cbd4d215ce83a149e461db2a3d5f0dce6532ec Mon Sep 17 00:00:00 2001 From: David Li Date: Wed, 15 Dec 2021 16:23:14 -0500 Subject: [PATCH 4/4] ARROW-15044: [C++] Fix lint --- cpp/src/arrow/util/tracing_internal.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/arrow/util/tracing_internal.cc b/cpp/src/arrow/util/tracing_internal.cc index 2e6a71a9621..d39f95061c7 100644 --- a/cpp/src/arrow/util/tracing_internal.cc +++ b/cpp/src/arrow/util/tracing_internal.cc @@ -64,7 +64,7 @@ namespace sdktrace = opentelemetry::sdk::trace; // to log the same format that would be sent to OTLP. class OtlpOStreamExporter final : public sdktrace::SpanExporter { public: - OtlpOStreamExporter(std::basic_ostream* out) : out_(out) { + explicit OtlpOStreamExporter(std::basic_ostream* out) : out_(out) { protobuf_json_options_.add_whitespace = false; }