From 07459dc435d536b402dff936ad088f2a21615ee8 Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Wed, 5 Aug 2020 19:08:25 -0400 Subject: [PATCH 1/8] add prometheus exporter utils implementation --- .../src/prometheus_exporter_utils.cc | 495 ++++++++++++++++++ 1 file changed, 495 insertions(+) create mode 100644 exporters/prometheus/src/prometheus_exporter_utils.cc diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc new file mode 100644 index 0000000000..56059148ab --- /dev/null +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -0,0 +1,495 @@ +#include +#include +#include +#include + +#include "opentelemetry/exporters/prometheus/prometheus_exporter_utils.h" +#include "opentelemetry/sdk/metrics/aggregator/aggregator.h" +#include "prometheus/metric_type.h" + +#define QUANTILE_STEP 0.25 + +OPENTELEMETRY_BEGIN_NAMESPACE +namespace exporter +{ +namespace prometheus +{ +/** + * Helper function to convert OpenTelemetry metrics data collection + * to Prometheus metrics data collection + * + * @param records a collection of metrics in OpenTelemetry + * @return a collection of translated metrics that is acceptable by Prometheus + */ +std::vector PrometheusExporterUtils::TranslateToPrometheus( + std::vector &records) +{ + if (records.empty()) + { + return {}; + } + + // initialize output vector + std::vector output(records.size()); + + // iterate through the vector and set result data into it + int i = 0; + for (auto r : records) + { + SetMetricFamily(r, &output[i]); + i++; + } + + return output; +} + +// ======================= private helper functions ========================= + +/** + * Set value to metric family according to record + * + * @param record + * @param metric_family + */ +void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, + prometheus_client::MetricFamily *metric_family) +{ + try + { + metric_family->name = SanitizeNames(record.GetName()); + } + catch (std::invalid_argument &e) + { + metric_family->name = record.GetName(); + } + + metric_family->help = record.GetDescription(); + + // unpack the variant and set the metric data to metric family struct + auto agg_var = record.GetAggregator(); + auto labels_str = record.GetLabels(); + if (nostd::holds_alternative>>(agg_var)) + { + auto aggregator = nostd::get>>(agg_var); + SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); + } + else if (nostd::holds_alternative>>(agg_var)) + { + auto aggregator = nostd::get>>(agg_var); + SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); + } + else if (nostd::holds_alternative>>( + record.GetAggregator())) + { + auto aggregator = nostd::get>>(agg_var); + SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); + } + else if (nostd::holds_alternative>>( + record.GetAggregator())) + { + auto aggregator = nostd::get>>(agg_var); + SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); + } +} + +/** + * Sanitize the given metric name or label according to Prometheus rule. + * + * This function is needed because names in OpenTelemetry can contain + * alphanumeric characters, '_', '.', and '-', whereas in Prometheus the + * name should only contain alphanumeric characters and '_'. + * + * @param key name in OpenTelemetry + * @return sanitized name in Prometheus + */ +std::string PrometheusExporterUtils::SanitizeNames(std::string name) +{ + // name cannot be null or empty. + // a valid OTel name should only contain alphanumeric characters, + // '_', '.', or '-'. + // meaningful exception message + if (!IsValidName(name)) + { + throw std::invalid_argument("Received an invalid OTel name.\n"); + } + + // replace all '.' and '-' with '_' + std::replace(name.begin(), name.end(), '.', '_'); + std::replace(name.begin(), name.end(), '-', '_'); + + // if the replaced name starts with '_', it's also invalid + if (name[0] == '_') + { + throw std::invalid_argument("Received an invalid OTel name.\n"); + } + + return name; +} + +/** + * Determine whether the input name is a valid OTel name or not + * + * From the spec: + * 1. They are non-empty strings + * 2. They are case-insensitive + * 3. The first character must be non-numeric, non-space, non-punctuation + * 4. Subsequent characters must belong to the alphanumeric characters, '_', '.', and '-'. + * + * @param name name to be determined + * @return a boolean flag + */ +bool PrometheusExporterUtils::IsValidName(const std::string &name) +{ + if (name.empty()) + { + return false; + } + + if (isnumber(name[0]) || isspace(name[0]) || ispunct(name[0])) + { + return false; + } + + for (std::string::size_type i = 1; i < name.size(); i++) + { + if (!isalnum(name[i]) && name[i] != '_' && name[i] != '.' && name[i] != '-') + { + return false; + } + } + + return true; +} + +template +void PrometheusExporterUtils::SetMetricFamilyByAggregator( + nostd::shared_ptr> aggregator, + std::string labels_str, + prometheus_client::MetricFamily *metric_family) +{ + // get aggregator kind and translate to Prometheus metric type + auto kind = aggregator->get_aggregator_kind(); + const prometheus_client::MetricType type = TranslateType(kind); + metric_family->type = type; + + // get check-pointed values, label string and check-pointed time + auto checkpointed_values = aggregator->get_checkpoint(); + auto time = aggregator->get_checkpoint_timestamp().time_since_epoch(); + + if (type == prometheus_client::MetricType::Histogram) // Histogram + { + auto boundaries = aggregator->get_boundaries(); + auto counts = aggregator->get_counts(); + SetData(checkpointed_values, boundaries, counts, labels_str, time, metric_family); + } + else if (type == prometheus_client::MetricType::Summary) // Sketch, Exact + { + if (kind == metric_sdk::AggregatorKind::Exact) + { + // TODO: 1. what if this Exact aggregator is not for quantile estimation? + // TODO: 2. how many quantile samples should I include in each metric? + if (aggregator->get_quant_estimation()) + { + auto quantiles = GetQuantilesVector(aggregator); + SetData(checkpointed_values, quantiles, labels_str, time, metric_family); + } + } + else if (kind == metric_sdk::AggregatorKind::Sketch) + {} + } + else // Counter, Gauge, MinMaxSumCount, Untyped + { + // Handle MinMaxSumCount: https://github.com/open-telemetry/opentelemetry-cpp/issues/228 + // Use sum/count is ok. + if (kind == metric_sdk::AggregatorKind::MinMaxSumCount) + { + int sum = checkpointed_values[2]; + int count = checkpointed_values[3]; + double avg = (double)sum / count; + SetData(avg, labels_str, time, metric_family); + } + else + { + SetData(checkpointed_values, labels_str, type, time, metric_family); + } + } +} + +/** + * Translate the OTel metric type to Prometheus metric type + * + * @param record + * @return + */ +prometheus_client::MetricType PrometheusExporterUtils::TranslateType( + metric_sdk::AggregatorKind kind) +{ + switch (kind) + { + case metric_sdk::AggregatorKind::Counter: + return prometheus_client::MetricType::Counter; + case metric_sdk::AggregatorKind::Gauge: + case metric_sdk::AggregatorKind::MinMaxSumCount: + return prometheus_client::MetricType::Gauge; + case metric_sdk::AggregatorKind::Histogram: + return prometheus_client::MetricType::Histogram; + case metric_sdk::AggregatorKind::Sketch: + case metric_sdk::AggregatorKind::Exact: + return prometheus_client::MetricType::Summary; + default: + return prometheus_client::MetricType::Untyped; + } +} + +/** + * Set metric data for: + * Counter => Prometheus Counter + * Gauge => Prometheus Gauge + */ +template +void PrometheusExporterUtils::SetData(std::vector values, + const std::string &labels, + prometheus_client::MetricType type, + std::chrono::nanoseconds time, + prometheus_client::MetricFamily *metric_family) +{ + metric_family->metric.emplace_back(); + prometheus_client::ClientMetric &metric = metric_family->metric.back(); + SetMetricBasic(metric, time, labels); + SetValue(values, type, &metric); +} + +/** + * Set metric data for: + * Histogram => Prometheus Histogram + */ +template +void PrometheusExporterUtils::SetData(std::vector values, + const std::vector &boundaries, + const std::vector &counts, + const std::string &labels, + std::chrono::nanoseconds time, + prometheus_client::MetricFamily *metric_family) +{ + metric_family->metric.emplace_back(); + prometheus_client::ClientMetric &metric = metric_family->metric.back(); + SetMetricBasic(metric, time, labels); + SetValue(values, boundaries, counts, &metric); +} + +/** + * Set metric data for: + * MinMaxSumCount => Prometheus Gauge + * Use Average (sum / count) as the gauge metric + */ +void PrometheusExporterUtils::SetData(double value, + const std::string &labels, + std::chrono::nanoseconds time, + prometheus_client::MetricFamily *metric_family) +{ + metric_family->metric.emplace_back(); + prometheus_client::ClientMetric &metric = metric_family->metric.back(); + SetMetricBasic(metric, time, labels); + SetValue(value, &metric); +} + +/** + * Set metric data for: + * Exact => Prometheus Summary + */ +template +void PrometheusExporterUtils::SetData(std::vector values, + const std::vector &quantiles, + const std::string &labels, + std::chrono::nanoseconds time, + prometheus_client::MetricFamily *metric_family) +{ + metric_family->metric.emplace_back(); + prometheus_client::ClientMetric &metric = metric_family->metric.back(); + SetMetricBasic(metric, time, labels); + SetValue(values, quantiles, &metric); +} + +void PrometheusExporterUtils::SetMetricBasic(prometheus_client::ClientMetric &metric, + std::chrono::nanoseconds time, + const std::string &labels) +{ + metric.timestamp_ms = time.count() / 1000; + + auto label_pairs = ParseLabel(labels); + if (!label_pairs.empty()) + { + metric.label.resize(label_pairs.size()); + for (int i = 0; i < label_pairs.size(); ++i) + { + try + { + metric.label[i].name = SanitizeNames(label_pairs[i].first); + } + catch (std::invalid_argument &e) + { + metric.label[i].name = label_pairs[i].first; + } + metric.label[i].value = label_pairs[i].second; + } + } +}; + +/** + * Parse a string of labels (key:value) into a vector of pairs + * {,} + * {l1:v1,l2:v2,...,} + * + * @param a string of labels + * @return a vector of key value pairs + */ +std::vector> PrometheusExporterUtils::ParseLabel( + std::string labels) +{ + labels = labels.substr(1, labels.size() - 2); + + std::vector paired_labels; + std::stringstream s_stream(labels); + while (s_stream.good()) + { + std::string substr; + getline(s_stream, substr, ','); // get first string delimited by comma + if (!substr.empty()) + { + paired_labels.push_back(substr); + } + } + + std::vector> result; + for (auto &paired : paired_labels) + { + std::size_t split_index = paired.find(':'); + std::string label = paired.substr(0, split_index); + std::string value = paired.substr(split_index + 1); + result.emplace_back(std::pair(label, value)); + } + + return result; +} + +/** + * Build a quantiles vector from aggregator + */ +template +std::vector PrometheusExporterUtils::GetQuantilesVector( + nostd::shared_ptr> aggregator) +{ + std::vector quantiles; + + for (double q = 0; q <= 1; q += QUANTILE_STEP) + { + T quantile = aggregator->get_quantiles(q); + quantiles.emplace_back(quantile); + } + + return quantiles; +} + +/** + * Handle Counter and Gauge. + */ +template +void PrometheusExporterUtils::SetValue(std::vector values, + prometheus_client::MetricType type, + prometheus_client::ClientMetric *metric) +{ + switch (type) + { + case prometheus_client::MetricType::Counter: { + metric->counter.value = values[0]; + break; + } + case prometheus_client::MetricType::Gauge: { + metric->gauge.value = values[0]; + break; + } + case prometheus_client::MetricType::Untyped: { + metric->untyped.value = values[0]; + break; + } + default: + return; + } +} + +/** + * Handle Gauge from MinMaxSumCount + */ +void PrometheusExporterUtils::SetValue(double value, prometheus_client::ClientMetric *metric) +{ + metric->gauge.value = value; +} + +/** + * Handle Histogram + */ +template +void PrometheusExporterUtils::SetValue(std::vector values, + std::vector boundaries, + std::vector counts, + prometheus_client::ClientMetric *metric) +{ + metric->histogram.sample_sum = values[0]; + metric->histogram.sample_count = values[1]; + + int cumulative = 0; + std::vector buckets; + for (int i = 0; i < boundaries.size() + 1; i++) + { + prometheus_client::ClientMetric::Bucket bucket; + cumulative += counts[i]; + bucket.cumulative_count = cumulative; + if (i != boundaries.size()) + { + bucket.upper_bound = boundaries[i]; + } + else + { + bucket.upper_bound = std::numeric_limits::infinity(); + } + + buckets.emplace_back(bucket); + } + + metric->histogram.bucket = buckets; +} + +/** + * Handle Exact + */ +template +void PrometheusExporterUtils::SetValue(std::vector values, + std::vector quantiles, + prometheus_client::ClientMetric *metric) +{ + metric->summary.sample_count = values.size(); + + auto sum = 0; + for (auto val : values) + { + sum += val; + } + metric->summary.sample_sum = sum; + + double quant = 0; + std::vector prometheus_quantiles; + for (int i = 0; i < quantiles.size(); i++) + { + prometheus_client::ClientMetric::Quantile quantile; + quantile.quantile = quant; + quantile.value = quantiles[i]; + quant += QUANTILE_STEP; + prometheus_quantiles.emplace_back(quantile); + } + + metric->summary.quantile = prometheus_quantiles; +} + +} // namespace prometheus +} // namespace exporter +OPENTELEMETRY_END_NAMESPACE \ No newline at end of file From 21fe6daee7ac86a1924d94857559c1cf413c5f89 Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Thu, 6 Aug 2020 13:39:27 -0400 Subject: [PATCH 2/8] parse Sketch to Prometheus Summary --- .../src/prometheus_exporter_utils.cc | 71 ++++++++----------- 1 file changed, 30 insertions(+), 41 deletions(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index 56059148ab..782db5b2f1 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -44,12 +44,8 @@ std::vector PrometheusExporterUtils::TranslateT } // ======================= private helper functions ========================= - /** * Set value to metric family according to record - * - * @param record - * @param metric_family */ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, prometheus_client::MetricFamily *metric_family) @@ -62,7 +58,6 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, { metric_family->name = record.GetName(); } - metric_family->help = record.GetDescription(); // unpack the variant and set the metric data to metric family struct @@ -98,16 +93,12 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, * This function is needed because names in OpenTelemetry can contain * alphanumeric characters, '_', '.', and '-', whereas in Prometheus the * name should only contain alphanumeric characters and '_'. - * - * @param key name in OpenTelemetry - * @return sanitized name in Prometheus */ std::string PrometheusExporterUtils::SanitizeNames(std::string name) { // name cannot be null or empty. // a valid OTel name should only contain alphanumeric characters, // '_', '.', or '-'. - // meaningful exception message if (!IsValidName(name)) { throw std::invalid_argument("Received an invalid OTel name.\n"); @@ -134,9 +125,6 @@ std::string PrometheusExporterUtils::SanitizeNames(std::string name) * 2. They are case-insensitive * 3. The first character must be non-numeric, non-space, non-punctuation * 4. Subsequent characters must belong to the alphanumeric characters, '_', '.', and '-'. - * - * @param name name to be determined - * @return a boolean flag */ bool PrometheusExporterUtils::IsValidName(const std::string &name) { @@ -161,6 +149,9 @@ bool PrometheusExporterUtils::IsValidName(const std::string &name) return true; } +/** + * Set value to metric family for different aggregator + */ template void PrometheusExporterUtils::SetMetricFamilyByAggregator( nostd::shared_ptr> aggregator, @@ -171,7 +162,6 @@ void PrometheusExporterUtils::SetMetricFamilyByAggregator( auto kind = aggregator->get_aggregator_kind(); const prometheus_client::MetricType type = TranslateType(kind); metric_family->type = type; - // get check-pointed values, label string and check-pointed time auto checkpointed_values = aggregator->get_checkpoint(); auto time = aggregator->get_checkpoint_timestamp().time_since_epoch(); @@ -184,18 +174,20 @@ void PrometheusExporterUtils::SetMetricFamilyByAggregator( } else if (type == prometheus_client::MetricType::Summary) // Sketch, Exact { + auto quantiles = GetQuantilesVector(aggregator); if (kind == metric_sdk::AggregatorKind::Exact) { // TODO: 1. what if this Exact aggregator is not for quantile estimation? // TODO: 2. how many quantile samples should I include in each metric? if (aggregator->get_quant_estimation()) { - auto quantiles = GetQuantilesVector(aggregator); - SetData(checkpointed_values, quantiles, labels_str, time, metric_family); + SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family); } } else if (kind == metric_sdk::AggregatorKind::Sketch) - {} + { + SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family); + } } else // Counter, Gauge, MinMaxSumCount, Untyped { @@ -203,9 +195,7 @@ void PrometheusExporterUtils::SetMetricFamilyByAggregator( // Use sum/count is ok. if (kind == metric_sdk::AggregatorKind::MinMaxSumCount) { - int sum = checkpointed_values[2]; - int count = checkpointed_values[3]; - double avg = (double)sum / count; + double avg = (double)checkpointed_values[2] / checkpointed_values[3]; SetData(avg, labels_str, time, metric_family); } else @@ -217,9 +207,6 @@ void PrometheusExporterUtils::SetMetricFamilyByAggregator( /** * Translate the OTel metric type to Prometheus metric type - * - * @param record - * @return */ prometheus_client::MetricType PrometheusExporterUtils::TranslateType( metric_sdk::AggregatorKind kind) @@ -299,6 +286,7 @@ void PrometheusExporterUtils::SetData(double value, */ template void PrometheusExporterUtils::SetData(std::vector values, + metric_sdk::AggregatorKind kind, const std::vector &quantiles, const std::string &labels, std::chrono::nanoseconds time, @@ -307,9 +295,12 @@ void PrometheusExporterUtils::SetData(std::vector values, metric_family->metric.emplace_back(); prometheus_client::ClientMetric &metric = metric_family->metric.back(); SetMetricBasic(metric, time, labels); - SetValue(values, quantiles, &metric); + SetValue(values, kind, quantiles, &metric); } +/** + * Set time and labels to metric data + */ void PrometheusExporterUtils::SetMetricBasic(prometheus_client::ClientMetric &metric, std::chrono::nanoseconds time, const std::string &labels) @@ -339,9 +330,6 @@ void PrometheusExporterUtils::SetMetricBasic(prometheus_client::ClientMetric &me * Parse a string of labels (key:value) into a vector of pairs * {,} * {l1:v1,l2:v2,...,} - * - * @param a string of labels - * @return a vector of key value pairs */ std::vector> PrometheusExporterUtils::ParseLabel( std::string labels) @@ -380,13 +368,11 @@ std::vector PrometheusExporterUtils::GetQuantilesVector( nostd::shared_ptr> aggregator) { std::vector quantiles; - for (double q = 0; q <= 1; q += QUANTILE_STEP) { T quantile = aggregator->get_quantiles(q); quantiles.emplace_back(quantile); } - return quantiles; } @@ -436,8 +422,7 @@ void PrometheusExporterUtils::SetValue(std::vector values, { metric->histogram.sample_sum = values[0]; metric->histogram.sample_count = values[1]; - - int cumulative = 0; + int cumulative = 0; std::vector buckets; for (int i = 0; i < boundaries.size() + 1; i++) { @@ -452,29 +437,35 @@ void PrometheusExporterUtils::SetValue(std::vector values, { bucket.upper_bound = std::numeric_limits::infinity(); } - buckets.emplace_back(bucket); } - metric->histogram.bucket = buckets; } /** - * Handle Exact + * Handle Exact and Sketch */ template void PrometheusExporterUtils::SetValue(std::vector values, + metric_sdk::AggregatorKind kind, std::vector quantiles, prometheus_client::ClientMetric *metric) { - metric->summary.sample_count = values.size(); - - auto sum = 0; - for (auto val : values) + if (kind == metric_sdk::AggregatorKind::Exact) { - sum += val; + metric->summary.sample_count = values.size(); + auto sum = 0; + for (auto val : values) + { + sum += val; + } + metric->summary.sample_sum = sum; + } + else if (kind == metric_sdk::AggregatorKind::Sketch) + { + metric->summary.sample_sum = values[0]; + metric->summary.sample_count = values[1]; } - metric->summary.sample_sum = sum; double quant = 0; std::vector prometheus_quantiles; @@ -486,10 +477,8 @@ void PrometheusExporterUtils::SetValue(std::vector values, quant += QUANTILE_STEP; prometheus_quantiles.emplace_back(quantile); } - metric->summary.quantile = prometheus_quantiles; } - } // namespace prometheus } // namespace exporter OPENTELEMETRY_END_NAMESPACE \ No newline at end of file From 7e16ed019702a0242722c951e49edfdee6ea684d Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Thu, 6 Aug 2020 15:04:55 -0400 Subject: [PATCH 3/8] modify comments --- exporters/prometheus/src/prometheus_exporter_utils.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index 782db5b2f1..5720e1a70c 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -74,13 +74,13 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); } else if (nostd::holds_alternative>>( - record.GetAggregator())) + record.GetAggregator())) { auto aggregator = nostd::get>>(agg_var); SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); } else if (nostd::holds_alternative>>( - record.GetAggregator())) + record.GetAggregator())) { auto aggregator = nostd::get>>(agg_var); SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); @@ -283,6 +283,7 @@ void PrometheusExporterUtils::SetData(double value, /** * Set metric data for: * Exact => Prometheus Summary + * Sketch => Prometheus Summary */ template void PrometheusExporterUtils::SetData(std::vector values, From f8eb1826414a6e9321b5fcca956e533d4fbe349c Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Fri, 7 Aug 2020 11:03:11 -0400 Subject: [PATCH 4/8] add copyright headers and add a new line at the end of file --- .../src/prometheus_exporter_utils.cc | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index 5720e1a70c..e3d53ceb58 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -1,3 +1,19 @@ +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + #include #include #include @@ -482,4 +498,4 @@ void PrometheusExporterUtils::SetValue(std::vector values, } } // namespace prometheus } // namespace exporter -OPENTELEMETRY_END_NAMESPACE \ No newline at end of file +OPENTELEMETRY_END_NAMESPACE From 95e7617b36b1d5c372dba33ec5d9184d9a6a9aaa Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Fri, 7 Aug 2020 13:24:58 -0400 Subject: [PATCH 5/8] run format script --- .../prometheus/src/prometheus_exporter_utils.cc | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index e3d53ceb58..7a71f7c0f4 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -90,13 +90,13 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); } else if (nostd::holds_alternative>>( - record.GetAggregator())) + record.GetAggregator())) { auto aggregator = nostd::get>>(agg_var); SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); } else if (nostd::holds_alternative>>( - record.GetAggregator())) + record.GetAggregator())) { auto aggregator = nostd::get>>(agg_var); SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); @@ -403,15 +403,18 @@ void PrometheusExporterUtils::SetValue(std::vector values, { switch (type) { - case prometheus_client::MetricType::Counter: { + case prometheus_client::MetricType::Counter: + { metric->counter.value = values[0]; break; } - case prometheus_client::MetricType::Gauge: { + case prometheus_client::MetricType::Gauge: + { metric->gauge.value = values[0]; break; } - case prometheus_client::MetricType::Untyped: { + case prometheus_client::MetricType::Untyped: + { metric->untyped.value = values[0]; break; } From bab51e870e71c0aa26d7e30fcda2bb9bbc1568e4 Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Fri, 7 Aug 2020 14:34:51 -0400 Subject: [PATCH 6/8] logging when sanitizing metric and label names --- exporters/prometheus/src/prometheus_exporter_utils.cc | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index 7a71f7c0f4..156bc79ac7 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -68,7 +68,10 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, { try { - metric_family->name = SanitizeNames(record.GetName()); + auto sanitized = SanitizeNames(record.GetName()); + metric_family->name = sanitized; + std::cout << "Sanitized metric name \"" << record.GetName() << "\" to \"" << sanitized << "\"" + << std::endl; } catch (std::invalid_argument &e) { @@ -332,7 +335,10 @@ void PrometheusExporterUtils::SetMetricBasic(prometheus_client::ClientMetric &me { try { - metric.label[i].name = SanitizeNames(label_pairs[i].first); + auto sanitized = SanitizeNames(label_pairs[i].first); + metric.label[i].name = sanitized; + std::cout << "Sanitized label name \"" << label_pairs[i].first << "\" to \"" << sanitized + << "\"" << std::endl; } catch (std::invalid_argument &e) { From 33d650216b40d8c0ae2a1e1d598bf88798117877 Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Mon, 10 Aug 2020 11:47:25 -0400 Subject: [PATCH 7/8] Parse Exact Aggregator to a summary without quantiles when it cannot collect quantile data --- .../src/prometheus_exporter_utils.cc | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index 156bc79ac7..fa7560ef28 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -93,13 +93,13 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); } else if (nostd::holds_alternative>>( - record.GetAggregator())) + record.GetAggregator())) { auto aggregator = nostd::get>>(agg_var); SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); } else if (nostd::holds_alternative>>( - record.GetAggregator())) + record.GetAggregator())) { auto aggregator = nostd::get>>(agg_var); SetMetricFamilyByAggregator(aggregator, labels_str, metric_family); @@ -193,19 +193,20 @@ void PrometheusExporterUtils::SetMetricFamilyByAggregator( } else if (type == prometheus_client::MetricType::Summary) // Sketch, Exact { - auto quantiles = GetQuantilesVector(aggregator); if (kind == metric_sdk::AggregatorKind::Exact) { - // TODO: 1. what if this Exact aggregator is not for quantile estimation? - // TODO: 2. how many quantile samples should I include in each metric? - if (aggregator->get_quant_estimation()) + std::vector quantiles; + bool do_quantile = aggregator->get_quant_estimation(); + if (do_quantile) { - SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family); + quantiles = GetQuantilesVector(aggregator); } + SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family, do_quantile); } else if (kind == metric_sdk::AggregatorKind::Sketch) { - SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family); + auto quantiles = GetQuantilesVector(aggregator); + SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family, true); } } else // Counter, Gauge, MinMaxSumCount, Untyped @@ -310,12 +311,13 @@ void PrometheusExporterUtils::SetData(std::vector values, const std::vector &quantiles, const std::string &labels, std::chrono::nanoseconds time, - prometheus_client::MetricFamily *metric_family) + prometheus_client::MetricFamily *metric_family, + bool do_quantile) { metric_family->metric.emplace_back(); prometheus_client::ClientMetric &metric = metric_family->metric.back(); SetMetricBasic(metric, time, labels); - SetValue(values, kind, quantiles, &metric); + SetValue(values, kind, quantiles, &metric, do_quantile); } /** @@ -409,18 +411,15 @@ void PrometheusExporterUtils::SetValue(std::vector values, { switch (type) { - case prometheus_client::MetricType::Counter: - { + case prometheus_client::MetricType::Counter: { metric->counter.value = values[0]; break; } - case prometheus_client::MetricType::Gauge: - { + case prometheus_client::MetricType::Gauge: { metric->gauge.value = values[0]; break; } - case prometheus_client::MetricType::Untyped: - { + case prometheus_client::MetricType::Untyped: { metric->untyped.value = values[0]; break; } @@ -475,7 +474,8 @@ template void PrometheusExporterUtils::SetValue(std::vector values, metric_sdk::AggregatorKind kind, std::vector quantiles, - prometheus_client::ClientMetric *metric) + prometheus_client::ClientMetric *metric, + bool do_quantile) { if (kind == metric_sdk::AggregatorKind::Exact) { @@ -493,17 +493,20 @@ void PrometheusExporterUtils::SetValue(std::vector values, metric->summary.sample_count = values[1]; } - double quant = 0; - std::vector prometheus_quantiles; - for (int i = 0; i < quantiles.size(); i++) + if (do_quantile) { - prometheus_client::ClientMetric::Quantile quantile; - quantile.quantile = quant; - quantile.value = quantiles[i]; - quant += QUANTILE_STEP; - prometheus_quantiles.emplace_back(quantile); + double quant = 0; + std::vector prometheus_quantiles; + for (int i = 0; i < quantiles.size(); i++) + { + prometheus_client::ClientMetric::Quantile quantile; + quantile.quantile = quant; + quantile.value = quantiles[i]; + quant += QUANTILE_STEP; + prometheus_quantiles.emplace_back(quantile); + } + metric->summary.quantile = prometheus_quantiles; } - metric->summary.quantile = prometheus_quantiles; } } // namespace prometheus } // namespace exporter From 781b792264e6232c035dbd294aae08c47952f60a Mon Sep 17 00:00:00 2001 From: Cunjun Wang Date: Fri, 14 Aug 2020 18:38:45 -0400 Subject: [PATCH 8/8] print logs for sanitizing name only when the name is changed; change default quantile setup --- .../src/prometheus_exporter_utils.cc | 50 +++++++++++-------- 1 file changed, 30 insertions(+), 20 deletions(-) diff --git a/exporters/prometheus/src/prometheus_exporter_utils.cc b/exporters/prometheus/src/prometheus_exporter_utils.cc index fa7560ef28..4ac9c7dc8c 100644 --- a/exporters/prometheus/src/prometheus_exporter_utils.cc +++ b/exporters/prometheus/src/prometheus_exporter_utils.cc @@ -23,8 +23,6 @@ #include "opentelemetry/sdk/metrics/aggregator/aggregator.h" #include "prometheus/metric_type.h" -#define QUANTILE_STEP 0.25 - OPENTELEMETRY_BEGIN_NAMESPACE namespace exporter { @@ -68,10 +66,14 @@ void PrometheusExporterUtils::SetMetricFamily(metric_sdk::Record &record, { try { - auto sanitized = SanitizeNames(record.GetName()); + auto origin_name = record.GetName(); + auto sanitized = SanitizeNames(origin_name); + if (origin_name != sanitized) + { + std::cout << "Sanitized metric name \"" << origin_name << "\" to \"" << sanitized << "\"" + << std::endl; + } metric_family->name = sanitized; - std::cout << "Sanitized metric name \"" << record.GetName() << "\" to \"" << sanitized << "\"" - << std::endl; } catch (std::invalid_argument &e) { @@ -193,20 +195,23 @@ void PrometheusExporterUtils::SetMetricFamilyByAggregator( } else if (type == prometheus_client::MetricType::Summary) // Sketch, Exact { + std::vector quantile_points = {0, 0.5, 0.9, 0.95, 0.99, 1}; if (kind == metric_sdk::AggregatorKind::Exact) { std::vector quantiles; bool do_quantile = aggregator->get_quant_estimation(); if (do_quantile) { - quantiles = GetQuantilesVector(aggregator); + quantiles = GetQuantilesVector(aggregator, quantile_points); } - SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family, do_quantile); + SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family, do_quantile, + quantile_points); } else if (kind == metric_sdk::AggregatorKind::Sketch) { - auto quantiles = GetQuantilesVector(aggregator); - SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family, true); + auto quantiles = GetQuantilesVector(aggregator, quantile_points); + SetData(checkpointed_values, kind, quantiles, labels_str, time, metric_family, true, + quantile_points); } } else // Counter, Gauge, MinMaxSumCount, Untyped @@ -312,12 +317,13 @@ void PrometheusExporterUtils::SetData(std::vector values, const std::string &labels, std::chrono::nanoseconds time, prometheus_client::MetricFamily *metric_family, - bool do_quantile) + bool do_quantile, + std::vector quantile_points) { metric_family->metric.emplace_back(); prometheus_client::ClientMetric &metric = metric_family->metric.back(); SetMetricBasic(metric, time, labels); - SetValue(values, kind, quantiles, &metric, do_quantile); + SetValue(values, kind, quantiles, &metric, do_quantile, quantile_points); } /** @@ -337,10 +343,14 @@ void PrometheusExporterUtils::SetMetricBasic(prometheus_client::ClientMetric &me { try { - auto sanitized = SanitizeNames(label_pairs[i].first); + auto origin_name = label_pairs[i].first; + auto sanitized = SanitizeNames(origin_name); + if (origin_name != sanitized) + { + std::cout << "Sanitized label name \"" << origin_name << "\" to \"" << sanitized << "\"" + << std::endl; + } metric.label[i].name = sanitized; - std::cout << "Sanitized label name \"" << label_pairs[i].first << "\" to \"" << sanitized - << "\"" << std::endl; } catch (std::invalid_argument &e) { @@ -390,10 +400,11 @@ std::vector> PrometheusExporterUtils::ParseL */ template std::vector PrometheusExporterUtils::GetQuantilesVector( - nostd::shared_ptr> aggregator) + nostd::shared_ptr> aggregator, + const std::vector &quantile_points) { std::vector quantiles; - for (double q = 0; q <= 1; q += QUANTILE_STEP) + for (double q : quantile_points) { T quantile = aggregator->get_quantiles(q); quantiles.emplace_back(quantile); @@ -475,7 +486,8 @@ void PrometheusExporterUtils::SetValue(std::vector values, metric_sdk::AggregatorKind kind, std::vector quantiles, prometheus_client::ClientMetric *metric, - bool do_quantile) + bool do_quantile, + const std::vector &quantile_points) { if (kind == metric_sdk::AggregatorKind::Exact) { @@ -495,14 +507,12 @@ void PrometheusExporterUtils::SetValue(std::vector values, if (do_quantile) { - double quant = 0; std::vector prometheus_quantiles; for (int i = 0; i < quantiles.size(); i++) { prometheus_client::ClientMetric::Quantile quantile; - quantile.quantile = quant; + quantile.quantile = quantile_points[i]; quantile.value = quantiles[i]; - quant += QUANTILE_STEP; prometheus_quantiles.emplace_back(quantile); } metric->summary.quantile = prometheus_quantiles;