From bc1716f2794a652aa5c02bd0755927632a5e0988 Mon Sep 17 00:00:00 2001 From: SystemsPurge Date: Wed, 16 Apr 2025 12:41:34 +0200 Subject: [PATCH 1/7] Add metrics path API handler Signed-off-by: SystemsPurge --- lib/api/CMakeLists.txt | 1 + lib/api/requests/metrics.cpp | 74 ++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 lib/api/requests/metrics.cpp diff --git a/lib/api/CMakeLists.txt b/lib/api/CMakeLists.txt index b52e3eee1..70974bdeb 100644 --- a/lib/api/CMakeLists.txt +++ b/lib/api/CMakeLists.txt @@ -26,6 +26,7 @@ set(API_SRC requests/node_stats.cpp requests/node_stats_reset.cpp requests/node_file.cpp + requests/metrics.cpp requests/paths.cpp requests/path_info.cpp requests/path_action.cpp diff --git a/lib/api/requests/metrics.cpp b/lib/api/requests/metrics.cpp new file mode 100644 index 000000000..b44994639 --- /dev/null +++ b/lib/api/requests/metrics.cpp @@ -0,0 +1,74 @@ +/* The "nodes" API ressource. + * + * Author: Steffen Vogel + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace villas { +namespace node { +namespace api { + +class MetricsRequest : public Request { +private: +std::unordered_map metrics_subset = { + {Stats::Metric::SMPS_SKIPPED,"skipped"}, + {Stats::Metric::OWD,"owd"}, + {Stats::Metric::AGE,"age"}, + {Stats::Metric::SIGNAL_COUNT,"signalcnt"}, + {Stats::Metric::RTP_PKTS_LOST,"rtp_pkts_lost"} +}; + +public: + using Request::Request; + + virtual Response *execute() { + if (method != Session::Method::GET) + throw InvalidMethod(this); + + if (body != nullptr) + throw BadRequest("Nodes endpoint does not accept any body data"); + + + std::string text_res = ""; + NodeList node_list = session->getSuperNode()->getNodes(); + for(Node* node: node_list){ + auto stats = node->getStats(); + if(!stats) + continue; + std::string node_name = node->getNameShort(); + for(auto& metric:metrics_subset){ + Hist histogram = stats->getHistogram(metric.first); + std::string t =std::to_string( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + ).count() + ); + text_res+=metric.second+" {node=\""+node_name+"\" acc=\"last\"} "+std::to_string(histogram.getLast())+" "+t+"\n"; + text_res+=metric.second+" {node=\""+node_name+"\" acc=\"total\"} "+std::to_string(histogram.getTotal())+" "+t+"\n"; + } + } + + return new Response(session,HTTP_STATUS_OK, "text/plain; charset=UTF-8", Buffer(text_res.c_str(),text_res.size())); + } +}; + +// Register API request +static char n[] = "metrics"; +static char r[] = "/metrics"; +static char d[] = "Get stats of all nodes in desired format"; +static RequestPlugin p; + +} // namespace api +} // namespace node +} // namespace villas From a33221a12ee9b8d71c4f0d0861f1e9282240b698 Mon Sep 17 00:00:00 2001 From: SystemsPurge Date: Fri, 25 Apr 2025 13:17:33 +0200 Subject: [PATCH 2/7] Change metrics endpoint to export histograms of all metrics Signed-off-by: SystemsPurge --- common/include/villas/hist.hpp | 5 ++++- common/lib/hist.cpp | 17 +++++++++++++++++ lib/api/requests/metrics.cpp | 16 +++++----------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/common/include/villas/hist.hpp b/common/include/villas/hist.hpp index 3c7acb33e..ba664a044 100644 --- a/common/include/villas/hist.hpp +++ b/common/include/villas/hist.hpp @@ -8,7 +8,7 @@ #pragma once #include - +#include #include #include @@ -49,6 +49,7 @@ class Hist { // Print ASCII style plot of histogram. void plot(Logger logger) const; + void test(); // Dump histogram data in Matlab format. // // @return The string containing the dump. The caller is responsible to free() the buffer. @@ -60,6 +61,8 @@ class Hist { // Write the histogram in JSON format to file \p f. int dumpJson(FILE *f) const; + std::string promFormat(std::string metric_name, std::string node_name) const; + // Build a libjansson / JSON object of the histogram. json_t *toJson() const; diff --git a/common/lib/hist.cpp b/common/lib/hist.cpp index d2ad73df9..73a613776 100644 --- a/common/lib/hist.cpp +++ b/common/lib/hist.cpp @@ -150,6 +150,23 @@ void Hist::plot(Logger logger) const { } } +std::string Hist::promFormat(std::string metric_name, std::string node_name) const{ + std::string base = "#TYPE HISTOGRAM "+metric_name; + //need this because prometheus understands quantiles + int s = data.size(); + for(int i=0;i metrics_subset = { if (body != nullptr) throw BadRequest("Nodes endpoint does not accept any body data"); - - + std::string text_res = ""; NodeList node_list = session->getSuperNode()->getNodes(); for(Node* node: node_list){ @@ -47,15 +46,10 @@ std::unordered_map metrics_subset = { if(!stats) continue; std::string node_name = node->getNameShort(); - for(auto& metric:metrics_subset){ - Hist histogram = stats->getHistogram(metric.first); - std::string t =std::to_string( - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch() - ).count() - ); - text_res+=metric.second+" {node=\""+node_name+"\" acc=\"last\"} "+std::to_string(histogram.getLast())+" "+t+"\n"; - text_res+=metric.second+" {node=\""+node_name+"\" acc=\"total\"} "+std::to_string(histogram.getTotal())+" "+t+"\n"; + for(auto& metric:Stats::metrics){ + std::string metric_name = metric.second.name; + std::replace(metric_name.begin(),metric_name.end(),'.','_'); + text_res+= stats->getHistogram(metric.first).promFormat(metric_name,node->getNameShort())+"\n"; } } From 328f9fa8528aec5a981763c56d8b15f7151a341c Mon Sep 17 00:00:00 2001 From: SystemsPurge Date: Fri, 25 Apr 2025 13:28:02 +0200 Subject: [PATCH 3/7] fix: export histogram prometheus format with cumulative sum rather than bucket values Signed-off-by: SystemsPurge --- common/lib/hist.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/lib/hist.cpp b/common/lib/hist.cpp index 73a613776..fd2b6fd12 100644 --- a/common/lib/hist.cpp +++ b/common/lib/hist.cpp @@ -154,8 +154,10 @@ std::string Hist::promFormat(std::string metric_name, std::string node_name) con std::string base = "#TYPE HISTOGRAM "+metric_name; //need this because prometheus understands quantiles int s = data.size(); + Hist::cnt_t cumsum = 0; for(int i=0;i Date: Mon, 16 Jun 2025 11:29:22 +0200 Subject: [PATCH 4/7] fix: Formatting, Add author, Use stringstream, proper comments and function name and remove useless code. Signed-off-by: SystemsPurge --- common/include/villas/hist.hpp | 9 ++++---- common/lib/hist.cpp | 26 ++++++++++++--------- lib/api/requests/metrics.cpp | 41 ++++++++++++++++------------------ 3 files changed, 40 insertions(+), 36 deletions(-) diff --git a/common/include/villas/hist.hpp b/common/include/villas/hist.hpp index ba664a044..5081356c5 100644 --- a/common/include/villas/hist.hpp +++ b/common/include/villas/hist.hpp @@ -7,8 +7,9 @@ #pragma once +#include #include -#include + #include #include @@ -49,7 +50,6 @@ class Hist { // Print ASCII style plot of histogram. void plot(Logger logger) const; - void test(); // Dump histogram data in Matlab format. // // @return The string containing the dump. The caller is responsible to free() the buffer. @@ -61,8 +61,9 @@ class Hist { // Write the histogram in JSON format to file \p f. int dumpJson(FILE *f) const; - std::string promFormat(std::string metric_name, std::string node_name) const; - + std::string toPrometheusText(std::string metric_name, + std::string node_name) const; + // Build a libjansson / JSON object of the histogram. json_t *toJson() const; diff --git a/common/lib/hist.cpp b/common/lib/hist.cpp index fd2b6fd12..969dbe62f 100644 --- a/common/lib/hist.cpp +++ b/common/lib/hist.cpp @@ -150,23 +150,29 @@ void Hist::plot(Logger logger) const { } } -std::string Hist::promFormat(std::string metric_name, std::string node_name) const{ - std::string base = "#TYPE HISTOGRAM "+metric_name; - //need this because prometheus understands quantiles +std::string Hist::toPrometheusText(std::string metric_name, + std::string node_name) const { + std::stringstream base; + base << "#TYPE HISTOGRAM " << metric_name; + // Needed because Prometheus understands quantiles. int s = data.size(); Hist::cnt_t cumsum = 0; - for(int i=0;i + * Author: Youssef Nakti * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University * SPDX-License-Identifier: Apache-2.0 */ +#include + #include -#include + #include #include #include @@ -20,15 +23,6 @@ namespace node { namespace api { class MetricsRequest : public Request { -private: -std::unordered_map metrics_subset = { - {Stats::Metric::SMPS_SKIPPED,"skipped"}, - {Stats::Metric::OWD,"owd"}, - {Stats::Metric::AGE,"age"}, - {Stats::Metric::SIGNAL_COUNT,"signalcnt"}, - {Stats::Metric::RTP_PKTS_LOST,"rtp_pkts_lost"} -}; - public: using Request::Request; @@ -37,30 +31,33 @@ std::unordered_map metrics_subset = { throw InvalidMethod(this); if (body != nullptr) - throw BadRequest("Nodes endpoint does not accept any body data"); - - std::string text_res = ""; + throw BadRequest("The metrics endpoint does not accept any body data"); + + std::stringstream res_stream; NodeList node_list = session->getSuperNode()->getNodes(); - for(Node* node: node_list){ + for (Node *node : node_list) { auto stats = node->getStats(); - if(!stats) + if (!stats) continue; std::string node_name = node->getNameShort(); - for(auto& metric:Stats::metrics){ + for (auto &metric : Stats::metrics) { std::string metric_name = metric.second.name; - std::replace(metric_name.begin(),metric_name.end(),'.','_'); - text_res+= stats->getHistogram(metric.first).promFormat(metric_name,node->getNameShort())+"\n"; + std::replace(metric_name.begin(), metric_name.end(), '.', '_'); + res_stream << stats->getHistogram(metric.first) + .toPrometheusText(metric_name, node->getNameShort()) + << "\n"; } } - - return new Response(session,HTTP_STATUS_OK, "text/plain; charset=UTF-8", Buffer(text_res.c_str(),text_res.size())); + auto text_res = res_stream.str(); + return new Response(session, HTTP_STATUS_OK, "text/plain; charset=UTF-8", + Buffer(text_res.c_str(), text_res.size())); } }; // Register API request static char n[] = "metrics"; static char r[] = "/metrics"; -static char d[] = "Get stats of all nodes in desired format"; +static char d[] = "Get stats of all nodes in Prometheus metrics format"; static RequestPlugin p; } // namespace api From 48b1175be2b558000b8fe723cfac37042f1034ff Mon Sep 17 00:00:00 2001 From: SystemsPurge Date: Wed, 16 Apr 2025 12:41:34 +0200 Subject: [PATCH 5/7] Add metrics path API handler Signed-off-by: SystemsPurge --- lib/api/CMakeLists.txt | 1 + lib/api/requests/metrics.cpp | 74 ++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 lib/api/requests/metrics.cpp diff --git a/lib/api/CMakeLists.txt b/lib/api/CMakeLists.txt index b52e3eee1..70974bdeb 100644 --- a/lib/api/CMakeLists.txt +++ b/lib/api/CMakeLists.txt @@ -26,6 +26,7 @@ set(API_SRC requests/node_stats.cpp requests/node_stats_reset.cpp requests/node_file.cpp + requests/metrics.cpp requests/paths.cpp requests/path_info.cpp requests/path_action.cpp diff --git a/lib/api/requests/metrics.cpp b/lib/api/requests/metrics.cpp new file mode 100644 index 000000000..b44994639 --- /dev/null +++ b/lib/api/requests/metrics.cpp @@ -0,0 +1,74 @@ +/* The "nodes" API ressource. + * + * Author: Steffen Vogel + * SPDX-FileCopyrightText: 2014-2023 Institute for Automation of Complex Power Systems, RWTH Aachen University + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace villas { +namespace node { +namespace api { + +class MetricsRequest : public Request { +private: +std::unordered_map metrics_subset = { + {Stats::Metric::SMPS_SKIPPED,"skipped"}, + {Stats::Metric::OWD,"owd"}, + {Stats::Metric::AGE,"age"}, + {Stats::Metric::SIGNAL_COUNT,"signalcnt"}, + {Stats::Metric::RTP_PKTS_LOST,"rtp_pkts_lost"} +}; + +public: + using Request::Request; + + virtual Response *execute() { + if (method != Session::Method::GET) + throw InvalidMethod(this); + + if (body != nullptr) + throw BadRequest("Nodes endpoint does not accept any body data"); + + + std::string text_res = ""; + NodeList node_list = session->getSuperNode()->getNodes(); + for(Node* node: node_list){ + auto stats = node->getStats(); + if(!stats) + continue; + std::string node_name = node->getNameShort(); + for(auto& metric:metrics_subset){ + Hist histogram = stats->getHistogram(metric.first); + std::string t =std::to_string( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch() + ).count() + ); + text_res+=metric.second+" {node=\""+node_name+"\" acc=\"last\"} "+std::to_string(histogram.getLast())+" "+t+"\n"; + text_res+=metric.second+" {node=\""+node_name+"\" acc=\"total\"} "+std::to_string(histogram.getTotal())+" "+t+"\n"; + } + } + + return new Response(session,HTTP_STATUS_OK, "text/plain; charset=UTF-8", Buffer(text_res.c_str(),text_res.size())); + } +}; + +// Register API request +static char n[] = "metrics"; +static char r[] = "/metrics"; +static char d[] = "Get stats of all nodes in desired format"; +static RequestPlugin p; + +} // namespace api +} // namespace node +} // namespace villas From a0c71ea27b8fb4060192615a3b7e2cf44fab6bd6 Mon Sep 17 00:00:00 2001 From: SystemsPurge Date: Fri, 25 Apr 2025 13:17:33 +0200 Subject: [PATCH 6/7] Change metrics endpoint to export histograms of all metrics Signed-off-by: SystemsPurge --- common/include/villas/hist.hpp | 5 ++++- common/lib/hist.cpp | 17 +++++++++++++++++ lib/api/requests/metrics.cpp | 16 +++++----------- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/common/include/villas/hist.hpp b/common/include/villas/hist.hpp index 3c7acb33e..ba664a044 100644 --- a/common/include/villas/hist.hpp +++ b/common/include/villas/hist.hpp @@ -8,7 +8,7 @@ #pragma once #include - +#include #include #include @@ -49,6 +49,7 @@ class Hist { // Print ASCII style plot of histogram. void plot(Logger logger) const; + void test(); // Dump histogram data in Matlab format. // // @return The string containing the dump. The caller is responsible to free() the buffer. @@ -60,6 +61,8 @@ class Hist { // Write the histogram in JSON format to file \p f. int dumpJson(FILE *f) const; + std::string promFormat(std::string metric_name, std::string node_name) const; + // Build a libjansson / JSON object of the histogram. json_t *toJson() const; diff --git a/common/lib/hist.cpp b/common/lib/hist.cpp index d2ad73df9..73a613776 100644 --- a/common/lib/hist.cpp +++ b/common/lib/hist.cpp @@ -150,6 +150,23 @@ void Hist::plot(Logger logger) const { } } +std::string Hist::promFormat(std::string metric_name, std::string node_name) const{ + std::string base = "#TYPE HISTOGRAM "+metric_name; + //need this because prometheus understands quantiles + int s = data.size(); + for(int i=0;i metrics_subset = { if (body != nullptr) throw BadRequest("Nodes endpoint does not accept any body data"); - - + std::string text_res = ""; NodeList node_list = session->getSuperNode()->getNodes(); for(Node* node: node_list){ @@ -47,15 +46,10 @@ std::unordered_map metrics_subset = { if(!stats) continue; std::string node_name = node->getNameShort(); - for(auto& metric:metrics_subset){ - Hist histogram = stats->getHistogram(metric.first); - std::string t =std::to_string( - std::chrono::duration_cast( - std::chrono::system_clock::now().time_since_epoch() - ).count() - ); - text_res+=metric.second+" {node=\""+node_name+"\" acc=\"last\"} "+std::to_string(histogram.getLast())+" "+t+"\n"; - text_res+=metric.second+" {node=\""+node_name+"\" acc=\"total\"} "+std::to_string(histogram.getTotal())+" "+t+"\n"; + for(auto& metric:Stats::metrics){ + std::string metric_name = metric.second.name; + std::replace(metric_name.begin(),metric_name.end(),'.','_'); + text_res+= stats->getHistogram(metric.first).promFormat(metric_name,node->getNameShort())+"\n"; } } From f4753f3828283153674b4988199eed08e3eda65b Mon Sep 17 00:00:00 2001 From: SystemsPurge Date: Fri, 25 Apr 2025 13:28:02 +0200 Subject: [PATCH 7/7] fix: export histogram prometheus format with cumulative sum rather than bucket values Signed-off-by: SystemsPurge --- common/lib/hist.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/lib/hist.cpp b/common/lib/hist.cpp index 73a613776..fd2b6fd12 100644 --- a/common/lib/hist.cpp +++ b/common/lib/hist.cpp @@ -154,8 +154,10 @@ std::string Hist::promFormat(std::string metric_name, std::string node_name) con std::string base = "#TYPE HISTOGRAM "+metric_name; //need this because prometheus understands quantiles int s = data.size(); + Hist::cnt_t cumsum = 0; for(int i=0;i