fix(metrics): group Prometheus output by family — fix duplicate HELP/TYPE (#716)

src/AbstractMetricsManager.h

@@ -19,6 +19,7 @@
 #endif
 #include "Configurable.h"
 #include "Metrics.h"
+#include "PrometheusSerializer.h"
 #include <bitset>
 #include <shared_mutex>
 #include <sstream>
@@ -217,7 +218,7 @@ class AbstractMetricsBucket
     }
 
     virtual void to_json(json &j) const = 0;
-    virtual void to_prometheus(std::stringstream &out, Metric::LabelMap add_labels = {}) const = 0;
+    virtual void to_prometheus(PrometheusSerializer &ser, Metric::LabelMap add_labels = {}) const = 0;
     virtual void to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &start_ts, timespec &end_ts, Metric::LabelMap add_labels = {}) const = 0;
     virtual void update_topn_metrics(size_t topn_count, uint64_t percentile_threshold) = 0;
 };
@@ -503,7 +504,7 @@ class AbstractMetricsManager
         _metric_buckets.at(period)->to_json(j[key]);
     }
 
-    void window_single_prometheus(std::stringstream &out, uint64_t period = 0, Metric::LabelMap add_labels = {}) const
+    void window_single_prometheus(PrometheusSerializer &ser, uint64_t period = 0, Metric::LabelMap add_labels = {}) const
     {
         std::shared_lock rl(_base_mutex);
         std::shared_lock rbl(_bucket_mutex);
@@ -527,7 +528,7 @@ class AbstractMetricsManager
             add_labels["tap"] = _tap_name;
         }
 
-        _metric_buckets.at(period)->to_prometheus(out, add_labels);
+        _metric_buckets.at(period)->to_prometheus(ser, add_labels);
     }
 
     void window_single_opentelemetry(metrics::v1::ScopeMetrics &scope, uint64_t period = 0, Metric::LabelMap add_labels = {}) const
@@ -577,13 +578,13 @@ class AbstractMetricsManager
         sbucket->to_opentelemetry(scope, start_ts, end_ts, add_labels);
     }
 
-    void window_external_prometheus(std::stringstream &out, AbstractMetricsBucket *bucket, Metric::LabelMap add_labels = {}) const
+    void window_external_prometheus(PrometheusSerializer &ser, AbstractMetricsBucket *bucket, Metric::LabelMap add_labels = {}) const
     {
         if (_groups && _groups->none()) {
             return;
         }
         // static because caller guarantees only our own bucket type
-        static_cast<MetricsBucketClass *>(bucket)->to_prometheus(out, add_labels);
+        static_cast<MetricsBucketClass *>(bucket)->to_prometheus(ser, add_labels);
     }
 
     void window_external_json(json &j, const std::string &key, AbstractMetricsBucket *bucket) const

src/CMakeLists.txt

@@ -87,6 +87,7 @@ add_executable(unit-tests-visor-core
         tests/test_policies.cpp
         tests/test_handlers.cpp
         tests/test_module_plugins.cpp
+        tests/test_prometheus_serializer.cpp
         )
 
 target_include_directories(unit-tests-visor-core PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})

src/CoreServer.cpp

@@ -6,6 +6,7 @@
 #include "HandlerManager.h"
 #include "Metrics.h"
 #include "Policies.h"
+#include "PrometheusSerializer.h"
 #include "Taps.h"
 #include "visor_config.h"
 #include <chrono>
@@ -173,15 +174,17 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
             }
             try {
                 std::stringstream output;
+                PrometheusSerializer ser;
                 auto [policy, lock] = _registry->policy_manager()->module_get_locked("default");
                 for (auto &mod : policy->modules()) {
                     auto hmod = dynamic_cast<StreamHandler *>(mod);
                     if (hmod) {
                         spdlog::stopwatch sw;
-                        hmod->window_prometheus(output, {{"policy", "default"}});
+                        hmod->window_prometheus(ser, {{"policy", "default"}});
                         _logger->debug("{} window_prometheus elapsed time: {}", hmod->name(), sw);
                     }
                 }
+                output << ser.finalize();
                 res.set_content(output.str(), "text/plain");
             } catch (const std::exception &e) {
                 res.status = 500;
@@ -431,16 +434,19 @@ void CoreServer::_setup_routes(const PrometheusConfig &prom_config)
             }
         }
         std::stringstream output;
-        for (const auto &p_mname : plist) {
-            try {
+        PrometheusSerializer ser;
+        try {
+            for (const auto &p_mname : plist) {
                 auto [policy, lock] = _registry->policy_manager()->module_get_locked(p_mname);
-                policy->prometheus_metrics(output);
-            } catch (const std::exception &e) {
-                res.status = 500;
-                res.set_content(e.what(), "text/plain");
+                policy->prometheus_metrics(ser);
             }
-            res.set_content(output.str(), "text/plain");
+        } catch (const std::exception &e) {
+            res.status = 500;
+            res.set_content(e.what(), "text/plain");
+            return;
         }
+        output << ser.finalize();
+        res.set_content(output.str(), "text/plain");
     });
     if (_otel) {
         _otel->OnInterval([&](metrics::v1::ResourceMetrics &resource) {

src/MetricLabels.h

@@ -0,0 +1,17 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
+#pragma once
+
+#include <map>
+#include <string>
+
+namespace visor {
+
+using LabelMap = std::map<std::string, std::string>;
+
+// Process-global static labels (e.g. instance). Storage defined in Metrics.cpp.
+LabelMap &prometheus_static_labels_mutable();
+inline const LabelMap &prometheus_static_labels() { return prometheus_static_labels_mutable(); }
+
+}

src/Metrics.cpp

@@ -3,6 +3,7 @@
  * file, You can obtain one at https://mozilla.org/MPL/2.0/. */
 
 #include "Metrics.h"
+#include "PrometheusSerializer.h"
 #include <cpc_union.hpp>
 
 namespace visor {
@@ -12,11 +13,9 @@ void Counter::to_json(json &j) const
     name_json_assign(j, _value);
 }
 
-void Counter::to_prometheus(std::stringstream &out, Metric::LabelMap add_labels) const
+void Counter::to_prometheus(PrometheusSerializer &ser, Metric::LabelMap add_labels) const
 {
-    out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
-    out << "# TYPE " << base_name_snake() << " gauge" << std::endl;
-    out << name_snake({}, add_labels) << ' ' << _value << std::endl;
+    ser.write(base_name_snake(), PrometheusSerializer::Type::Gauge, _desc, {}, add_labels, _value);
 }
 
 void Counter::to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &start, timespec &end, Metric::LabelMap add_labels) const
@@ -49,10 +48,10 @@ void Rate::to_json(visor::json &j) const
     _quantile.to_json(j);
 }
 
-void Rate::to_prometheus(std::stringstream &out, Metric::LabelMap add_labels) const
+void Rate::to_prometheus(PrometheusSerializer &ser, Metric::LabelMap add_labels) const
 {
     std::shared_lock lock(_sketch_mutex);
-    _quantile.to_prometheus(out, add_labels);
+    _quantile.to_prometheus(ser, add_labels);
 }
 
 void Rate::to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &start, timespec &end, Metric::LabelMap add_labels) const
@@ -72,11 +71,9 @@ void Cardinality::to_json(json &j) const
 {
     name_json_assign(j, lround(_set.get_estimate()));
 }
-void Cardinality::to_prometheus(std::stringstream &out, Metric::LabelMap add_labels) const
+void Cardinality::to_prometheus(PrometheusSerializer &ser, Metric::LabelMap add_labels) const
 {
-    out << "# HELP " << base_name_snake() << ' ' << _desc << std::endl;
-    out << "# TYPE " << base_name_snake() << " gauge" << std::endl;
-    out << name_snake({}, add_labels) << ' ' << lround(_set.get_estimate()) << std::endl;
+    ser.write(base_name_snake(), PrometheusSerializer::Type::Gauge, _desc, {}, add_labels, lround(_set.get_estimate()));
 }
 
 void Cardinality::to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &start, timespec &end, Metric::LabelMap add_labels) const
@@ -95,8 +92,11 @@ void Cardinality::to_opentelemetry(metrics::v1::ScopeMetrics &scope, timespec &s
     }
 }
 
-// static storage for base labels
-Metric::LabelMap Metric::_static_labels;
+LabelMap &prometheus_static_labels_mutable()
+{
+    static LabelMap labels;
+    return labels;
+}
 
 void Metric::name_json_assign(json &j, const json &val) const
 {
@@ -126,32 +126,4 @@ std::string Metric::base_name_snake() const
     return name_text;
 }
 
-std::string Metric::name_snake(std::initializer_list<std::string> add_names, Metric::LabelMap add_labels) const
-{
-    std::string label_text{"{"};
-    if (!_static_labels.empty()) {
-        for (const auto &[key, value] : _static_labels) {
-            label_text.append(key + "=\"" + value + "\",");
-        }
-    }
-    if (add_labels.size()) {
-        for (const auto &[key, value] : add_labels) {
-            label_text.append(key + "=\"" + value + "\",");
-        }
-    }
-    if (label_text.back() == ',') {
-        label_text.pop_back();
-    }
-    label_text.push_back('}');
-    auto snake = [](const std::string &ss, const std::string &s) {
-        return ss.empty() ? s : ss + "_" + s;
-    };
-    std::string name_text = _schema_key + "_" + std::accumulate(std::begin(_name), std::end(_name), std::string(), snake);
-    if (add_names.size()) {
-        name_text.push_back('_');
-        name_text.append(std::accumulate(std::begin(add_names), std::end(add_names), std::string(), snake));
-    }
-    return name_text + label_text;
-}
-
 }