Add support for callbacks for gauges/counters, restore process stat metrics

CMakeLists.txt

@@ -234,13 +234,13 @@ if (ZEEK_STANDALONE)
     endif ()
     # Tell zeek_target_link_libraries to add library dependencies as PRIVATE.
     set(zeek_exe_access PRIVATE)
-else ()
-    add_library(zeek_lib STATIC)
 
     if (${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
         target_link_libraries(zeek_exe PRIVATE util)
         target_link_libraries(zeek_exe PRIVATE procstat)
     endif ()
+else ()
+    add_library(zeek_lib STATIC)
 endif ()
 
 if (TARGET zeek_lib)
@@ -258,6 +258,7 @@ if (TARGET zeek_lib)
         target_link_libraries(zeek_exe PRIVATE util)
         target_link_libraries(zeek_exe PRIVATE procstat)
     endif ()
+
 endif ()
 
 # When building our fuzzers, we also need one extra top-level target that

auxil/prometheus-cpp

@@ -1 +1 @@
-Subproject commit 4bd38da318ec54af8e2d8d5d0bdbd5eb9bc0784f
+Subproject commit cdb357ad556c9ba96cbfa90fed2940fedf101673

src/telemetry/Counter.h

@@ -41,7 +41,12 @@ public:
         return Value();
     }
 
-    BaseType Value() const noexcept { return static_cast<BaseType>(handle.Value()); }
+    BaseType Value() const noexcept {
+        // Use Collect() here instead of Value() to correctly handle metrics with
+        // callbacks.
+        auto metric = handle.Collect();
+        return static_cast<BaseType>(metric.counter.value);
+    }
 
     bool operator==(const BaseCounter<BaseType>& rhs) const noexcept { return &handle == &rhs.handle; }
     bool operator!=(const BaseCounter<BaseType>& rhs) const noexcept { return &handle != &rhs.handle; }
@@ -49,8 +54,12 @@ public:
     bool CompareLabels(const prometheus::Labels& lbls) const { return labels == lbls; }
 
 protected:
-    explicit BaseCounter(FamilyType& family, const prometheus::Labels& labels) noexcept
+    explicit BaseCounter(FamilyType& family, const prometheus::Labels& labels,
-        : handle(family.Add(labels)), labels(labels) {}
+                         prometheus::CollectCallbackPtr callback = nullptr) noexcept
+        : handle(family.Add(labels)), labels(labels) {
+        if ( callback )
+            handle.AddCollectCallback(callback);
+    }
 
     Handle& handle;
     prometheus::Labels labels;
@@ -63,7 +72,9 @@ protected:
 class IntCounter : public BaseCounter<uint64_t> {
 public:
     static inline const char* OpaqueName = "IntCounterMetricVal";
-    explicit IntCounter(FamilyType& family, const prometheus::Labels& labels) noexcept : BaseCounter(family, labels) {}
+    explicit IntCounter(FamilyType& family, const prometheus::Labels& labels,
+                        prometheus::CollectCallbackPtr callback = nullptr) noexcept
+        : BaseCounter(family, labels, callback) {}
 };
 
 /**
@@ -72,7 +83,9 @@ public:
 class DblCounter : public BaseCounter<double> {
 public:
     static inline const char* OpaqueName = "DblCounterMetricVal";
-    explicit DblCounter(FamilyType& family, const prometheus::Labels& labels) noexcept : BaseCounter(family, labels) {}
+    explicit DblCounter(FamilyType& family, const prometheus::Labels& labels,
+                        prometheus::CollectCallbackPtr callback = nullptr) noexcept
+        : BaseCounter(family, labels, callback) {}
 };
 
 template<class CounterType, typename BaseType>
@@ -89,7 +102,8 @@ public:
      * Returns the metrics handle for given labels, creating a new instance
      * lazily if necessary.
      */
-    std::shared_ptr<CounterType> GetOrAdd(Span<const LabelView> labels) {
+    std::shared_ptr<CounterType> GetOrAdd(Span<const LabelView> labels,
+                                          prometheus::CollectCallbackPtr callback = nullptr) {
         prometheus::Labels p_labels = BuildPrometheusLabels(labels);
 
         auto check = [&](const std::shared_ptr<CounterType>& counter) { return counter->CompareLabels(p_labels); };
@@ -97,7 +111,7 @@ public:
         if ( auto it = std::find_if(counters.begin(), counters.end(), check); it != counters.end() )
             return *it;
 
-        auto counter = std::make_shared<CounterType>(family, p_labels);
+        auto counter = std::make_shared<CounterType>(family, p_labels, callback);
         counters.push_back(counter);
         return counter;
     }
@@ -105,8 +119,9 @@ public:
     /**
      * @copydoc GetOrAdd
      */
-    std::shared_ptr<CounterType> GetOrAdd(std::initializer_list<LabelView> labels) {
+    std::shared_ptr<CounterType> GetOrAdd(std::initializer_list<LabelView> labels,
-        return GetOrAdd(Span{labels.begin(), labels.size()});
+                                          prometheus::CollectCallbackPtr callback = nullptr) {
+        return GetOrAdd(Span{labels.begin(), labels.size()}, callback);
     }
 
     std::vector<std::shared_ptr<CounterType>>& GetAllCounters() { return counters; }

src/telemetry/Gauge.h

@@ -59,7 +59,12 @@ public:
         return Value();
     }
 
-    BaseType Value() const noexcept { return static_cast<BaseType>(handle.Value()); }
+    BaseType Value() const noexcept {
+        // Use Collect() here instead of Value() to correctly handle metrics
+        // with callbacks.
+        auto metric = handle.Collect();
+        return static_cast<BaseType>(metric.gauge.value);
+    }
 
     /**
      * Directly sets the value of the gauge.
@@ -72,8 +77,12 @@ public:
     bool CompareLabels(const prometheus::Labels& lbls) const { return labels == lbls; }
 
 protected:
-    explicit BaseGauge(FamilyType& family, const prometheus::Labels& labels) noexcept
+    explicit BaseGauge(FamilyType& family, const prometheus::Labels& labels,
-        : handle(family.Add(labels)), labels(labels) {}
+                       prometheus::CollectCallbackPtr callback = nullptr) noexcept
+        : handle(family.Add(labels)), labels(labels) {
+        if ( callback )
+            handle.AddCollectCallback(callback);
+    }
 
     Handle& handle;
     prometheus::Labels labels;
@@ -88,7 +97,9 @@ class IntGauge : public BaseGauge<int64_t> {
 public:
     static inline const char* OpaqueName = "IntGaugeMetricVal";
 
-    explicit IntGauge(FamilyType& family, const prometheus::Labels& labels) noexcept : BaseGauge(family, labels) {}
+    explicit IntGauge(FamilyType& family, const prometheus::Labels& labels,
+                      prometheus::CollectCallbackPtr callback = nullptr) noexcept
+        : BaseGauge(family, labels, callback) {}
 
     IntGauge(const IntGauge&) = delete;
     IntGauge& operator=(const IntGauge&) = delete;
@@ -102,7 +113,9 @@ class DblGauge : public BaseGauge<double> {
 public:
     static inline const char* OpaqueName = "DblGaugeMetricVal";
 
-    explicit DblGauge(FamilyType& family, const prometheus::Labels& labels) noexcept : BaseGauge(family, labels) {}
+    explicit DblGauge(FamilyType& family, const prometheus::Labels& labels,
+                      prometheus::CollectCallbackPtr callback = nullptr) noexcept
+        : BaseGauge(family, labels, callback) {}
 
     DblGauge(const DblGauge&) = delete;
     DblGauge& operator=(const DblGauge&) = delete;
@@ -121,7 +134,8 @@ public:
      * Returns the metrics handle for given labels, creating a new instance
      * lazily if necessary.
      */
-    std::shared_ptr<GaugeType> GetOrAdd(Span<const LabelView> labels) {
+    std::shared_ptr<GaugeType> GetOrAdd(Span<const LabelView> labels,
+                                        prometheus::CollectCallbackPtr callback = nullptr) {
         prometheus::Labels p_labels = BuildPrometheusLabels(labels);
 
         auto check = [&](const std::shared_ptr<GaugeType>& gauge) { return gauge->CompareLabels(p_labels); };
@@ -129,7 +143,7 @@ public:
         if ( auto it = std::find_if(gauges.begin(), gauges.end(), check); it != gauges.end() )
             return *it;
 
-        auto gauge = std::make_shared<GaugeType>(family, p_labels);
+        auto gauge = std::make_shared<GaugeType>(family, p_labels, callback);
         gauges.push_back(gauge);
         return gauge;
     }
@@ -137,8 +151,9 @@ public:
     /**
      * @copydoc GetOrAdd
      */
-    std::shared_ptr<GaugeType> GetOrAdd(std::initializer_list<LabelView> labels) {
+    std::shared_ptr<GaugeType> GetOrAdd(std::initializer_list<LabelView> labels,
-        return GetOrAdd(Span{labels.begin(), labels.size()});
+                                        prometheus::CollectCallbackPtr callback = nullptr) {
+        return GetOrAdd(Span{labels.begin(), labels.size()}, callback);
     }
 
     std::vector<std::shared_ptr<GaugeType>>& GetAllGauges() { return gauges; }
@@ -196,7 +211,6 @@ public:
                    std::string_view unit = "", bool is_sum = false)
         : BaseGaugeFamily(prefix, name, labels, helptext, std::move(registry), unit, is_sum) {}
 
-
     IntGaugeFamily(const IntGaugeFamily&) noexcept = default;
     IntGaugeFamily& operator=(const IntGaugeFamily&) noexcept = delete;
 
@@ -215,7 +229,6 @@ public:
                    std::string_view unit = "", bool is_sum = false)
         : BaseGaugeFamily(prefix, name, labels, helptext, std::move(registry), unit, is_sum) {}
 
-
     DblGaugeFamily(const DblGaugeFamily&) noexcept = default;
     DblGaugeFamily& operator=(const DblGaugeFamily&) noexcept = delete;
 

src/telemetry/Histogram.h

@@ -28,7 +28,6 @@ public:
     void Observe(BaseType value) noexcept { handle.Observe(value); }
 
     /// @return The sum of all observed values.
-    // TODO
     BaseType Sum() const noexcept {
         auto metric = handle.Collect();
         return static_cast<BaseType>(metric.histogram.sample_sum);

src/telemetry/Manager.cc

@@ -89,42 +89,37 @@ void Manager::InitPostScript() {
 
         return &this->current_process_stats;
     };
-/*
+    rss_gauge = GaugeInstance<int64_t>("process", "resident_memory", {}, "Resident memory size", "bytes", false,
-    rss_gauge =
+                                       []() -> prometheus::ClientMetric {
-        GaugeInstance<int64_t>("process", "resident_memory", {}, "Resident memory size", "bytes", false,
-                               [](metrics_api::ObserverResult r, void* state) {
                                            auto* s = get_stats();
-                                   opentelemetry::nostd::get<
+                                           prometheus::ClientMetric metric;
-                                       opentelemetry::nostd::shared_ptr<metrics_api::ObserverResultT<int64_t>>>(r)
+                                           metric.gauge.value = static_cast<double>(s->rss);
-                                       ->Observe(s->rss);
+                                           return metric;
                                        });
 
-    vms_gauge =
+    vms_gauge = GaugeInstance<int64_t>("process", "virtual_memory", {}, "Virtual memory size", "bytes", false,
-        GaugeInstance<int64_t>("process", "virtual_memory", {}, "Virtual memory size", "bytes", false,
+                                       []() -> prometheus::ClientMetric {
-                               [](metrics_api::ObserverResult r, void* state) {
                                            auto* s = get_stats();
-                                   opentelemetry::nostd::get<
+                                           prometheus::ClientMetric metric;
-                                       opentelemetry::nostd::shared_ptr<metrics_api::ObserverResultT<int64_t>>>(r)
+                                           metric.gauge.value = static_cast<double>(s->vms);
-                                       ->Observe(s->vms);
+                                           return metric;
                                        });
 
     cpu_gauge = GaugeInstance<double>("process", "cpu", {}, "Total user and system CPU time spent", "seconds", false,
-                                      [](metrics_api::ObserverResult r, void* state) {
+                                      []() -> prometheus::ClientMetric {
                                           auto* s = get_stats();
-                                          opentelemetry::nostd::get<
+                                          prometheus::ClientMetric metric;
-                                              opentelemetry::nostd::shared_ptr<metrics_api::ObserverResultT<double>>>(r)
+                                          metric.gauge.value = s->cpu;
-                                              ->Observe(s->cpu);
+                                          return metric;
                                       });
 
-    fds_gauge =
+    fds_gauge = GaugeInstance<int64_t>("process", "open_fds", {}, "Number of open file descriptors", "", false,
-        GaugeInstance<int64_t>("process", "open_fds", {}, "Number of open file descriptors", "", false,
+                                       []() -> prometheus::ClientMetric {
-                               [](metrics_api::ObserverResult r, void* state) {
                                            auto* s = get_stats();
-                                   opentelemetry::nostd::get<
+                                           prometheus::ClientMetric metric;
-                                       opentelemetry::nostd::shared_ptr<metrics_api::ObserverResultT<int64_t>>>(r)
+                                           metric.gauge.value = static_cast<double>(s->fds);
-                                       ->Observe(s->fds);
+                                           return metric;
                                        });
-*/
 #endif
 }
 

src/telemetry/Manager.h

@@ -122,10 +122,11 @@ public:
     template<class ValueType = int64_t>
     std::shared_ptr<Counter<ValueType>> CounterInstance(std::string_view prefix, std::string_view name,
                                                         Span<const LabelView> labels, std::string_view helptext,
-                                                        std::string_view unit = "", bool is_sum = false) {
+                                                        std::string_view unit = "", bool is_sum = false,
+                                                        prometheus::CollectCallbackPtr callback = nullptr) {
         return WithLabelNames(labels, [&, this](auto labelNames) {
             auto family = CounterFamily<ValueType>(prefix, name, labelNames, helptext, unit, is_sum);
-            return family->GetOrAdd(labels);
+            return family->GetOrAdd(labels, callback);
         });
     }
 
@@ -134,9 +135,10 @@ public:
     std::shared_ptr<Counter<ValueType>> CounterInstance(std::string_view prefix, std::string_view name,
                                                         std::initializer_list<LabelView> labels,
                                                         std::string_view helptext, std::string_view unit = "",
-                                                        bool is_sum = false) {
+                                                        bool is_sum = false,
+                                                        prometheus::CollectCallbackPtr callback = nullptr) {
         auto lbl_span = Span{labels.begin(), labels.size()};
-        return CounterInstance<ValueType>(prefix, name, lbl_span, helptext, unit, is_sum);
+        return CounterInstance<ValueType>(prefix, name, lbl_span, helptext, unit, is_sum, callback);
     }
 
     /**
@@ -199,10 +201,11 @@ public:
     template<class ValueType = int64_t>
     std::shared_ptr<Gauge<ValueType>> GaugeInstance(std::string_view prefix, std::string_view name,
                                                     Span<const LabelView> labels, std::string_view helptext,
-                                                    std::string_view unit = "", bool is_sum = false) {
+                                                    std::string_view unit = "", bool is_sum = false,
+                                                    prometheus::CollectCallbackPtr callback = nullptr) {
         return WithLabelNames(labels, [&, this](auto labelNames) {
             auto family = GaugeFamily<ValueType>(prefix, name, labelNames, helptext, unit, is_sum);
-            return family->GetOrAdd(labels);
+            return family->GetOrAdd(labels, callback);
         });
     }
 
@@ -210,9 +213,10 @@ public:
     template<class ValueType = int64_t>
     std::shared_ptr<Gauge<ValueType>> GaugeInstance(std::string_view prefix, std::string_view name,
                                                     std::initializer_list<LabelView> labels, std::string_view helptext,
-                                                    std::string_view unit = "", bool is_sum = false) {
+                                                    std::string_view unit = "", bool is_sum = false,
+                                                    prometheus::CollectCallbackPtr callback = nullptr) {
         auto lbl_span = Span{labels.begin(), labels.size()};
-        return GaugeInstance<ValueType>(prefix, name, lbl_span, helptext, unit, is_sum);
+        return GaugeInstance<ValueType>(prefix, name, lbl_span, helptext, unit, is_sum, callback);
     }
 
     // Forces the compiler to use the type `Span<const T>` instead of trying to

src/telemetry/ProcessStats.cc

@@ -173,6 +173,7 @@ process_stats get_process_stats() {
     process_stats result;
 
     struct kinfo_proc* kp = kinfo_getproc(getpid());
+    if ( kp ) {
         result.vms = kp->ki_size;
         result.rss = kp->ki_rssize * getpagesize();
         result.cpu = static_cast<double>(kp->ki_runtime) / 1000000.0;
@@ -188,17 +189,11 @@ process_stats get_process_stats() {
 
         procstat_freeprocs(procstat, kp);
         procstat_close(procstat);
+    }
 
     return result;
 }
 
-#else
-
-process_stats get_process_stats() {
-    process_stats result = {0};
-    return result;
-}
-
 } // namespace zeek::telemetry::detail
 
 #endif