Expose telemetry API to scripting land

scripts/base/init-bare.zeek

@@ -46,6 +46,20 @@ type count_set: set[count];
 ##    directly and then remove this alias.
 type index_vec: vector of count;
 
+## A vector of integers, used by telemetry builtin functions to store histogram bounds.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type int_vec: vector of int;
+
+## A vector of floating point numbers, used by telemetry builtin functions to store histogram bounds.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type double_vec: vector of double;
+
 ## A vector of subnets.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions

src/OpaqueVal.cc

@@ -1033,4 +1033,63 @@ ValPtr ParaglobVal::DoClone(CloneState* state)
 		}
 	}
 
+broker::expected<broker::data> TelemetryVal::DoSerialize() const
+	{
+	return broker::make_error(broker::ec::invalid_data,
+	                          "cannot serialize metric handles");
+	}
+
+bool TelemetryVal::DoUnserialize(const broker::data&)
+	{
+	return false;
+	}
+
+TelemetryVal::TelemetryVal(telemetry::IntCounter) :OpaqueVal(int_counter_metric_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::IntCounterFamily) :OpaqueVal(int_counter_metric_family_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::DblCounter) :OpaqueVal(dbl_counter_metric_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::DblCounterFamily) :OpaqueVal(dbl_counter_metric_family_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::IntGauge) :OpaqueVal(int_gauge_metric_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::IntGaugeFamily) :OpaqueVal(int_gauge_metric_family_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::DblGauge) :OpaqueVal(dbl_gauge_metric_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::DblGaugeFamily) :OpaqueVal(dbl_gauge_metric_family_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::IntHistogram) :OpaqueVal(int_histogram_metric_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::IntHistogramFamily) :OpaqueVal(int_histogram_metric_family_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::DblHistogram) :OpaqueVal(dbl_histogram_metric_type)
+	{
+	}
+
+TelemetryVal::TelemetryVal(telemetry::DblHistogramFamily) :OpaqueVal(dbl_histogram_metric_family_type)
+	{
+	}
+
 }

src/OpaqueVal.h

@@ -10,6 +10,9 @@
 #include "zeek/RandTest.h"
 #include "zeek/Val.h"
 #include "zeek/digest.h"
+#include "zeek/telemetry/Counter.h"
+#include "zeek/telemetry/Gauge.h"
+#include "zeek/telemetry/Histogram.h"
 
 namespace broker { class data; }
 
@@ -373,4 +376,66 @@ private:
 	std::unique_ptr<paraglob::Paraglob> internal_paraglob;
 };
 
+/**
+ * Base class for metric handles. Handle types are not serializable.
+ */
+class TelemetryVal : public OpaqueVal {
+protected:
+	explicit TelemetryVal(telemetry::IntCounter);
+	explicit TelemetryVal(telemetry::IntCounterFamily);
+	explicit TelemetryVal(telemetry::DblCounter);
+	explicit TelemetryVal(telemetry::DblCounterFamily);
+	explicit TelemetryVal(telemetry::IntGauge);
+	explicit TelemetryVal(telemetry::IntGaugeFamily);
+	explicit TelemetryVal(telemetry::DblGauge);
+	explicit TelemetryVal(telemetry::DblGaugeFamily);
+	explicit TelemetryVal(telemetry::IntHistogram);
+	explicit TelemetryVal(telemetry::IntHistogramFamily);
+	explicit TelemetryVal(telemetry::DblHistogram);
+	explicit TelemetryVal(telemetry::DblHistogramFamily);
+
+	broker::expected<broker::data> DoSerialize() const override;
+	bool DoUnserialize(const broker::data& data) override;
+};
+
+template <class Handle>
+class TelemetryValImpl : public TelemetryVal {
+public:
+	using HandleType = Handle;
+
+	explicit TelemetryValImpl(Handle hdl) : TelemetryVal(hdl), hdl(hdl) { }
+
+	Handle GetHandle() const noexcept
+		{
+		return hdl;
+		}
+
+protected:
+	ValPtr DoClone(CloneState*) override
+		{
+		return make_intrusive<TelemetryValImpl>(hdl);
+		}
+
+	const char* OpaqueName() const override
+		{
+		return Handle::OpaqueName;
+		}
+
+private:
+	Handle hdl;
+};
+
+using IntCounterMetricVal = TelemetryValImpl<telemetry::IntCounter>;
+using IntCounterMetricFamilyVal = TelemetryValImpl<telemetry::IntCounterFamily>;
+using DblCounterMetricVal = TelemetryValImpl<telemetry::DblCounter>;
+using DblCounterMetricFamilyVal = TelemetryValImpl<telemetry::DblCounterFamily>;
+using IntGaugeMetricVal = TelemetryValImpl<telemetry::IntGauge>;
+using IntGaugeMetricFamilyVal = TelemetryValImpl<telemetry::IntGaugeFamily>;
+using DblGaugeMetricVal = TelemetryValImpl<telemetry::DblGauge>;
+using DblGaugeMetricFamilyVal = TelemetryValImpl<telemetry::DblGaugeFamily>;
+using IntHistogramMetricVal = TelemetryValImpl<telemetry::IntHistogram>;
+using IntHistogramMetricFamilyVal = TelemetryValImpl<telemetry::IntHistogramFamily>;
+using DblHistogramMetricVal = TelemetryValImpl<telemetry::DblHistogram>;
+using DblHistogramMetricFamilyVal = TelemetryValImpl<telemetry::DblHistogramFamily>;
+
 } // namespace zeek

src/Type.h

@@ -913,3 +913,15 @@ extern zeek::OpaqueTypePtr bloomfilter_type;
 extern zeek::OpaqueTypePtr x509_opaque_type;
 extern zeek::OpaqueTypePtr ocsp_resp_opaque_type;
 extern zeek::OpaqueTypePtr paraglob_type;
+extern zeek::OpaqueTypePtr int_counter_metric_type;
+extern zeek::OpaqueTypePtr int_counter_metric_family_type;
+extern zeek::OpaqueTypePtr dbl_counter_metric_type;
+extern zeek::OpaqueTypePtr dbl_counter_metric_family_type;
+extern zeek::OpaqueTypePtr int_gauge_metric_type;
+extern zeek::OpaqueTypePtr int_gauge_metric_family_type;
+extern zeek::OpaqueTypePtr dbl_gauge_metric_type;
+extern zeek::OpaqueTypePtr dbl_gauge_metric_family_type;
+extern zeek::OpaqueTypePtr int_histogram_metric_type;
+extern zeek::OpaqueTypePtr int_histogram_metric_family_type;
+extern zeek::OpaqueTypePtr dbl_histogram_metric_type;
+extern zeek::OpaqueTypePtr dbl_histogram_metric_family_type;

src/broker/Manager.cc

@@ -1815,23 +1815,45 @@ std::unique_ptr<telemetry::Manager> Manager::NewTelemetryManager()
 	// owns the CAF actor system, lives for as long as necessary. This also
 	// makes sure that the Broker Manager may even get destroyed before the
 	// telemetry Manager.
-	struct TM : public telemetry::Manager
+	struct TM final : public telemetry::Manager
 	{
+		using MetricRegistryPtr = std::unique_ptr<caf::telemetry::metric_registry>;
+
 		static auto getPimpl(BrokerState& st)
 			{
 			auto registry = std::addressof(st.endpoint.system().metrics());
 			return reinterpret_cast<telemetry::Manager::Impl*>(registry);
 			}
 
-		explicit TM(const std::shared_ptr<BrokerState>& bsptr)
-		: telemetry::Manager(getPimpl(*bsptr)), ptr(bsptr)
+		static auto getPimpl(MetricRegistryPtr& ptr)
 			{
+			return reinterpret_cast<telemetry::Manager::Impl*>(ptr.get());
 			}
 
+		explicit TM(Broker::Manager* parent, MetricRegistryPtr ptr)
+		: telemetry::Manager(getPimpl(ptr)), parent(parent), tmp(std::move(ptr))
+			{
+			assert(tmp != nullptr);
+			assert(parent != nullptr);
+			}
+
+		void InitPostScript() override
+			{
+			assert(parent->bstate != nullptr);
+			ptr = parent->bstate;
+			auto registry = std::addressof(ptr->endpoint.system().metrics());
+			registry->merge(*tmp);
+			tmp.reset();
+			pimpl = reinterpret_cast<telemetry::Manager::Impl*>(registry);
+			}
+
+		Broker::Manager* parent;
+		MetricRegistryPtr tmp;
 		std::shared_ptr<BrokerState> ptr;
 	};
 
-	return std::make_unique<TM>(bstate);
+	auto tmp = std::make_unique<caf::telemetry::metric_registry>();
+	return std::make_unique<TM>(this, std::move(tmp));
 	}
 
 } // namespace zeek::Broker

src/telemetry/CMakeLists.txt

@@ -13,10 +13,7 @@ set(telemetry_SRCS
     MetricFamily.cc
 )
 
-# bif_target(comm.bif)
-# bif_target(data.bif)
-# bif_target(messaging.bif)
-# bif_target(store.bif)
+bif_target(telemetry.bif)
 
 bro_add_subdir_library(telemetry ${telemetry_SRCS})
 

src/telemetry/Counter.h

@@ -24,6 +24,8 @@ public:
 
 	struct Impl;
 
+	static inline const char* OpaqueName = "IntCounterMetricVal";
+
 	IntCounter() = delete;
 	IntCounter(const IntCounter&) noexcept = default;
 	IntCounter& operator=(const IntCounter&) noexcept = default;
@@ -91,6 +93,8 @@ public:
 
 	class Impl;
 
+	static inline const char* OpaqueName = "IntCounterMetricFamilyVal";
+
 	using InstanceType = IntCounter;
 
 	IntCounterFamily(const IntCounterFamily&) noexcept = default;
@@ -124,6 +128,8 @@ public:
 
 	struct Impl;
 
+	static inline const char* OpaqueName = "DblCounterMetricVal";
+
 	DblCounter() = delete;
 	DblCounter(const DblCounter&) noexcept = default;
 	DblCounter& operator=(const DblCounter&) noexcept = default;
@@ -185,6 +191,8 @@ public:
 
 	class Impl;
 
+	static inline const char* OpaqueName = "DblCounterMetricFamilyVal";
+
 	using InstanceType = DblCounter;
 
 	DblCounterFamily(const DblCounterFamily&) noexcept = default;

src/telemetry/Gauge.h

@@ -25,6 +25,8 @@ public:
 
 	struct Impl;
 
+	static inline const char* OpaqueName = "IntGaugeMetricVal";
+
 	IntGauge() = delete;
 	IntGauge(const IntGauge&) noexcept = default;
 	IntGauge& operator=(const IntGauge&) noexcept = default;
@@ -107,6 +109,8 @@ public:
 
 	class Impl;
 
+	static inline const char* OpaqueName = "IntGaugeMetricFamilyVal";
+
 	using InstanceType = IntGauge;
 
 	IntGaugeFamily(const IntGaugeFamily&) noexcept = default;
@@ -141,6 +145,8 @@ public:
 
 	struct Impl;
 
+	static inline const char* OpaqueName = "DblGaugeMetricVal";
+
 	DblGauge() = delete;
 	DblGauge(const DblGauge&) noexcept = default;
 	DblGauge& operator=(const DblGauge&) noexcept = default;
@@ -211,6 +217,8 @@ public:
 
 	class Impl;
 
+	static inline const char* OpaqueName = "DblGaugeMetricFamilyVal";
+
 	using InstanceType = DblGauge;
 
 	DblGaugeFamily(const DblGaugeFamily&) noexcept = default;

src/telemetry/Histogram.h

@@ -26,6 +26,8 @@ public:
 
 	struct Impl;
 
+	static inline const char* OpaqueName = "IntHistogramMetricVal";
+
 	IntHistogram() = delete;
 	IntHistogram(const IntHistogram&) noexcept = default;
 	IntHistogram& operator=(const IntHistogram&) noexcept = default;
@@ -90,6 +92,8 @@ public:
 
 	class Impl;
 
+	static inline const char* OpaqueName = "IntHistogramMetricFamilyVal";
+
 	using InstanceType = IntHistogram;
 
 	IntHistogramFamily(const IntHistogramFamily&) noexcept = default;
@@ -124,6 +128,8 @@ public:
 
 	struct Impl;
 
+	static inline const char* OpaqueName = "DblHistogramMetricVal";
+
 	DblHistogram() = delete;
 	DblHistogram(const DblHistogram&) noexcept = default;
 	DblHistogram& operator=(const DblHistogram&) noexcept = default;
@@ -188,6 +194,8 @@ public:
 
 	class Impl;
 
+	static inline const char* OpaqueName = "DblHistogramMetricFamilyVal";
+
 	using InstanceType = DblHistogram;
 
 	DblHistogramFamily(const DblHistogramFamily&) noexcept = default;

src/telemetry/Manager.cc

@@ -14,6 +14,10 @@ Manager::~Manager()
 	{
 	}
 
+void Manager::InitPostScript()
+	{
+	}
+
 IntCounterFamily Manager::IntCounterFam(std::string_view prefix,
                                         std::string_view name,
                                         Span<const std::string_view> labels,

src/telemetry/Manager.h

@@ -31,6 +31,12 @@ public:
 
 	virtual ~Manager();
 
+	/**
+	 * Initialization of the manager. This is called late during Bro's
+	 * initialization after any scripts are processed.
+	 */
+	virtual void InitPostScript();
+
 	/**
 	 * @return A counter metric family. Creates the family lazily if necessary.
 	 * @param prefix The prefix (namespace) this family belongs to.
@@ -384,7 +390,7 @@ public:
 		return fam.GetOrAdd({});
 		}
 
-private:
+protected:
 	IntCounterFamily
 	IntCounterFam(std::string_view prefix, std::string_view name,
 	              Span<const std::string_view> labels,

src/telemetry/telemetry.bif

@@ -0,0 +1,550 @@
+
+##! Functions for accessing counter metrics from script land.
+
+module Telemetry;
+
+%%{
+#include "zeek/telemetry/Counter.h"
+#include "zeek/telemetry/Gauge.h"
+#include "zeek/telemetry/Histogram.h"
+#include "zeek/telemetry/Manager.h"
+
+namespace {
+
+template <class ScriptHandle, class Fun>
+auto with(zeek::Val* val, const char* error_msg, Fun fun)
+	{
+	if ( auto ptr = dynamic_cast<ScriptHandle*>(val) )
+		{
+		fun(ptr->GetHandle());
+		return zeek::val_mgr->True();
+		}
+	else
+		{
+		zeek::reporter->Error("%s", error_msg);
+		return zeek::val_mgr->False();
+		}
+	}
+
+std::string_view sv(const zeek::String* str) noexcept
+	{
+	auto len = static_cast<size_t>(str->Len());
+	return {reinterpret_cast<char*>(str->Bytes()), len};
+	};
+
+std::string_view sv(const zeek::StringVal* val) noexcept
+	{
+	return sv(val->AsString());
+	};
+
+std::vector<std::string_view> sv_vec(zeek::VectorVal* xs)
+	{
+	std::vector<std::string_view> result;
+	if ( xs )
+		for ( unsigned index = 0; index < xs->Size(); ++index )
+			if ( auto& ptr = xs->At(index) )
+				if ( auto* str = ptr->AsString() )
+					result.emplace_back(sv(str));
+	return result;
+	}
+
+std::vector<zeek::telemetry::LabelView> sv_tbl(zeek::TableVal* xs)
+	{
+	std::vector<zeek::telemetry::LabelView> result;
+	if ( xs )
+		{
+		for ( auto& val : *xs->Get() )
+			{
+			auto val_ptr = val.GetValue<zeek::TableEntryVal*>()->GetVal();
+			result.emplace_back(std::string_view{val.GetKey(), val.key_size},
+			                    sv(val_ptr->AsStringVal()));
+			}
+		}
+	return result;
+	}
+
+bool is_valid(zeek::Span<const zeek::telemetry::LabelView> labels,
+              zeek::Span<const std::string> label_names)
+	{
+	auto key_in_label_names = [keys{label_names}](auto x)
+		{
+		return std::find(keys.begin(), keys.end(), x.first) != keys.end();
+		};
+	return labels.size() == label_names.size()
+	       && std::all_of(labels.begin(), labels.end(), key_in_label_names);
+	}
+
+template <class IntOrDbl>
+auto to_std_vec(zeek::VectorVal* xs)
+	{
+	std::vector<IntOrDbl> result;
+	if( xs )
+		for ( unsigned index = 0; index < xs->Size() ; ++index )
+			{
+			if constexpr (std::is_same<IntOrDbl, int64_t>::value)
+				result.emplace_back(xs->At(index)->AsInt());
+			else
+				result.emplace_back(xs->At(index)->AsDouble());
+			}
+	return result;
+	}
+
+template <class IntOrDbl>
+auto to_std_vec(zeek::Val* xs)
+	{
+	return to_std_vec<IntOrDbl>(xs->AsVectorVal());
+	}
+
+}
+%%}
+
+# -- IntCounter ----------------------------------------------------------------
+
+function Telemetry::int_counter_family%(prefix: string,
+                                        name: string,
+                                        labels: string_vec,
+                                        helptext: string &default = "Zeek Script Metric",
+                                        unit: string &default = "1",
+                                        is_sum: bool &default = F%): opaque of int_counter_metric_family
+	%{
+	auto lbl_vec = sv_vec(labels->AsVectorVal());
+	auto hdl = telemetry_mgr->CounterFamily(sv(prefix), sv(name), lbl_vec,
+	                                        sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<IntCounterMetricFamilyVal>(hdl);
+	%}
+
+function Telemetry::int_counter_metric_get_or_add%(family: opaque of int_counter_metric_family,
+                                                   labels: table_string_of_string%): opaque of int_counter_metric
+	%{
+	using ResultType = zeek::IntrusivePtr<IntCounterMetricFamilyVal>;
+	if ( auto ptr = dynamic_cast<zeek::IntCounterMetricFamilyVal*>(family) )
+		{
+		auto hdl = ptr->GetHandle();
+		auto lbl_map = sv_tbl(labels->AsTableVal());
+		if ( is_valid(lbl_map, hdl.LabelNames()) )
+			{
+			auto res = hdl.GetOrAdd(lbl_map);
+			return zeek::make_intrusive<IntCounterMetricVal>(res);
+			}
+		else
+			{
+			zeek::reporter->Error("Telemetry::int_counter_metric_get_or_add: invalid label dimensions.");
+			return ResultType{nullptr};
+			}
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::int_counter_metric_get_or_add: invalid handle.");
+		return ResultType{nullptr};
+		}
+	%}
+
+function Telemetry::int_counter_singleton%(prefix: string,
+                                           name: string,
+                                           helptext: string &default = "Zeek Script Metric",
+                                           unit: string &default = "1",
+                                           is_sum: bool &default = F%): opaque of int_counter_metric
+	%{
+	auto hdl = telemetry_mgr->CounterSingleton(sv(prefix), sv(name),
+	                                           sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<IntCounterMetricVal>(hdl);
+	%}
+
+function Telemetry::int_counter_inc%(val: opaque of int_counter_metric,
+                                     amount: int &default = 1%): bool
+	%{
+	return with<IntCounterMetricVal>(val, "Telemetry::int_counter_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); });
+	%}
+
+function Telemetry::int_counter_value%(val: opaque of int_counter_metric%): int
+	%{
+	if ( auto ptr = dynamic_cast<zeek::IntCounterMetricVal*>(val) )
+		{
+		return zeek::val_mgr->Int(ptr->GetHandle().Value());
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::int_counter_value: invalid handle.");
+		return zeek::val_mgr->Int(0);
+		}
+	%}
+
+# -- DblCounter ----------------------------------------------------------------
+
+function Telemetry::dbl_counter_family%(prefix: string,
+                                        name: string,
+                                        labels: string_vec,
+                                        helptext: string &default = "Zeek Script Metric",
+                                        unit: string &default = "1",
+                                        is_sum: bool &default = F%): opaque of dbl_counter_metric_family
+	%{
+	auto lbl_vec = sv_vec(labels->AsVectorVal());
+	auto hdl = telemetry_mgr->CounterFamily<double>(sv(prefix), sv(name), lbl_vec,
+	                                                sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<DblCounterMetricFamilyVal>(hdl);
+	%}
+
+function Telemetry::dbl_counter_metric_get_or_add%(family: opaque of dbl_counter_metric_family,
+                                                   labels: table_string_of_string%): opaque of dbl_counter_metric
+	%{
+	using ResultType = zeek::IntrusivePtr<DblCounterMetricFamilyVal>;
+	if ( auto ptr = dynamic_cast<zeek::DblCounterMetricFamilyVal*>(family) )
+		{
+		auto hdl = ptr->GetHandle();
+		auto lbl_map = sv_tbl(labels->AsTableVal());
+		if ( is_valid(lbl_map, hdl.LabelNames()) )
+			{
+			auto res = hdl.GetOrAdd(lbl_map);
+			return zeek::make_intrusive<DblCounterMetricVal>(res);
+			}
+		else
+			{
+			zeek::reporter->Error("Telemetry::dbl_counter_metric_get_or_add: invalid label dimensions.");
+			return ResultType{nullptr};
+			}
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::dbl_counter_metric_get_or_add: invalid handle.");
+		return ResultType{nullptr};
+		}
+	%}
+
+function Telemetry::dbl_counter_singleton%(prefix: string,
+                                           name: string,
+                                           helptext: string &default = "Zeek Script Metric",
+                                           unit: string &default = "1",
+                                           is_sum: bool &default = F%): opaque of dbl_counter_metric
+	%{
+	auto hdl = telemetry_mgr->CounterSingleton<double>(sv(prefix), sv(name),
+	                                                   sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<DblCounterMetricVal>(hdl);
+	%}
+
+function Telemetry::dbl_counter_inc%(val: opaque of dbl_counter_metric,
+                                     amount: double &default = 1.0%): bool
+	%{
+	return with<DblCounterMetricVal>(val, "Telemetry::dbl_counter_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); });
+	%}
+
+function Telemetry::dbl_counter_value%(val: opaque of dbl_counter_metric%): double
+	%{
+	if ( auto ptr = dynamic_cast<zeek::DblCounterMetricVal*>(val) )
+		{
+		return zeek::make_intrusive<DoubleVal>(ptr->GetHandle().Value());
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::dbl_counter_value: invalid handle.");
+		return zeek::make_intrusive<DoubleVal>(0.0);
+		}
+	%}
+
+# -- IntGauge ------------------------------------------------------------------
+
+function Telemetry::int_gauge_family%(prefix: string,
+                                      name: string,
+                                      labels: string_vec,
+                                      helptext: string &default = "Zeek Script Metric",
+                                      unit: string &default = "1",
+                                      is_sum: bool &default = F%): opaque of int_gauge_metric_family
+	%{
+	auto lbl_vec = sv_vec(labels->AsVectorVal());
+	auto hdl = telemetry_mgr->GaugeFamily(sv(prefix), sv(name), lbl_vec,
+	                                        sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<IntGaugeMetricFamilyVal>(hdl);
+	%}
+
+function Telemetry::int_gauge_metric_get_or_add%(family: opaque of int_gauge_metric_family,
+                                                 labels: table_string_of_string%): opaque of int_gauge_metric
+	%{
+	using ResultType = zeek::IntrusivePtr<IntGaugeMetricFamilyVal>;
+	if ( auto ptr = dynamic_cast<zeek::IntGaugeMetricFamilyVal*>(family) )
+		{
+		auto hdl = ptr->GetHandle();
+		auto lbl_map = sv_tbl(labels->AsTableVal());
+		if ( is_valid(lbl_map, hdl.LabelNames()) )
+			{
+			auto res = hdl.GetOrAdd(lbl_map);
+			return zeek::make_intrusive<IntGaugeMetricVal>(res);
+			}
+		else
+			{
+			zeek::reporter->Error("Telemetry::int_gauge_metric_get_or_add: invalid label dimensions.");
+			return ResultType{nullptr};
+			}
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::int_gauge_metric_get_or_add: invalid handle.");
+		return ResultType{nullptr};
+		}
+	%}
+
+function Telemetry::int_gauge_singleton%(prefix: string,
+                                         name: string,
+                                         helptext: string &default = "Zeek Script Metric",
+                                         unit: string &default = "1",
+                                         is_sum: bool &default = F%): opaque of int_gauge_metric
+	%{
+	auto hdl = telemetry_mgr->GaugeSingleton(sv(prefix), sv(name),
+	                                         sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<IntGaugeMetricVal>(hdl);
+	%}
+
+function Telemetry::int_gauge_inc%(val: opaque of int_gauge_metric,
+                                   amount: int &default = 1%): bool
+	%{
+	return with<IntGaugeMetricVal>(val, "Telemetry::int_gauge_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); });
+	%}
+
+function Telemetry::int_gauge_dec%(val: opaque of int_gauge_metric,
+                                   amount: int &default = 1%): bool
+	%{
+	return with<IntGaugeMetricVal>(val, "Telemetry::int_gauge_dec: invalid handle.", [amount](auto hdl) { hdl.Dec(amount); });
+	%}
+
+function Telemetry::int_gauge_value%(val: opaque of int_gauge_metric%): int
+	%{
+	if ( auto ptr = dynamic_cast<zeek::IntGaugeMetricVal*>(val) )
+		{
+		return zeek::val_mgr->Int(ptr->GetHandle().Value());
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::int_gauge_value: invalid handle.");
+		return zeek::val_mgr->Int(0);
+		}
+	%}
+
+# -- DblGauge ------------------------------------------------------------------
+
+function Telemetry::dbl_gauge_family%(prefix: string,
+                                      name: string,
+                                      labels: string_vec,
+                                      helptext: string &default = "Zeek Script Metric",
+                                      unit: string &default = "1",
+                                      is_sum: bool &default = F%): opaque of dbl_gauge_metric_family
+	%{
+	auto lbl_vec = sv_vec(labels->AsVectorVal());
+	auto hdl = telemetry_mgr->GaugeFamily<double>(sv(prefix), sv(name), lbl_vec,
+	                                              sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<DblGaugeMetricFamilyVal>(hdl);
+	%}
+
+function Telemetry::dbl_gauge_metric_get_or_add%(family: opaque of dbl_gauge_metric_family,
+                                                 labels: table_string_of_string%): opaque of dbl_gauge_metric
+	%{
+	using ResultType = zeek::IntrusivePtr<DblGaugeMetricFamilyVal>;
+	if ( auto ptr = dynamic_cast<zeek::DblGaugeMetricFamilyVal*>(family) )
+		{
+		auto hdl = ptr->GetHandle();
+		auto lbl_map = sv_tbl(labels->AsTableVal());
+		if ( is_valid(lbl_map, hdl.LabelNames()) )
+			{
+			auto res = hdl.GetOrAdd(lbl_map);
+			return zeek::make_intrusive<DblGaugeMetricVal>(res);
+			}
+		else
+			{
+			zeek::reporter->Error("Telemetry::dbl_gauge_metric_get_or_add: invalid label dimensions.");
+			return ResultType{nullptr};
+			}
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::dbl_gauge_metric_get_or_add: invalid handle.");
+		return ResultType{nullptr};
+		}
+	%}
+
+function Telemetry::dbl_gauge_singleton%(prefix: string,
+                                         name: string,
+                                         helptext: string &default = "Zeek Script Metric",
+                                         unit: string &default = "1",
+                                         is_sum: bool &default = F%): opaque of dbl_gauge_metric
+	%{
+	auto hdl = telemetry_mgr->GaugeSingleton<double>(sv(prefix), sv(name),
+	                                                 sv(helptext), sv(unit), is_sum);
+	return zeek::make_intrusive<DblGaugeMetricVal>(hdl);
+	%}
+
+function Telemetry::dbl_gauge_inc%(val: opaque of dbl_gauge_metric,
+                                   amount: double &default = 1.0%): bool
+	%{
+	return with<DblGaugeMetricVal>(val, "Telemetry::dbl_gauge_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); });
+	%}
+
+function Telemetry::dbl_gauge_dec%(val: opaque of dbl_gauge_metric,
+                                   amount: double &default = 1.0%): bool
+	%{
+	return with<DblGaugeMetricVal>(val, "Telemetry::dbl_gauge_dec: invalid handle.", [amount](auto hdl) { hdl.Dec(amount); });
+	%}
+
+function Telemetry::dbl_gauge_value%(val: opaque of dbl_gauge_metric%): double
+	%{
+	if ( auto ptr = dynamic_cast<zeek::DblGaugeMetricVal*>(val) )
+		{
+		return zeek::make_intrusive<DoubleVal>(ptr->GetHandle().Value());
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::dbl_gauge_value: invalid handle.");
+		return zeek::make_intrusive<DoubleVal>(0.0);
+		}
+	%}
+
+# -- IntHistogram --------------------------------------------------------------
+
+function Telemetry::int_histogram_family%(prefix: string,
+                                          name: string,
+                                          labels: string_vec,
+                                          bounds: int_vec,
+                                          helptext: string &default = "Zeek Script Metric",
+                                          unit: string &default = "1",
+                                          is_sum: bool &default = F%): opaque of int_histogram_metric_family
+	%{
+	auto lbl_vec = sv_vec(labels->AsVectorVal());
+	auto std_bounds = to_std_vec<int64_t>(bounds);
+	auto hdl = telemetry_mgr->HistogramFamily(sv(prefix), sv(name), lbl_vec,
+	                                          std_bounds, sv(helptext),
+	                                          sv(unit), is_sum);
+	return zeek::make_intrusive<IntHistogramMetricFamilyVal>(hdl);
+	%}
+
+function Telemetry::int_histogram_metric_get_or_add%(family: opaque of int_histogram_metric_family,
+                                                     labels: table_string_of_string%): opaque of int_histogram_metric
+	%{
+	using ResultType = zeek::IntrusivePtr<IntHistogramMetricFamilyVal>;
+	if ( auto ptr = dynamic_cast<zeek::IntHistogramMetricFamilyVal*>(family) )
+		{
+		auto hdl = ptr->GetHandle();
+		auto lbl_map = sv_tbl(labels->AsTableVal());
+		if ( is_valid(lbl_map, hdl.LabelNames()) )
+			{
+			auto res = hdl.GetOrAdd(lbl_map);
+			return zeek::make_intrusive<IntHistogramMetricVal>(res);
+			}
+		else
+			{
+			zeek::reporter->Error("Telemetry::int_histogram_metric_get_or_add: invalid label dimensions.");
+			return ResultType{nullptr};
+			}
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::int_histogram_metric_get_or_add: invalid handle.");
+		return ResultType{nullptr};
+		}
+	%}
+
+function Telemetry::int_histogram_singleton%(prefix: string,
+                                             name: string,
+                                             bounds: int_vec,
+                                             helptext: string &default = "Zeek Script Metric",
+                                             unit: string &default = "1",
+                                             is_sum: bool &default = F%): opaque of int_histogram_metric
+	%{
+	auto std_bounds = to_std_vec<int64_t>(bounds);
+	auto hdl = telemetry_mgr->HistogramSingleton(sv(prefix), sv(name),
+	                                             std_bounds, sv(helptext),
+	                                             sv(unit), is_sum);
+	return zeek::make_intrusive<IntHistogramMetricVal>(hdl);
+	%}
+
+function Telemetry::int_histogram_observe%(val: opaque of int_histogram_metric,
+                                           measurement: int%): bool
+	%{
+	return with<IntHistogramMetricVal>(val, "Telemetry::int_histogram_inc: invalid handle.", [measurement](auto hdl) { hdl.Observe(measurement); });
+	%}
+
+function Telemetry::int_histogram_sum%(val: opaque of int_histogram_metric%): int
+	%{
+	if ( auto ptr = dynamic_cast<zeek::IntHistogramMetricVal*>(val) )
+		{
+		return zeek::val_mgr->Int(ptr->GetHandle().Sum());
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::int_histogram_sum: invalid handle.");
+		return zeek::val_mgr->Int(0);
+		}
+	%}
+
+# -- DblHistogram --------------------------------------------------------------
+
+function Telemetry::dbl_histogram_family%(prefix: string,
+                                          name: string,
+                                          labels: string_vec,
+                                          bounds: double_vec,
+                                          helptext: string &default = "Zeek Script Metric",
+                                          unit: string &default = "1",
+                                          is_sum: bool &default = F%): opaque of dbl_histogram_metric_family
+	%{
+	auto lbl_vec = sv_vec(labels->AsVectorVal());
+	auto std_bounds = to_std_vec<double>(bounds);
+	auto hdl = telemetry_mgr->HistogramFamily<double>(sv(prefix), sv(name),
+	                                                  lbl_vec, std_bounds,
+	                                                  sv(helptext), sv(unit),
+	                                                  is_sum);
+	return zeek::make_intrusive<DblHistogramMetricFamilyVal>(hdl);
+	%}
+
+function Telemetry::dbl_histogram_metric_get_or_add%(family: opaque of dbl_histogram_metric_family,
+                                                     labels: table_string_of_string%): opaque of dbl_histogram_metric
+	%{
+	using ResultType = zeek::IntrusivePtr<DblHistogramMetricFamilyVal>;
+	if ( auto ptr = dynamic_cast<zeek::DblHistogramMetricFamilyVal*>(family) )
+		{
+		auto hdl = ptr->GetHandle();
+		auto lbl_map = sv_tbl(labels->AsTableVal());
+		if ( is_valid(lbl_map, hdl.LabelNames()) )
+			{
+			auto res = hdl.GetOrAdd(lbl_map);
+			return zeek::make_intrusive<DblHistogramMetricVal>(res);
+			}
+		else
+			{
+			zeek::reporter->Error("Telemetry::dbl_histogram_metric_get_or_add: invalid label dimensions.");
+			return ResultType{nullptr};
+			}
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::dbl_histogram_metric_get_or_add: invalid handle.");
+		return ResultType{nullptr};
+		}
+	%}
+
+function Telemetry::dbl_histogram_singleton%(prefix: string,
+                                             name: string,
+                                             bounds: double_vec,
+                                             helptext: string &default = "Zeek Script Metric",
+                                             unit: string &default = "1",
+                                             is_sum: bool &default = F%): opaque of dbl_histogram_metric
+	%{
+	auto std_bounds = to_std_vec<double>(bounds);
+	auto hdl = telemetry_mgr->HistogramSingleton<double>(sv(prefix), sv(name),
+	                                                     std_bounds, sv(helptext),
+	                                                     sv(unit), is_sum);
+	return zeek::make_intrusive<DblHistogramMetricVal>(hdl);
+	%}
+
+function Telemetry::dbl_histogram_observe%(val: opaque of dbl_histogram_metric,
+                                           measurement: double%): bool
+	%{
+	return with<DblHistogramMetricVal>(val, "Telemetry::dbl_histogram_inc: invalid handle.", [measurement](auto hdl) { hdl.Observe(measurement); });
+	%}
+
+function Telemetry::dbl_histogram_sum%(val: opaque of dbl_histogram_metric%): double
+	%{
+	if ( auto ptr = dynamic_cast<zeek::DblHistogramMetricVal*>(val) )
+		{
+		return zeek::make_intrusive<DoubleVal>(ptr->GetHandle().Sum());
+		}
+	else
+		{
+		zeek::reporter->Error("Telemetry::dbl_histogram_sum: invalid handle.");
+		return zeek::make_intrusive<DoubleVal>(0.0);
+		}
+	%}

src/zeek-setup.cc

@@ -138,6 +138,18 @@ zeek::OpaqueTypePtr bloomfilter_type;
 zeek::OpaqueTypePtr x509_opaque_type;
 zeek::OpaqueTypePtr ocsp_resp_opaque_type;
 zeek::OpaqueTypePtr paraglob_type;
+zeek::OpaqueTypePtr int_counter_metric_type;
+zeek::OpaqueTypePtr int_counter_metric_family_type;
+zeek::OpaqueTypePtr dbl_counter_metric_type;
+zeek::OpaqueTypePtr dbl_counter_metric_family_type;
+zeek::OpaqueTypePtr int_gauge_metric_type;
+zeek::OpaqueTypePtr int_gauge_metric_family_type;
+zeek::OpaqueTypePtr dbl_gauge_metric_type;
+zeek::OpaqueTypePtr dbl_gauge_metric_family_type;
+zeek::OpaqueTypePtr int_histogram_metric_type;
+zeek::OpaqueTypePtr int_histogram_metric_family_type;
+zeek::OpaqueTypePtr dbl_histogram_metric_type;
+zeek::OpaqueTypePtr dbl_histogram_metric_family_type;
 
 // Keep copy of command line
 int zeek::detail::zeek_argc;
@@ -604,6 +616,18 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	x509_opaque_type = make_intrusive<OpaqueType>("x509");
 	ocsp_resp_opaque_type = make_intrusive<OpaqueType>("ocsp_resp");
 	paraglob_type = make_intrusive<OpaqueType>("paraglob");
+	int_counter_metric_type = make_intrusive<OpaqueType>("int_counter_metric");
+	int_counter_metric_family_type = make_intrusive<OpaqueType>("int_counter_metric_family");
+	dbl_counter_metric_type = make_intrusive<OpaqueType>("dbl_counter_metric");
+	dbl_counter_metric_family_type = make_intrusive<OpaqueType>("dbl_counter_metric_family");
+	int_gauge_metric_type = make_intrusive<OpaqueType>("int_gauge_metric");
+	int_gauge_metric_family_type = make_intrusive<OpaqueType>("int_gauge_metric_family");
+	dbl_gauge_metric_type = make_intrusive<OpaqueType>("dbl_gauge_metric");
+	dbl_gauge_metric_family_type = make_intrusive<OpaqueType>("dbl_gauge_metric_family");
+	int_histogram_metric_type = make_intrusive<OpaqueType>("int_histogram_metric");
+	int_histogram_metric_family_type = make_intrusive<OpaqueType>("int_histogram_metric_family");
+	dbl_histogram_metric_type = make_intrusive<OpaqueType>("dbl_histogram_metric");
+	dbl_histogram_metric_family_type = make_intrusive<OpaqueType>("dbl_histogram_metric_family");
 
 	// The leak-checker tends to produce some false
 	// positives (memory which had already been
@@ -671,6 +695,7 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	plugin_mgr->InitPostScript();
 	zeekygen_mgr->InitPostScript();
 	broker_mgr->InitPostScript();
+	telemetry_mgr->InitPostScript();
 	timer_mgr->InitPostScript();
 	event_mgr.InitPostScript();
 

testing/btest/Baseline/telemetry.counter/output

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+cnt1_bar: 1
+cnt2_bar: 42
+cnt3_bar: 1.000000
+cnt4_bar: 42.000000

testing/btest/Baseline/telemetry.gauge/output

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+gg1_bar: 1
+gg2_bar: 23
+gg3_bar: 1.000000
+gg4_bar: 23.000000

testing/btest/Baseline/telemetry.histogram/output

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+hst1_bar: 12
+hst2_bar: 31337
+hst3_bar: 6.000000
+hst4_bar: 64.000000

testing/btest/btest.cfg

@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts coverage signatures plugins broker supervisor
+TestDirs    = doc bifs language core scripts coverage signatures plugins broker supervisor telemetry
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp

testing/btest/telemetry/counter.zeek

@@ -0,0 +1,25 @@
+# @TEST-GROUP: Telemetry
+
+# @TEST-EXEC: zeek -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+global cnt1 = Telemetry::int_counter_family("cnt1", "bar", vector("dim1", "dim2"));
+global cnt2_bar = Telemetry::int_counter_singleton("cnt2", "bar");
+global cnt3 = Telemetry::dbl_counter_family("cnt3", "bar", vector("dim1", "dim2"));
+global cnt4_bar = Telemetry::dbl_counter_singleton("cnt4", "bar");
+
+event zeek_init()
+	{
+	local cnt1_bar = Telemetry::int_counter_metric_get_or_add(cnt1, table(["dim1"] = "val1", ["dim2"] = "val2"));
+	Telemetry::int_counter_inc(cnt1_bar);
+	Telemetry::int_counter_inc(cnt2_bar);
+	Telemetry::int_counter_inc(cnt2_bar, 41);
+	print fmt("cnt1_bar: %d", Telemetry::int_counter_value(cnt1_bar));
+	print fmt("cnt2_bar: %d", Telemetry::int_counter_value(cnt2_bar));
+	local cnt3_bar = Telemetry::dbl_counter_metric_get_or_add(cnt3, table(["dim1"] = "val1", ["dim2"] = "val2"));
+	Telemetry::dbl_counter_inc(cnt3_bar);
+	Telemetry::dbl_counter_inc(cnt4_bar);
+	Telemetry::dbl_counter_inc(cnt4_bar, 41.0);
+	print fmt("cnt3_bar: %f", Telemetry::dbl_counter_value(cnt3_bar));
+	print fmt("cnt4_bar: %f", Telemetry::dbl_counter_value(cnt4_bar));
+	}

testing/btest/telemetry/gauge.zeek

@@ -0,0 +1,30 @@
+# @TEST-GROUP: Telemetry
+
+# @TEST-EXEC: zeek -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+global gg1 = Telemetry::int_gauge_family("gg1", "bar", vector("dim1", "dim2"));
+global gg2_bar = Telemetry::int_gauge_singleton("gg2", "bar");
+global gg3 = Telemetry::dbl_gauge_family("gg3", "bar", vector("dim1", "dim2"));
+global gg4_bar = Telemetry::dbl_gauge_singleton("gg4", "bar");
+
+event zeek_init()
+	{
+	local gg1_bar = Telemetry::int_gauge_metric_get_or_add(gg1, table(["dim1"] = "val1", ["dim2"] = "val2"));
+	Telemetry::int_gauge_inc(gg1_bar);
+	Telemetry::int_gauge_inc(gg2_bar);
+	Telemetry::int_gauge_inc(gg2_bar, 41);
+	Telemetry::int_gauge_dec(gg2_bar);
+	Telemetry::int_gauge_dec(gg2_bar, 18);
+	print fmt("gg1_bar: %d", Telemetry::int_gauge_value(gg1_bar));
+	print fmt("gg2_bar: %d", Telemetry::int_gauge_value(gg2_bar));
+	local gg3_bar = Telemetry::dbl_gauge_metric_get_or_add(gg3, table(["dim1"] = "val1", ["dim2"] = "val2"));
+	Telemetry::dbl_gauge_inc(gg3_bar);
+	Telemetry::dbl_gauge_inc(gg4_bar);
+	Telemetry::dbl_gauge_inc(gg4_bar, 41.0);
+	Telemetry::dbl_gauge_dec(gg4_bar);
+	Telemetry::dbl_gauge_dec(gg4_bar, 18.0);
+	print fmt("gg3_bar: %f", Telemetry::dbl_gauge_value(gg3_bar));
+	print fmt("gg4_bar: %f", Telemetry::dbl_gauge_value(gg4_bar));
+	}
+

testing/btest/telemetry/histogram.zeek

@@ -0,0 +1,28 @@
+# @TEST-GROUP: Telemetry
+
+# @TEST-EXEC: zeek -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+const int_bounds = vector(+10, +20);
+const dbl_bounds = vector(10.0, 20.0);
+
+global hst1 = Telemetry::int_histogram_family("hst1", "bar", vector("dim1", "dim2"), int_bounds);
+global hst2_bar = Telemetry::int_histogram_singleton("hst2", "bar", int_bounds);
+global hst3 = Telemetry::dbl_histogram_family("hst3", "bar", vector("dim1", "dim2"), dbl_bounds);
+global hst4_bar = Telemetry::dbl_histogram_singleton("hst4", "bar", dbl_bounds);
+
+event zeek_init()
+	{
+	local hst1_bar = Telemetry::int_histogram_metric_get_or_add(hst1, table(["dim1"] = "val1", ["dim2"] = "val2"));
+	Telemetry::int_histogram_observe(hst1_bar, 1);
+	Telemetry::int_histogram_observe(hst1_bar, 11);
+	Telemetry::int_histogram_observe(hst2_bar, 31337);
+	print fmt("hst1_bar: %d", Telemetry::int_histogram_sum(hst1_bar));
+	print fmt("hst2_bar: %d", Telemetry::int_histogram_sum(hst2_bar));
+	local hst3_bar = Telemetry::dbl_histogram_metric_get_or_add(hst3, table(["dim1"] = "val1", ["dim2"] = "val2"));
+	Telemetry::dbl_histogram_observe(hst3_bar, 2.0);
+	Telemetry::dbl_histogram_observe(hst3_bar, 4.0);
+	Telemetry::dbl_histogram_observe(hst4_bar, 64.0);
+	print fmt("hst3_bar: %f", Telemetry::dbl_histogram_sum(hst3_bar));
+	print fmt("hst4_bar: %f", Telemetry::dbl_histogram_sum(hst4_bar));
+	}