diff --git a/CHANGES b/CHANGES index 7a87054e76..6746f8fd2e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +4.1.0-dev.490 | 2021-04-05 14:13:48 -0700 + + * Add Telemetry API (C++/BIFs) for gathering runtime metrics (Dominik Charousset, Corelight) + + This relies on the CAF metrics API/implementation and allows potential + export to Prometheus. These typical metric types are supported: + counters, gauges, histograms, timers. + 4.1.0-dev.475 | 2021-04-03 09:39:10 -0700 * Increase timeout of plugins.reader btest (Jon Siwek, Corelight) diff --git a/NEWS b/NEWS index 410302b161..58d705863c 100644 --- a/NEWS +++ b/NEWS @@ -46,6 +46,9 @@ New Functionality variable or a record field to inform Zeek's analysis that the script writer asserts the value will be set, suppressing the associated warnings. +- A Telemetry API was added to assist in gathering arbitrary runtime metrics + and allows potential export to Prometheus. + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 424bab5056..46c3348727 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.1.0-dev.475 +4.1.0-dev.490 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index d57786c74a..a4292c35cb 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -46,6 +46,20 @@ type count_set: set[count]; ## directly and then remove this alias. type index_vec: vector of count; +## A vector of integers, used by telemetry builtin functions to store histogram bounds. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type int_vec: vector of int; + +## A vector of floating point numbers, used by telemetry builtin functions to store histogram bounds. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type double_vec: vector of double; + ## A vector of subnets. ## ## .. todo:: We need this type definition only for declaring builtin functions @@ -3844,7 +3858,7 @@ type dns_loc_rr: record { vert_pre: count; ##< The vertical precision of the data, in centimeters. latitude: count; ##< The latitude of the center of the sphere. longitude: count; ##< The longitude of the center of the sphere. - altitude: count; ##< The altitude of the center of the sphere. + altitude: count; ##< The altitude of the center of the sphere. is_query: count; ##< The RR is a query/Response. }; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 93d25c63ea..94cacbd5ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -159,6 +159,7 @@ set(bro_PLUGIN_DEPS CACHE INTERNAL "plugin dependencies" FORCE) add_subdirectory(analyzer) add_subdirectory(packet_analysis) add_subdirectory(broker) +add_subdirectory(telemetry) add_subdirectory(zeekygen) add_subdirectory(file_analysis) add_subdirectory(input) diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index f4439e39eb..a0d50f569d 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -1033,4 +1033,63 @@ ValPtr ParaglobVal::DoClone(CloneState* state) } } +broker::expected TelemetryVal::DoSerialize() const + { + return broker::make_error(broker::ec::invalid_data, + "cannot serialize metric handles"); + } + +bool TelemetryVal::DoUnserialize(const broker::data&) + { + return false; + } + +TelemetryVal::TelemetryVal(telemetry::IntCounter) :OpaqueVal(int_counter_metric_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::IntCounterFamily) :OpaqueVal(int_counter_metric_family_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::DblCounter) :OpaqueVal(dbl_counter_metric_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::DblCounterFamily) :OpaqueVal(dbl_counter_metric_family_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::IntGauge) :OpaqueVal(int_gauge_metric_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::IntGaugeFamily) :OpaqueVal(int_gauge_metric_family_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::DblGauge) :OpaqueVal(dbl_gauge_metric_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::DblGaugeFamily) :OpaqueVal(dbl_gauge_metric_family_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::IntHistogram) :OpaqueVal(int_histogram_metric_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::IntHistogramFamily) :OpaqueVal(int_histogram_metric_family_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::DblHistogram) :OpaqueVal(dbl_histogram_metric_type) + { + } + +TelemetryVal::TelemetryVal(telemetry::DblHistogramFamily) :OpaqueVal(dbl_histogram_metric_family_type) + { + } + } diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 1898618c25..3aef775712 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -10,6 +10,9 @@ #include "zeek/RandTest.h" #include "zeek/Val.h" #include "zeek/digest.h" +#include "zeek/telemetry/Counter.h" +#include "zeek/telemetry/Gauge.h" +#include "zeek/telemetry/Histogram.h" namespace broker { class data; } @@ -373,4 +376,66 @@ private: std::unique_ptr internal_paraglob; }; +/** + * Base class for metric handles. Handle types are not serializable. + */ +class TelemetryVal : public OpaqueVal { +protected: + explicit TelemetryVal(telemetry::IntCounter); + explicit TelemetryVal(telemetry::IntCounterFamily); + explicit TelemetryVal(telemetry::DblCounter); + explicit TelemetryVal(telemetry::DblCounterFamily); + explicit TelemetryVal(telemetry::IntGauge); + explicit TelemetryVal(telemetry::IntGaugeFamily); + explicit TelemetryVal(telemetry::DblGauge); + explicit TelemetryVal(telemetry::DblGaugeFamily); + explicit TelemetryVal(telemetry::IntHistogram); + explicit TelemetryVal(telemetry::IntHistogramFamily); + explicit TelemetryVal(telemetry::DblHistogram); + explicit TelemetryVal(telemetry::DblHistogramFamily); + + broker::expected DoSerialize() const override; + bool DoUnserialize(const broker::data& data) override; +}; + +template +class TelemetryValImpl : public TelemetryVal { +public: + using HandleType = Handle; + + explicit TelemetryValImpl(Handle hdl) : TelemetryVal(hdl), hdl(hdl) { } + + Handle GetHandle() const noexcept + { + return hdl; + } + +protected: + ValPtr DoClone(CloneState*) override + { + return make_intrusive(hdl); + } + + const char* OpaqueName() const override + { + return Handle::OpaqueName; + } + +private: + Handle hdl; +}; + +using IntCounterMetricVal = TelemetryValImpl; +using IntCounterMetricFamilyVal = TelemetryValImpl; +using DblCounterMetricVal = TelemetryValImpl; +using DblCounterMetricFamilyVal = TelemetryValImpl; +using IntGaugeMetricVal = TelemetryValImpl; +using IntGaugeMetricFamilyVal = TelemetryValImpl; +using DblGaugeMetricVal = TelemetryValImpl; +using DblGaugeMetricFamilyVal = TelemetryValImpl; +using IntHistogramMetricVal = TelemetryValImpl; +using IntHistogramMetricFamilyVal = TelemetryValImpl; +using DblHistogramMetricVal = TelemetryValImpl; +using DblHistogramMetricFamilyVal = TelemetryValImpl; + } // namespace zeek diff --git a/src/Span.h b/src/Span.h new file mode 100644 index 0000000000..86b1411a94 --- /dev/null +++ b/src/Span.h @@ -0,0 +1,218 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include + +namespace zeek { + +/** + * Drop-in replacement for C++20's @c std::span with dynamic extent only: + * https://en.cppreference.com/w/cpp/container/span. After upgrading to C++20, + * this class may get replaced with a type alias instead and/or deprecated. + */ +template +class Span { +public: + // -- member types --------------------------------------------------------- + + using element_type = T; + + using value_type = typename std::remove_cv::type; + + using index_type = size_t; + + using difference_type = ptrdiff_t; + + using pointer = T*; + + using const_pointer = const T*; + + using reference = T&; + + using const_reference = T&; + + using iterator = pointer; + + using const_iterator = const_pointer; + + using reverse_iterator = std::reverse_iterator; + + using const_reverse_iterator = std::reverse_iterator; + + // -- constructors, destructors, and assignment operators ------------------ + + constexpr Span() noexcept : memory_block(nullptr), num_elements(0) + { + } + + constexpr Span(pointer ptr, size_t size) + : memory_block(ptr), num_elements(size) + { + } + + constexpr Span(pointer first, pointer last) + : memory_block(first), num_elements(static_cast(last - first)) + { + } + + template + constexpr Span(element_type (&arr)[Size]) noexcept + : memory_block(arr), num_elements(Size) + { + } + + template >> + Span(Container& xs) noexcept + : memory_block(xs.data()), num_elements(xs.size()) + { + } + + template >> + Span(const Container& xs) noexcept + : memory_block(xs.data()), num_elements(xs.size()) + { + } + + constexpr Span(const Span&) noexcept = default; + + Span& operator=(const Span&) noexcept = default; + + // -- iterators ------------------------------------------------------------ + + constexpr iterator begin() const noexcept + { + return memory_block; + } + + constexpr const_iterator cbegin() const noexcept + { + return memory_block; + } + + constexpr iterator end() const noexcept + { + return begin() + num_elements; + } + + constexpr const_iterator cend() const noexcept + { + return cbegin() + num_elements; + } + + constexpr reverse_iterator rbegin() const noexcept + { + return reverse_iterator{end()}; + } + + constexpr const_reverse_iterator crbegin() const noexcept + { + return const_reverse_iterator{end()}; + } + + constexpr reverse_iterator rend() const noexcept + { + return reverse_iterator{begin()}; + } + + constexpr const_reverse_iterator crend() const noexcept + { + return const_reverse_iterator{begin()}; + } + + // -- element access ------------------------------------------------------- + + constexpr reference operator[](size_t index) const noexcept + { + return memory_block[index]; + } + + constexpr reference front() const noexcept + { + return *memory_block; + } + + constexpr reference back() const noexcept + { + return (*this)[num_elements - 1]; + } + + // -- properties ----------------------------------------------------------- + + constexpr size_t size() const noexcept + { + return num_elements; + } + + constexpr size_t size_bytes() const noexcept + { + return num_elements * sizeof(element_type); + } + + constexpr bool empty() const noexcept + { + return num_elements == 0; + } + + constexpr pointer data() const noexcept + { + return memory_block; + } + + // -- subviews ------------------------------------------------------------- + + constexpr Span subspan(size_t offset, size_t num_elements) const + { + return {memory_block + offset, num_elements}; + } + + constexpr Span subspan(size_t offset) const + { + return {memory_block + offset, num_elements - offset}; + } + + constexpr Span first(size_t num_elements) const + { + return {memory_block, num_elements}; + } + + constexpr Span last(size_t num_elements) const + { + return subspan(num_elements - num_elements, num_elements); + } + +private: + // -- member variables ----------------------------------------------------- + + /// Points to the first element in the contiguous memory block. + pointer memory_block; + + /// Stores the number of elements in the contiguous memory block. + size_t num_elements; +}; + +// -- deduction guides --------------------------------------------------------- + +template +Span(T*, size_t) -> Span; + +template +Span(Iter, Iter) -> Span::value_type>; + +template +Span(T (&)[N]) -> Span; + +template +Span(Container&) -> Span; + +template +Span(const Container&) -> Span; + +} // namespace zeek diff --git a/src/Type.h b/src/Type.h index 0ed8548d96..574bf5d95d 100644 --- a/src/Type.h +++ b/src/Type.h @@ -927,3 +927,15 @@ extern zeek::OpaqueTypePtr bloomfilter_type; extern zeek::OpaqueTypePtr x509_opaque_type; extern zeek::OpaqueTypePtr ocsp_resp_opaque_type; extern zeek::OpaqueTypePtr paraglob_type; +extern zeek::OpaqueTypePtr int_counter_metric_type; +extern zeek::OpaqueTypePtr int_counter_metric_family_type; +extern zeek::OpaqueTypePtr dbl_counter_metric_type; +extern zeek::OpaqueTypePtr dbl_counter_metric_family_type; +extern zeek::OpaqueTypePtr int_gauge_metric_type; +extern zeek::OpaqueTypePtr int_gauge_metric_family_type; +extern zeek::OpaqueTypePtr dbl_gauge_metric_type; +extern zeek::OpaqueTypePtr dbl_gauge_metric_family_type; +extern zeek::OpaqueTypePtr int_histogram_metric_type; +extern zeek::OpaqueTypePtr int_histogram_metric_family_type; +extern zeek::OpaqueTypePtr dbl_histogram_metric_type; +extern zeek::OpaqueTypePtr dbl_histogram_metric_family_type; diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 0ce23d9508..8128dcf8b7 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -10,6 +10,7 @@ #include "zeek/Func.h" #include "zeek/broker/Data.h" #include "zeek/broker/Store.h" +#include "zeek/telemetry/Manager.h" #include "zeek/util.h" #include "zeek/Var.h" #include "zeek/Desc.h" @@ -1806,4 +1807,53 @@ void Manager::PrepareForwarding(const std::string &name) DBG_LOG(DBG_BROKER, "Resolved table forward for data store %s", name.c_str()); } +std::unique_ptr Manager::NewTelemetryManager() + { + // The telemetry Manager actually only has a dependency on the actor system, + // not to the Broker Manager. By having the telemetry Manager hold on to a + // shared_ptr to our Broker state, we make sure the Broker endpoint, which + // owns the CAF actor system, lives for as long as necessary. This also + // makes sure that the Broker Manager may even get destroyed before the + // telemetry Manager. + struct TM final : public telemetry::Manager + { + using MetricRegistryPtr = std::unique_ptr; + + static auto getPimpl(BrokerState& st) + { + auto registry = std::addressof(st.endpoint.system().metrics()); + return reinterpret_cast(registry); + } + + static auto getPimpl(MetricRegistryPtr& ptr) + { + return reinterpret_cast(ptr.get()); + } + + explicit TM(Broker::Manager* parent, MetricRegistryPtr ptr) + : telemetry::Manager(getPimpl(ptr)), parent(parent), tmp(std::move(ptr)) + { + assert(tmp != nullptr); + assert(parent != nullptr); + } + + void InitPostScript() override + { + assert(parent->bstate != nullptr); + ptr = parent->bstate; + auto registry = std::addressof(ptr->endpoint.system().metrics()); + registry->merge(*tmp); + tmp.reset(); + pimpl = reinterpret_cast(registry); + } + + Broker::Manager* parent; + MetricRegistryPtr tmp; + std::shared_ptr ptr; + }; + + auto tmp = std::make_unique(); + return std::make_unique(this, std::move(tmp)); + } + } // namespace zeek::Broker diff --git a/src/broker/Manager.h b/src/broker/Manager.h index af09855c6b..670a9bcd79 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -30,6 +30,8 @@ class TableVal; using VectorTypePtr = IntrusivePtr; using TableValPtr = IntrusivePtr; +namespace telemetry { class Manager; } + namespace detail { class Frame; } namespace Broker { @@ -357,6 +359,11 @@ public: ~ScriptScopeGuard() { --script_scope; } }; + /** + * Allocates a new manager for telemetry data. + */ + std::unique_ptr NewTelemetryManager(); + private: void DispatchMessage(const broker::topic& topic, broker::data msg); diff --git a/src/telemetry/CMakeLists.txt b/src/telemetry/CMakeLists.txt new file mode 100644 index 0000000000..c0c2153387 --- /dev/null +++ b/src/telemetry/CMakeLists.txt @@ -0,0 +1,19 @@ +include(ZeekSubdir) + +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +set(telemetry_SRCS + Counter.cc + Gauge.cc + Histogram.cc + Manager.cc + MetricFamily.cc +) + +bif_target(telemetry.bif) + +bro_add_subdir_library(telemetry ${telemetry_SRCS}) + diff --git a/src/telemetry/Counter.cc b/src/telemetry/Counter.cc new file mode 100644 index 0000000000..b8e4684ad7 --- /dev/null +++ b/src/telemetry/Counter.cc @@ -0,0 +1,78 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/telemetry/Counter.h" + +#include "caf/telemetry/counter.hpp" +#include "caf/telemetry/metric_family.hpp" +#include "caf/telemetry/metric_family_impl.hpp" + +#include "zeek/telemetry/Detail.h" + +namespace zeek::telemetry { + +// -- IntCounter --------------------------------------------------------------- + +void IntCounter::Inc() noexcept + { + deref(pimpl).inc(); + } + +void IntCounter::Inc(int64_t amount) noexcept + { + deref(pimpl).inc(amount); + } + +int64_t IntCounter::operator++() noexcept + { + return ++deref(pimpl); + } + +int64_t IntCounter::Value() const noexcept + { + return deref(pimpl).value(); + } + +IntCounterFamily::IntCounterFamily(Impl* ptr) : MetricFamily(upcast(ptr)) + { + } + +IntCounter IntCounterFamily::GetOrAdd(Span labels) + { + return with_native_labels(labels, [this](auto nativeLabels) + { + auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); + return IntCounter{hdl}; + }); + } + +// -- DblCounter --------------------------------------------------------------- + +void DblCounter::Inc() noexcept + { + deref(pimpl).inc(); + } + +void DblCounter::Inc(double amount) noexcept + { + deref(pimpl).inc(amount); + } + +double DblCounter::Value() const noexcept + { + return deref(pimpl).value(); + } + +DblCounterFamily::DblCounterFamily(Impl* ptr) : MetricFamily(upcast(ptr)) + { + } + +DblCounter DblCounterFamily::GetOrAdd(Span labels) + { + return with_native_labels(labels, [this](auto nativeLabels) + { + auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); + return DblCounter{hdl}; + }); + } + +} // namespace zeek::telemetry diff --git a/src/telemetry/Counter.h b/src/telemetry/Counter.h new file mode 100644 index 0000000000..78be178efb --- /dev/null +++ b/src/telemetry/Counter.h @@ -0,0 +1,239 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include "zeek/Span.h" +#include "zeek/telemetry/MetricFamily.h" + +namespace zeek::telemetry { + +class DblCounterFamily; +class IntCounterFamily; +class Manager; + +/** + * A handle to a metric that represents an integer value that can only go up. + */ +class IntCounter { +public: + friend class IntCounterFamily; + + struct Impl; + + static inline const char* OpaqueName = "IntCounterMetricVal"; + + IntCounter() = delete; + IntCounter(const IntCounter&) noexcept = default; + IntCounter& operator=(const IntCounter&) noexcept = default; + + /** + * Increments the value by 1. + */ + void Inc() noexcept; + + /** + * Increments the value by @p amount. + * @pre `amount >= 0` + */ + void Inc(int64_t amount) noexcept; + + /** + * Increments the value by 1. + * @return The new value. + */ + int64_t operator++() noexcept; + + /** + * @return The current value. + */ + int64_t Value() const noexcept; + + /** + * @return Whether @c this and @p other refer to the same counter. + */ + constexpr bool IsSameAs(IntCounter other) const noexcept + { + return pimpl == other.pimpl; + } + +private: + explicit IntCounter(Impl* ptr) noexcept : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +/** + * Checks whether two @ref IntCounter handles are identical. + * @return Whether @p lhs and @p rhs refer to the same object. + * @note compare their @c value instead to check for equality. + */ +constexpr bool operator==(IntCounter lhs, IntCounter rhs) noexcept + { + return lhs.IsSameAs(rhs); + } + +/// @relates IntCounter +constexpr bool operator!=(IntCounter lhs, IntCounter rhs) noexcept + { + return !(lhs == rhs); + } + +/** + * Manages a collection of IntCounter metrics. + */ +class IntCounterFamily : public MetricFamily { +public: + friend class Manager; + + class Impl; + + static inline const char* OpaqueName = "IntCounterMetricFamilyVal"; + + using InstanceType = IntCounter; + + IntCounterFamily(const IntCounterFamily&) noexcept = default; + IntCounterFamily& operator=(const IntCounterFamily&) noexcept = default; + + /** + * Returns the metrics handle for given labels, creating a new instance + * lazily if necessary. + */ + IntCounter GetOrAdd(Span labels); + + /** + * @copydoc GetOrAdd + */ + IntCounter GetOrAdd(std::initializer_list labels) + { + return GetOrAdd(Span{labels.begin(), labels.size()}); + } + +private: + explicit IntCounterFamily(Impl* ptr); +}; + +/** + * A handle to a metric that represents a floating point value that can only go + * up. + */ +class DblCounter { +public: + friend class DblCounterFamily; + + struct Impl; + + static inline const char* OpaqueName = "DblCounterMetricVal"; + + DblCounter() = delete; + DblCounter(const DblCounter&) noexcept = default; + DblCounter& operator=(const DblCounter&) noexcept = default; + + /** + * Increments the value by 1. + */ + void Inc() noexcept; + + /** + * Increments the value by @p amount. + * @pre `amount >= 0` + */ + void Inc(double amount) noexcept; + + /** + * @return The current value. + */ + double Value() const noexcept; + + /** + * @return Whether @c this and @p other refer to the same counter. + */ + constexpr bool IsSameAs(DblCounter other) const noexcept + { + return pimpl == other.pimpl; + } + +private: + explicit DblCounter(Impl* ptr) noexcept : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +/** + * Checks whether two @ref DblCounter handles are identical. + * @return Whether @p lhs and @p rhs refer to the same object. + * @note compare their @c value instead to check for equality. + */ +constexpr bool operator==(DblCounter lhs, DblCounter rhs) noexcept + { + return lhs.IsSameAs(rhs); + } + +/// @relates DblCounter +constexpr bool operator!=(DblCounter lhs, DblCounter rhs) noexcept + { + return !(lhs == rhs); + } + +/** + * Manages a collection of DblCounter metrics. + */ +class DblCounterFamily : public MetricFamily { +public: + friend class Manager; + + class Impl; + + static inline const char* OpaqueName = "DblCounterMetricFamilyVal"; + + using InstanceType = DblCounter; + + DblCounterFamily(const DblCounterFamily&) noexcept = default; + DblCounterFamily& operator=(const DblCounterFamily&) noexcept = default; + + /** + * Returns the metrics handle for given labels, creating a new instance + * lazily if necessary. + */ + DblCounter GetOrAdd(Span labels); + + /** + * @copydoc GetOrAdd + */ + DblCounter GetOrAdd(std::initializer_list labels) + { + return GetOrAdd(Span{labels.begin(), labels.size()}); + } + +private: + explicit DblCounterFamily(Impl* ptr); +}; + +namespace detail { + +template +struct CounterOracle { + static_assert(std::is_same::value, + "Counter only supports int64_t and double"); + + using type = IntCounter; +}; + +template <> +struct CounterOracle { + using type = DblCounter; +}; + +} // namespace detail + +template +using Counter = typename detail::CounterOracle::type; + +} // namespace zeek::telemetry diff --git a/src/telemetry/Detail.h b/src/telemetry/Detail.h new file mode 100644 index 0000000000..0a235df868 --- /dev/null +++ b/src/telemetry/Detail.h @@ -0,0 +1,264 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +// This header contains private implementation details for telemetry classes +// and should not get included outside of .cc files. + +#pragma once + +#include +#include +#include + +#include "zeek/Span.h" +#include "zeek/telemetry/Counter.h" +#include "zeek/telemetry/Gauge.h" +#include "zeek/telemetry/Histogram.h" +#include "zeek/telemetry/Manager.h" +#include "zeek/telemetry/MetricFamily.h" + +#include "caf/telemetry/label_view.hpp" +#include "caf/telemetry/metric_family.hpp" + +namespace zeek::telemetry { + +// -- traits for converting between opaque handles and native pointers --------- + +/** + * This trait must provide the member types @c Native for referring to the CAF + * type, @c Opaque for referring to the @c Impl type. For instance types such as + * @c IntCounter, the trait must also provide the member type @c NativeFamily. + */ +template +struct PimplTrait; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::int_counter; + using Oqaque = IntCounter::Impl; + using NativeFamily = caf::telemetry::metric_family_impl; + }; + +template <> +struct PimplTrait +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = typename PimplTrait::NativeFamily; + using Oqaque = IntCounterFamily::Impl; + }; + +template <> +struct PimplTrait::NativeFamily> +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::dbl_counter; + using Oqaque = DblCounter::Impl; + using NativeFamily = caf::telemetry::metric_family_impl; + }; + +template <> +struct PimplTrait +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = typename PimplTrait::NativeFamily; + using Oqaque = DblCounterFamily::Impl; + }; + +template <> +struct PimplTrait::NativeFamily> +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::int_gauge; + using Oqaque = IntGauge::Impl; + using NativeFamily = caf::telemetry::metric_family_impl; + }; + +template <> +struct PimplTrait : PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = typename PimplTrait::NativeFamily; + using Oqaque = IntGaugeFamily::Impl; + }; + +template <> +struct PimplTrait::NativeFamily> +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::dbl_gauge; + using Oqaque = DblGauge::Impl; + using NativeFamily = caf::telemetry::metric_family_impl; + }; + +template <> +struct PimplTrait : PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = typename PimplTrait::NativeFamily; + using Oqaque = DblGaugeFamily::Impl; + }; + +template <> +struct PimplTrait::NativeFamily> +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::int_histogram; + using Oqaque = IntHistogram::Impl; + using NativeFamily = caf::telemetry::metric_family_impl; + }; + +template <> +struct PimplTrait +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = typename PimplTrait::NativeFamily; + using Oqaque = IntHistogramFamily::Impl; + }; + +template <> +struct PimplTrait::NativeFamily> +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::dbl_histogram; + using Oqaque = DblHistogram::Impl; + using NativeFamily = caf::telemetry::metric_family_impl; + }; + +template <> +struct PimplTrait +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = typename PimplTrait::NativeFamily; + using Oqaque = DblHistogramFamily::Impl; + }; + +template <> +struct PimplTrait::NativeFamily> +: PimplTrait { }; + +template <> +struct PimplTrait + { + using Native = caf::telemetry::metric_registry; + using Oqaque = Manager::Impl; + }; + +template <> +struct PimplTrait::Native> +: PimplTrait { }; + +// -- free functions ----------------------------------------------------------- + +template ::Native> +auto& deref(T* ptr) + { + return *reinterpret_cast(ptr); + } + +template +auto& deref(Family*, MetricFamily::Impl* ptr) + { + using InstanceType = typename Family::InstanceType; + using ImplType = typename InstanceType::Impl; + using NativeType = typename PimplTrait::NativeFamily; + return *reinterpret_cast(ptr); + } + +template ::Oqaque> +auto opaque(T* ptr) + { + return reinterpret_cast(ptr); + } + +template +auto opaque(const Family*, MetricFamily::Impl* ptr) + { + using InstanceType = typename Family::InstanceType; + using ImplType = typename InstanceType::Impl; + using OpaqueType = typename PimplTrait::NativeFamily; + return reinterpret_cast(ptr); + } + +template ::Native> +auto upcast(T* ptr) + { + auto native = reinterpret_cast(ptr); + auto base_ptr = static_cast(native); + return reinterpret_cast(base_ptr); + } + +template +auto with_native_labels(Span xs, F continuation) + { + namespace ct = caf::telemetry; + + if ( xs.size() <= 10 ) + { + ct::label_view buf[10]={ + {{},{}}, {{},{}}, {{},{}}, {{},{}}, {{},{}}, + {{},{}}, {{},{}}, {{},{}}, {{},{}}, {{},{}}, + }; + for ( size_t index = 0; index < xs.size(); ++index ) + buf[index] = ct::label_view{xs[index].first, xs[index].second}; + return continuation(Span{buf, xs.size()}); + } + else + { + std::vector buf; + for ( auto x : xs ) + buf.emplace_back(x.first, x.second); + return continuation(Span{buf}); + } + } + +template +auto with_native_labels(Span xs, F continuation) + { + if ( xs.size() <= 10 ) + { + caf::string_view buf[10]; + for ( size_t index = 0; index < xs.size(); ++index ) + buf[index] = xs[index]; + return continuation(Span{buf, xs.size()}); + } + else + { + std::vector buf; + for ( auto x : xs ) + buf.emplace_back(x); + return continuation(Span{buf}); + } + } + +} // namespace zeek::telemetry diff --git a/src/telemetry/Gauge.cc b/src/telemetry/Gauge.cc new file mode 100644 index 0000000000..2db5311cdb --- /dev/null +++ b/src/telemetry/Gauge.cc @@ -0,0 +1,103 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/telemetry/Gauge.h" + +#include "caf/telemetry/gauge.hpp" +#include "caf/telemetry/metric_family.hpp" +#include "caf/telemetry/metric_family_impl.hpp" + +#include "zeek/telemetry/Detail.h" + +namespace zeek::telemetry { + +// -- IntGauge --------------------------------------------------------------- + +void IntGauge::Inc() noexcept + { + deref(pimpl).inc(); + } + +void IntGauge::Inc(int64_t amount) noexcept + { + deref(pimpl).inc(amount); + } + +void IntGauge::Dec() noexcept + { + deref(pimpl).dec(); + } + +void IntGauge::Dec(int64_t amount) noexcept + { + deref(pimpl).dec(amount); + } + +int64_t IntGauge::operator++() noexcept + { + return ++deref(pimpl); + } + +int64_t IntGauge::operator--() noexcept + { + return --deref(pimpl); + } + +int64_t IntGauge::Value() const noexcept + { + return deref(pimpl).value(); + } + +IntGaugeFamily::IntGaugeFamily(Impl* ptr) : MetricFamily(upcast(ptr)) + { + } + +IntGauge IntGaugeFamily::GetOrAdd(Span labels) + { + return with_native_labels(labels, [this](auto nativeLabels) + { + auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); + return IntGauge{hdl}; + }); + } + +// -- DblGauge --------------------------------------------------------------- + +void DblGauge::Inc() noexcept + { + deref(pimpl).inc(); + } + +void DblGauge::Inc(double amount) noexcept + { + deref(pimpl).inc(amount); + } + +void DblGauge::Dec() noexcept + { + deref(pimpl).dec(); + } + +void DblGauge::Dec(double amount) noexcept + { + deref(pimpl).dec(amount); + } + +double DblGauge::Value() const noexcept + { + return deref(pimpl).value(); + } + +DblGaugeFamily::DblGaugeFamily(Impl* ptr) : MetricFamily(upcast(ptr)) + { + } + +DblGauge DblGaugeFamily::GetOrAdd(Span labels) + { + return with_native_labels(labels, [this](auto nativeLabels) + { + auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); + return DblGauge{hdl}; + }); + } + +} // namespace zeek::telemetry diff --git a/src/telemetry/Gauge.h b/src/telemetry/Gauge.h new file mode 100644 index 0000000000..0eda5dbf8d --- /dev/null +++ b/src/telemetry/Gauge.h @@ -0,0 +1,265 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include "zeek/Span.h" +#include "zeek/telemetry/MetricFamily.h" + +namespace zeek::telemetry { + +class DblGaugeFamily; +class IntGaugeFamily; +class Manager; + +/** + * A handle to a metric that represents an integer value. Gauges are less + * permissive than counters and also allow decrementing the value. + */ +class IntGauge { +public: + friend class IntGaugeFamily; + + struct Impl; + + static inline const char* OpaqueName = "IntGaugeMetricVal"; + + IntGauge() = delete; + IntGauge(const IntGauge&) noexcept = default; + IntGauge& operator=(const IntGauge&) noexcept = default; + + /** + * Increments the value by 1. + */ + void Inc() noexcept; + + /** + * Increments the value by @p amount. + */ + void Inc(int64_t amount) noexcept; + + /** + * Increments the value by 1. + * @return The new value. + */ + int64_t operator++() noexcept; + + /** + * Decrements the value by 1. + */ + void Dec() noexcept; + + /** + * Decrements the value by @p amount. + */ + void Dec(int64_t amount) noexcept; + + /** + * Decrements the value by 1. + * @return The new value. + */ + int64_t operator--() noexcept; + + /** + * @return The current value. + */ + int64_t Value() const noexcept; + + /** + * @return Whether @c this and @p other refer to the same counter. + */ + constexpr bool IsSameAs(IntGauge other) const noexcept + { + return pimpl == other.pimpl; + } + +private: + explicit IntGauge(Impl* ptr) noexcept : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +/** + * Checks whether two @ref IntGauge handles are identical. + * @return Whether @p lhs and @p rhs refer to the same object. + * @note compare their @c value instead to check for equality. + */ +constexpr bool operator==(IntGauge lhs, IntGauge rhs) noexcept + { + return lhs.IsSameAs(rhs); + } + +/// @relates IntGauge +constexpr bool operator!=(IntGauge lhs, IntGauge rhs) noexcept + { + return !(lhs == rhs); + } + +/** + * Manages a collection of IntGauge metrics. + */ +class IntGaugeFamily : public MetricFamily { +public: + friend class Manager; + + class Impl; + + static inline const char* OpaqueName = "IntGaugeMetricFamilyVal"; + + using InstanceType = IntGauge; + + IntGaugeFamily(const IntGaugeFamily&) noexcept = default; + IntGaugeFamily& operator=(const IntGaugeFamily&) noexcept = default; + + /** + * Returns the metrics handle for given labels, creating a new instance + * lazily if necessary. + */ + IntGauge GetOrAdd(Span labels); + + /** + * @copydoc GetOrAdd + */ + IntGauge GetOrAdd(std::initializer_list labels) + { + return GetOrAdd(Span{labels.begin(), labels.size()}); + } + +private: + explicit IntGaugeFamily(Impl* ptr); +}; + +/** + * A handle to a metric that represents a floating point value. Gauges are less + * permissive than counters and also allow decrementing the value. + * up. + */ +class DblGauge { +public: + friend class DblGaugeFamily; + + struct Impl; + + static inline const char* OpaqueName = "DblGaugeMetricVal"; + + DblGauge() = delete; + DblGauge(const DblGauge&) noexcept = default; + DblGauge& operator=(const DblGauge&) noexcept = default; + + /** + * Increments the value by 1. + */ + void Inc() noexcept; + + /** + * Increments the value by @p amount. + */ + void Inc(double amount) noexcept; + + /** + * Increments the value by 1. + */ + void Dec() noexcept; + + /** + * Increments the value by @p amount. + */ + void Dec(double amount) noexcept; + + /** + * @return The current value. + */ + double Value() const noexcept; + + /** + * @return Whether @c this and @p other refer to the same counter. + */ + constexpr bool IsSameAs(DblGauge other) const noexcept + { + return pimpl == other.pimpl; + } + +private: + explicit DblGauge(Impl* ptr) noexcept : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +/** + * Checks whether two @ref DblGauge handles are identical. + * @return Whether @p lhs and @p rhs refer to the same object. + * @note compare their @c value instead to check for equality. + */ +constexpr bool operator==(DblGauge lhs, DblGauge rhs) noexcept + { + return lhs.IsSameAs(rhs); + } + +/// @relates DblGauge +constexpr bool operator!=(DblGauge lhs, DblGauge rhs) noexcept + { + return !(lhs == rhs); + } + +/** + * Manages a collection of DblGauge metrics. + */ +class DblGaugeFamily : public MetricFamily { +public: + friend class Manager; + + class Impl; + + static inline const char* OpaqueName = "DblGaugeMetricFamilyVal"; + + using InstanceType = DblGauge; + + DblGaugeFamily(const DblGaugeFamily&) noexcept = default; + DblGaugeFamily& operator=(const DblGaugeFamily&) noexcept = default; + + /** + * Returns the metrics handle for given labels, creating a new instance + * lazily if necessary. + */ + DblGauge GetOrAdd(Span labels); + + /** + * @copydoc GetOrAdd + */ + DblGauge GetOrAdd(std::initializer_list labels) + { + return GetOrAdd(Span{labels.begin(), labels.size()}); + } + +private: + explicit DblGaugeFamily(Impl* ptr); +}; + +namespace detail { + +template +struct GaugeOracle { + static_assert(std::is_same::value, + "Gauge only supports int64_t and double"); + + using type = IntGauge; +}; + +template <> +struct GaugeOracle { + using type = DblGauge; +}; + +} // namespace detail + +template +using Gauge = typename detail::GaugeOracle::type; + +} // namespace zeek::telemetry diff --git a/src/telemetry/Histogram.cc b/src/telemetry/Histogram.cc new file mode 100644 index 0000000000..b02fe88bd1 --- /dev/null +++ b/src/telemetry/Histogram.cc @@ -0,0 +1,103 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/telemetry/Histogram.h" + +#include "caf/telemetry/histogram.hpp" +#include "caf/telemetry/metric_family.hpp" +#include "caf/telemetry/metric_family_impl.hpp" + +#include "zeek/telemetry/Detail.h" + +#include + +namespace zeek::telemetry { + +// -- IntHistogram --------------------------------------------------------------- + +void IntHistogram::Observe(int64_t value) noexcept + { + deref(pimpl).observe(value); + } + +int64_t IntHistogram::Sum() const noexcept + { + return deref(pimpl).sum(); + } + +size_t IntHistogram::NumBuckets() const noexcept + { + return deref(pimpl).buckets().size(); + } + +int64_t IntHistogram::CountAt(size_t index) const noexcept + { + auto xs = deref(pimpl).buckets(); + assert(index < xs.size()); + return xs[index].count.value(); + } + +int64_t IntHistogram::UpperBoundAt(size_t index) const noexcept + { + auto xs = deref(pimpl).buckets(); + assert(index < xs.size()); + return xs[index].upper_bound; + } + +IntHistogramFamily::IntHistogramFamily(Impl* ptr) : MetricFamily(upcast(ptr)) + { + } + +IntHistogram IntHistogramFamily::GetOrAdd(Span labels) + { + return with_native_labels(labels, [this](auto nativeLabels) + { + auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); + return IntHistogram{hdl}; + }); + } + +// -- DblHistogram --------------------------------------------------------------- + +void DblHistogram::Observe(double amount) noexcept + { + deref(pimpl).observe(amount); + } + +double DblHistogram::Sum() const noexcept + { + return deref(pimpl).sum(); + } + +size_t DblHistogram::NumBuckets() const noexcept + { + return deref(pimpl).buckets().size(); + } + +int64_t DblHistogram::CountAt(size_t index) const noexcept + { + auto xs = deref(pimpl).buckets(); + assert(index < xs.size()); + return xs[index].count.value(); + } + +double DblHistogram::UpperBoundAt(size_t index) const noexcept + { + auto xs = deref(pimpl).buckets(); + assert(index < xs.size()); + return xs[index].upper_bound; + } + +DblHistogramFamily::DblHistogramFamily(Impl* ptr) : MetricFamily(upcast(ptr)) + { + } + +DblHistogram DblHistogramFamily::GetOrAdd(Span labels) + { + return with_native_labels(labels, [this](auto nativeLabels) + { + auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); + return DblHistogram{hdl}; + }); + } + +} // namespace zeek::telemetry diff --git a/src/telemetry/Histogram.h b/src/telemetry/Histogram.h new file mode 100644 index 0000000000..68f7781c3e --- /dev/null +++ b/src/telemetry/Histogram.h @@ -0,0 +1,242 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include "zeek/Span.h" +#include "zeek/telemetry/MetricFamily.h" + +namespace zeek::telemetry { + +class DblHistogramFamily; +class IntHistogramFamily; +class Manager; + +/** + * A handle to a metric that represents an aggregatable distribution of observed + * measurements with integer precision. Sorts individual measurements into + * configurable buckets. + */ +class IntHistogram { +public: + friend class IntHistogramFamily; + + struct Impl; + + static inline const char* OpaqueName = "IntHistogramMetricVal"; + + IntHistogram() = delete; + IntHistogram(const IntHistogram&) noexcept = default; + IntHistogram& operator=(const IntHistogram&) noexcept = default; + + /** + * Increments all buckets with an upper bound less than or equal to @p value + * by one and adds @p value to the total sum of all observed values. + */ + void Observe(int64_t value) noexcept; + + /// @return The sum of all observed values. + int64_t Sum() const noexcept; + + /// @return The number of buckets, including the implicit "infinite" bucket. + size_t NumBuckets() const noexcept; + + /// @return The number of observations in the bucket at @p index. + /// @pre index < NumBuckets() + int64_t CountAt(size_t index) const noexcept; + + /// @return The upper bound of the bucket at @p index. + /// @pre index < NumBuckets() + int64_t UpperBoundAt(size_t index) const noexcept; + + /** + * @return Whether @c this and @p other refer to the same histogram. + */ + constexpr bool IsSameAs(IntHistogram other) const noexcept + { + return pimpl == other.pimpl; + } + +private: + explicit IntHistogram(Impl* ptr) noexcept : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +/** + * Checks whether two @ref IntHistogram handles are identical. + * @return Whether @p lhs and @p rhs refer to the same object. + */ +constexpr bool operator==(IntHistogram lhs, IntHistogram rhs) noexcept + { + return lhs.IsSameAs(rhs); + } + +/// @relates IntHistogram +constexpr bool operator!=(IntHistogram lhs, IntHistogram rhs) noexcept + { + return !(lhs == rhs); + } + +/** + * Manages a collection of IntHistogram metrics. + */ +class IntHistogramFamily : public MetricFamily { +public: + friend class Manager; + + class Impl; + + static inline const char* OpaqueName = "IntHistogramMetricFamilyVal"; + + using InstanceType = IntHistogram; + + IntHistogramFamily(const IntHistogramFamily&) noexcept = default; + IntHistogramFamily& operator=(const IntHistogramFamily&) noexcept = default; + + /** + * Returns the metrics handle for given labels, creating a new instance + * lazily if necessary. + */ + IntHistogram GetOrAdd(Span labels); + + /** + * @copydoc GetOrAdd + */ + IntHistogram GetOrAdd(std::initializer_list labels) + { + return GetOrAdd(Span{labels.begin(), labels.size()}); + } + +private: + explicit IntHistogramFamily(Impl* ptr); +}; + +/** + * A handle to a metric that represents an aggregatable distribution of observed + * measurements with floating point precision. Sorts individual measurements + * into configurable buckets. + */ +class DblHistogram { +public: + friend class DblHistogramFamily; + + struct Impl; + + static inline const char* OpaqueName = "DblHistogramMetricVal"; + + DblHistogram() = delete; + DblHistogram(const DblHistogram&) noexcept = default; + DblHistogram& operator=(const DblHistogram&) noexcept = default; + + /** + * Increments all buckets with an upper bound less than or equal to @p value + * by one and adds @p value to the total sum of all observed values. + */ + void Observe(double value) noexcept; + + /// @return The sum of all observed values. + double Sum() const noexcept; + + /// @return The number of buckets, including the implicit "infinite" bucket. + size_t NumBuckets() const noexcept; + + /// @return The number of observations in the bucket at @p index. + /// @pre index < NumBuckets() + int64_t CountAt(size_t index) const noexcept; + + /// @return The upper bound of the bucket at @p index. + /// @pre index < NumBuckets() + double UpperBoundAt(size_t index) const noexcept; + + /** + * @return Whether @c this and @p other refer to the same histogram. + */ + constexpr bool IsSameAs(DblHistogram other) const noexcept + { + return pimpl == other.pimpl; + } + +private: + explicit DblHistogram(Impl* ptr) noexcept : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +/** + * Checks whether two @ref DblHistogram handles are identical. + * @return Whether @p lhs and @p rhs refer to the same object. + */ +constexpr bool operator==(DblHistogram lhs, DblHistogram rhs) noexcept + { + return lhs.IsSameAs(rhs); + } + +/// @relates DblHistogram +constexpr bool operator!=(DblHistogram lhs, DblHistogram rhs) noexcept + { + return !(lhs == rhs); + } + +/** + * Manages a collection of DblHistogram metrics. + */ +class DblHistogramFamily : public MetricFamily { +public: + friend class Manager; + + class Impl; + + static inline const char* OpaqueName = "DblHistogramMetricFamilyVal"; + + using InstanceType = DblHistogram; + + DblHistogramFamily(const DblHistogramFamily&) noexcept = default; + DblHistogramFamily& operator=(const DblHistogramFamily&) noexcept = default; + + /** + * Returns the metrics handle for given labels, creating a new instance + * lazily if necessary. + */ + DblHistogram GetOrAdd(Span labels); + + /** + * @copydoc GetOrAdd + */ + DblHistogram GetOrAdd(std::initializer_list labels) + { + return GetOrAdd(Span{labels.begin(), labels.size()}); + } + +private: + explicit DblHistogramFamily(Impl* ptr); +}; + +namespace detail { + +template +struct HistogramOracle { + static_assert(std::is_same::value, + "Histogram only supports int64_t and double"); + + using type = IntHistogram; +}; + +template <> +struct HistogramOracle { + using type = DblHistogram; +}; + +} // namespace detail + +template +using Histogram = typename detail::HistogramOracle::type; + +} // namespace zeek::telemetry diff --git a/src/telemetry/Manager.cc b/src/telemetry/Manager.cc new file mode 100644 index 0000000000..5bee940a25 --- /dev/null +++ b/src/telemetry/Manager.cc @@ -0,0 +1,542 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/telemetry/Manager.h" + +#include "caf/telemetry/metric_registry.hpp" + +#include + +#include "zeek/3rdparty/doctest.h" + +#include "zeek/telemetry/Detail.h" +#include "zeek/telemetry/Timer.h" + +namespace zeek::telemetry { + +Manager::~Manager() + { + } + +void Manager::InitPostScript() + { + } + +IntCounterFamily Manager::IntCounterFam(std::string_view prefix, + std::string_view name, + Span labels, + std::string_view helptext, + std::string_view unit, bool is_sum) + { + return with_native_labels(labels, [&, this](auto xs) + { + auto ptr = deref(pimpl).counter_family(prefix, name, xs, + helptext, unit, is_sum); + return IntCounterFamily{opaque(ptr)}; + }); + } + +DblCounterFamily Manager::DblCounterFam(std::string_view prefix, + std::string_view name, + Span labels, + std::string_view helptext, + std::string_view unit, bool is_sum) + { + return with_native_labels(labels, [&, this](auto xs) + { + auto ptr = deref(pimpl).counter_family(prefix, name, xs, + helptext, unit, is_sum); + return DblCounterFamily{opaque(ptr)}; + }); + } + +IntGaugeFamily Manager::IntGaugeFam(std::string_view prefix, + std::string_view name, + Span labels, + std::string_view helptext, + std::string_view unit, bool is_sum) + { + return with_native_labels(labels, [&, this](auto xs) + { + auto ptr = deref(pimpl).gauge_family(prefix, name, xs, + helptext, unit, is_sum); + return IntGaugeFamily{opaque(ptr)}; + }); + } + +DblGaugeFamily Manager::DblGaugeFam(std::string_view prefix, + std::string_view name, + Span labels, + std::string_view helptext, + std::string_view unit, bool is_sum) + { + return with_native_labels(labels, [&, this](auto xs) + { + auto ptr = deref(pimpl).gauge_family(prefix, name, xs, + helptext, unit, is_sum); + return DblGaugeFamily{opaque(ptr)}; + }); + } + +IntHistogramFamily Manager::IntHistoFam(std::string_view prefix, + std::string_view name, + Span labels, + Span ubounds, + std::string_view helptext, + std::string_view unit, bool is_sum) + { + return with_native_labels(labels, [&, this](auto xs) + { + auto bounds = caf::span{ubounds.data(), ubounds.size()}; + auto ptr = deref(pimpl).histogram_family(prefix, name, xs, bounds, + helptext, unit, is_sum); + return IntHistogramFamily{opaque(ptr)}; + }); + } + +DblHistogramFamily Manager::DblHistoFam(std::string_view prefix, + std::string_view name, + Span labels, + Span ubounds, + std::string_view helptext, + std::string_view unit, bool is_sum) + { + return with_native_labels(labels, [&, this](auto xs) + { + auto bounds = caf::span{ubounds.data(), ubounds.size()}; + auto ptr = deref(pimpl).histogram_family(prefix, name, xs, + bounds, helptext, + unit, is_sum); + return DblHistogramFamily{opaque(ptr)}; + }); + } + +} // namespace zeek::telemetry + +// -- unit tests --------------------------------------------------------------- + +using namespace std::literals; +using namespace zeek::telemetry; + +using NativeManager = caf::telemetry::metric_registry; + +namespace { + +template +auto toVector(zeek::Span xs) + { + std::vector> result; + for ( auto&& x : xs ) + result.emplace_back(x); + return result; + } + + +} // namespace + +SCENARIO("telemetry managers provide access to counter singletons") + { + GIVEN("a telemetry manager") + { + NativeManager native_mgr; + Manager mgr{opaque(&native_mgr)}; + WHEN("retrieving an IntCounter singleton") + { + auto first = mgr.CounterSingleton("zeek", "int-count", "test"); + THEN("its initial value is zero") + { + CHECK_EQ(first.Value(), 0); + } + AND_THEN("calling Inc() or operator++ changes the value") + { + first.Inc(); + CHECK_EQ(first.Value(), 1); + first.Inc(2); + CHECK_EQ(first.Value(), 3); + CHECK_EQ(++first, 4); + CHECK_EQ(first.Value(), 4); + } + AND_THEN("calling counterSingleton again for the same name returns the same handle") + { + auto second = mgr.CounterSingleton("zeek", "int-count", "test"); + CHECK_EQ(first, second); + } + AND_THEN("calling counterSingleton for a different name returns another handle") + { + auto third = mgr.CounterSingleton("zeek", "int-count-2", "test"); + CHECK_NE(first, third); + } + } + WHEN("retrieving a DblCounter singleton") + { + auto first = mgr.CounterSingleton("zeek", "dbl-count", "test"); + THEN("its initial value is zero") + { + CHECK_EQ(first.Value(), 0.0); + } + AND_THEN("calling Inc() changes the value") + { + first.Inc(); + CHECK_EQ(first.Value(), 1.0); + first.Inc(3.0); + CHECK_EQ(first.Value(), 4.0); + } + AND_THEN("calling counterSingleton again for the same name returns the same handle") + { + auto second = mgr.CounterSingleton("zeek", "dbl-count", "test"); + CHECK_EQ(first, second); + } + AND_THEN("calling counterSingleton for a different name returns another handle") + { + auto third = mgr.CounterSingleton("zeek", "dbl-count-2", "test"); + CHECK_NE(first, third); + } + } + } + } + +SCENARIO("telemetry managers provide access to counter families") + { + GIVEN("a telemetry manager") + { + NativeManager native_mgr; + Manager mgr{opaque(&native_mgr)}; + WHEN("retrieving an IntCounter family") + { + auto family = mgr.CounterFamily("zeek", "requests", {"method"}, "test", "1", true); + THEN("the family object stores the parameters") + { + CHECK_EQ(family.Prefix(), "zeek"sv); + CHECK_EQ(family.Name(), "requests"sv); + CHECK_EQ(toVector(family.LabelNames()), std::vector{"method"s}); + CHECK_EQ(family.Helptext(), "test"sv); + CHECK_EQ(family.Unit(), "1"sv); + CHECK_EQ(family.IsSum(), true); + } + AND_THEN("GetOrAdd returns the same metric for the same labels") + { + auto first = family.GetOrAdd({{"method", "get"}}); + auto second = family.GetOrAdd({{"method", "get"}}); + CHECK_EQ(first, second); + } + AND_THEN("GetOrAdd returns different metric for the disjoint labels") + { + auto first = family.GetOrAdd({{"method", "get"}}); + auto second = family.GetOrAdd({{"method", "put"}}); + CHECK_NE(first, second); + } + } + WHEN("retrieving a DblCounter family") + { + auto family = mgr.CounterFamily("zeek", "runtime", {"query"}, "test", "seconds", true); + THEN("the family object stores the parameters") + { + CHECK_EQ(family.Prefix(), "zeek"sv); + CHECK_EQ(family.Name(), "runtime"sv); + CHECK_EQ(toVector(family.LabelNames()), std::vector{"query"s}); + CHECK_EQ(family.Helptext(), "test"sv); + CHECK_EQ(family.Unit(), "seconds"sv); + CHECK_EQ(family.IsSum(), true); + } + AND_THEN("GetOrAdd returns the same metric for the same labels") + { + auto first = family.GetOrAdd({{"query", "foo"}}); + auto second = family.GetOrAdd({{"query", "foo"}}); + CHECK_EQ(first, second); + } + AND_THEN("GetOrAdd returns different metric for the disjoint labels") + { + auto first = family.GetOrAdd({{"query", "foo"}}); + auto second = family.GetOrAdd({{"query", "bar"}}); + CHECK_NE(first, second); + } + } + } + } + +SCENARIO("telemetry managers provide access to gauge singletons") + { + GIVEN("a telemetry manager") + { + NativeManager native_mgr; + Manager mgr{opaque(&native_mgr)}; + WHEN("retrieving an IntGauge singleton") + { + auto first = mgr.GaugeSingleton("zeek", "int-gauge", "test"); + THEN("its initial value is zero") + { + CHECK_EQ(first.Value(), 0); + } + AND_THEN("calling Inc(), Dec(), operator++ or operator-- changes the value") + { + first.Inc(); + CHECK_EQ(first.Value(), 1); + first.Inc(2); + CHECK_EQ(first.Value(), 3); + first.Dec(); + CHECK_EQ(first.Value(), 2); + CHECK_EQ(++first, 3); + CHECK_EQ(first.Value(), 3); + CHECK_EQ(--first, 2); + CHECK_EQ(first.Value(), 2); + first.Dec(2); + CHECK_EQ(first.Value(), 0); + } + AND_THEN("calling gaugeSingleton again for the same name returns the same handle") + { + auto second = mgr.GaugeSingleton("zeek", "int-gauge", "test"); + CHECK_EQ(first, second); + } + AND_THEN("calling gaugeSingleton for a different name returns another handle") + { + auto third = mgr.GaugeSingleton("zeek", "int-gauge-2", "test"); + CHECK_NE(first, third); + } + } + WHEN("retrieving a DblGauge singleton") + { + auto first = mgr.GaugeSingleton("zeek", "dbl-gauge", "test"); + THEN("its initial value is zero") + { + CHECK_EQ(first.Value(), 0.0); + } + AND_THEN("calling Inc() or Dec() changes the value") + { + first.Inc(); + CHECK_EQ(first.Value(), 1.0); + first.Inc(3.0); + CHECK_EQ(first.Value(), 4.0); + first.Dec(2.0); + CHECK_EQ(first.Value(), 2.0); + first.Dec(); + CHECK_EQ(first.Value(), 1.0); + } + AND_THEN("calling gaugeSingleton again for the same name returns the same handle") + { + auto second = mgr.GaugeSingleton("zeek", "dbl-gauge", "test"); + CHECK_EQ(first, second); + } + AND_THEN("calling gaugeSingleton for a different name returns another handle") + { + auto third = mgr.GaugeSingleton("zeek", "dbl-gauge-2", "test"); + CHECK_NE(first, third); + } + } + } + } + +SCENARIO("telemetry managers provide access to gauge families") + { + GIVEN("a telemetry manager") + { + NativeManager native_mgr; + Manager mgr{opaque(&native_mgr)}; + WHEN("retrieving an IntGauge family") + { + auto family = mgr.GaugeFamily("zeek", "open-connections", {"protocol"}, "test"); + THEN("the family object stores the parameters") + { + CHECK_EQ(family.Prefix(), "zeek"sv); + CHECK_EQ(family.Name(), "open-connections"sv); + CHECK_EQ(toVector(family.LabelNames()), std::vector{"protocol"s}); + CHECK_EQ(family.Helptext(), "test"sv); + CHECK_EQ(family.Unit(), "1"sv); + CHECK_EQ(family.IsSum(), false); + } + AND_THEN("GetOrAdd returns the same metric for the same labels") + { + auto first = family.GetOrAdd({{"protocol", "tcp"}}); + auto second = family.GetOrAdd({{"protocol", "tcp"}}); + CHECK_EQ(first, second); + } + AND_THEN("GetOrAdd returns different metric for the disjoint labels") + { + auto first = family.GetOrAdd({{"protocol", "tcp"}}); + auto second = family.GetOrAdd({{"protocol", "quic"}}); + CHECK_NE(first, second); + } + } + WHEN("retrieving a DblGauge family") + { + auto family = mgr.GaugeFamily("zeek", "water-level", {"river"}, "test", "meters"); + THEN("the family object stores the parameters") + { + CHECK_EQ(family.Prefix(), "zeek"sv); + CHECK_EQ(family.Name(), "water-level"sv); + CHECK_EQ(toVector(family.LabelNames()), std::vector{"river"s}); + CHECK_EQ(family.Helptext(), "test"sv); + CHECK_EQ(family.Unit(), "meters"sv); + CHECK_EQ(family.IsSum(), false); + } + AND_THEN("GetOrAdd returns the same metric for the same labels") + { + auto first = family.GetOrAdd({{"river", "Sacramento"}}); + auto second = family.GetOrAdd({{"river", "Sacramento"}}); + CHECK_EQ(first, second); + } + AND_THEN("GetOrAdd returns different metric for the disjoint labels") + { + auto first = family.GetOrAdd({{"query", "Sacramento"}}); + auto second = family.GetOrAdd({{"query", "San Joaquin"}}); + CHECK_NE(first, second); + } + } + } + } + +SCENARIO("telemetry managers provide access to histogram singletons") + { + GIVEN("a telemetry manager") + { + NativeManager native_mgr; + Manager mgr{opaque(&native_mgr)}; + WHEN("retrieving an IntHistogram singleton") + { + const auto max_int = std::numeric_limits::max(); + int64_t buckets [] = { 10, 20 }; + auto first = mgr.HistogramSingleton("zeek", "int-hist", buckets, "test"); + THEN("it initially has no observations") + { + REQUIRE_EQ(first.NumBuckets(), 3u); + CHECK_EQ(first.Sum(), 0); + CHECK_EQ(first.CountAt(0), 0); + CHECK_EQ(first.CountAt(1), 0); + CHECK_EQ(first.CountAt(2), 0); + CHECK_EQ(first.UpperBoundAt(0), 10); + CHECK_EQ(first.UpperBoundAt(1), 20); + CHECK_EQ(first.UpperBoundAt(2), max_int); + } + AND_THEN("calling Observe() increments bucket counters") + { + first.Observe(1); + first.Observe(9); + first.Observe(10); + first.Observe(11); + first.Observe(19); + first.Observe(20); + first.Observe(21); + CHECK_EQ(first.Sum(), 91); + CHECK_EQ(first.CountAt(0), 3); + CHECK_EQ(first.CountAt(1), 3); + CHECK_EQ(first.CountAt(2), 1); + } + AND_THEN("calling HistogramSingleton again for the same name returns the same handle") + { + auto second = mgr.HistogramSingleton("zeek", "int-hist", buckets, "test"); + CHECK_EQ(first, second); + } + AND_THEN("calling HistogramSingleton for a different name returns another handle") + { + auto third = mgr.HistogramSingleton("zeek", "int-hist-2", buckets, "test"); + CHECK_NE(first, third); + } + } + WHEN("retrieving a DblHistogram singleton") + { + double buckets [] = { 10.0, 20.0 }; + auto first = mgr.HistogramSingleton("zeek", "dbl-count", buckets, "test"); + THEN("it initially has no observations") + { + REQUIRE_EQ(first.NumBuckets(), 3u); + CHECK_EQ(first.Sum(), 0.0); + CHECK_EQ(first.CountAt(0), 0); + CHECK_EQ(first.CountAt(1), 0); + CHECK_EQ(first.CountAt(2), 0); + CHECK_EQ(first.UpperBoundAt(0), 10.0); + CHECK_EQ(first.UpperBoundAt(1), 20.0); + } + AND_THEN("calling Observe() increments bucket counters") + { + first.Observe(2.0); + first.Observe(4.0); + first.Observe(8.0); + first.Observe(16.0); + first.Observe(32.0); + CHECK_EQ(first.Sum(), 62.0); + CHECK_EQ(first.CountAt(0), 3); + CHECK_EQ(first.CountAt(1), 1); + CHECK_EQ(first.CountAt(2), 1); + } + AND_THEN("calling histogramSingleton again for the same name returns the same handle") + { + auto second = mgr.HistogramSingleton("zeek", "dbl-count", buckets, "test"); + CHECK_EQ(first, second); + } + AND_THEN("calling histogramSingleton for a different name returns another handle") + { + auto third = mgr.HistogramSingleton("zeek", "dbl-count-2", buckets, "test"); + CHECK_NE(first, third); + } + } + } + } + +SCENARIO("telemetry managers provide access to histogram families") + { + GIVEN("a telemetry manager") + { + NativeManager native_mgr; + Manager mgr{opaque(&native_mgr)}; + WHEN("retrieving an IntHistogram family") + { + int64_t buckets [] = { 10, 20 }; + auto family = mgr.HistogramFamily("zeek", "payload-size", {"protocol"}, buckets, "test", "bytes"); + THEN("the family object stores the parameters") + { + CHECK_EQ(family.Prefix(), "zeek"sv); + CHECK_EQ(family.Name(), "payload-size"sv); + CHECK_EQ(toVector(family.LabelNames()), std::vector{"protocol"s}); + CHECK_EQ(family.Helptext(), "test"sv); + CHECK_EQ(family.Unit(), "bytes"sv); + CHECK_EQ(family.IsSum(), false); + } + AND_THEN("GetOrAdd returns the same metric for the same labels") + { + auto first = family.GetOrAdd({{"protocol", "tcp"}}); + auto second = family.GetOrAdd({{"protocol", "tcp"}}); + CHECK_EQ(first, second); + } + AND_THEN("GetOrAdd returns different metric for the disjoint labels") + { + auto first = family.GetOrAdd({{"protocol", "tcp"}}); + auto second = family.GetOrAdd({{"protocol", "udp"}}); + CHECK_NE(first, second); + } + } + WHEN("retrieving a DblHistogram family") + { + double buckets [] = { 10.0, 20.0 }; + auto family = mgr.HistogramFamily("zeek", "parse-time", {"protocol"}, buckets, "test", "seconds"); + THEN("the family object stores the parameters") + { + CHECK_EQ(family.Prefix(), "zeek"sv); + CHECK_EQ(family.Name(), "parse-time"sv); + CHECK_EQ(toVector(family.LabelNames()), std::vector{"protocol"s}); + CHECK_EQ(family.Helptext(), "test"sv); + CHECK_EQ(family.Unit(), "seconds"sv); + CHECK_EQ(family.IsSum(), false); + } + AND_THEN("GetOrAdd returns the same metric for the same labels") + { + auto first = family.GetOrAdd({{"protocol", "tcp"}}); + auto second = family.GetOrAdd({{"protocol", "tcp"}}); + CHECK_EQ(first, second); + } + AND_THEN("GetOrAdd returns different metric for the disjoint labels") + { + auto first = family.GetOrAdd({{"protocol", "tcp"}}); + auto second = family.GetOrAdd({{"protocol", "udp"}}); + CHECK_NE(first, second); + } + AND_THEN("Timers add observations to histograms") + { + auto hg = family.GetOrAdd({{"protocol", "tst"}}); + CHECK_EQ(hg.Sum(), 0.0); + { + Timer observer{hg}; + std::this_thread::sleep_for(1ms); + } + CHECK_NE(hg.Sum(), 0.0); + } + } + } + } diff --git a/src/telemetry/Manager.h b/src/telemetry/Manager.h new file mode 100644 index 0000000000..9fa478e845 --- /dev/null +++ b/src/telemetry/Manager.h @@ -0,0 +1,455 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include +#include + +#include "zeek/Span.h" +#include "zeek/telemetry/Counter.h" +#include "zeek/telemetry/Gauge.h" +#include "zeek/telemetry/Histogram.h" + +namespace zeek::telemetry { + +/** + * Manages a collection of metric families. + */ +class Manager { +public: + class Impl; + + explicit Manager(Impl* ptr) : pimpl(ptr) + { + } + + Manager(const Manager&) = delete; + + Manager& operator=(const Manager&) = delete; + + virtual ~Manager(); + + /** + * Initialization of the manager. This is called late during Bro's + * initialization after any scripts are processed. + */ + virtual void InitPostScript(); + + /** + * @return A counter metric family. Creates the family lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param labels Names for all label dimensions of the metric. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. + * @param is_sum Indicates whether this metric accumulates something, where + * only the total value is of interest. + */ + template + auto CounterFamily(std::string_view prefix, std::string_view name, + Span labels, + std::string_view helptext, + std::string_view unit = "1", bool is_sum = false) { + if constexpr (std::is_same::value) + { + return IntCounterFam(prefix, name, labels, helptext, unit, is_sum); + } + else + { + static_assert(std::is_same::value, + "metrics only support int64_t and double values"); + return DblCounterFam(prefix, name, labels, helptext, unit, is_sum); + } + } + + /// @copydoc CounterFamily + template + auto CounterFamily(std::string_view prefix, std::string_view name, + std::initializer_list labels, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbl_span = Span{labels.begin(), labels.size()}; + return CounterFamily(prefix, name, lbl_span, helptext, + unit, is_sum); + } + + /** + * Accesses a counter instance. Creates the hosting metric family as well + * as the counter lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param labels Values for all label dimensions of the metric. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. + * @param is_sum Indicates whether this metric accumulates something, where + * only the total value is of interest. + */ + template + Counter + CounterInstance(std::string_view prefix, std::string_view name, + Span labels, std::string_view helptext, + std::string_view unit = "1", bool is_sum = false) + { + return WithLabelNames(labels, [&, this](auto labelNames) + { + auto family = CounterFamily(prefix, name, labelNames, + helptext, unit, is_sum); + return family.getOrAdd(labels); + }); + } + + /// @copydoc counterInstance + template + Counter + CounterInstance(std::string_view prefix, std::string_view name, + std::initializer_list labels, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbl_span = Span{labels.begin(), labels.size()}; + return CounterInstance(prefix, name, lbl_span, helptext, unit, is_sum); + } + + /** + * Accesses a counter singleton, i.e., a counter that belongs to a family + * without label dimensions (which thus only has a single member). Creates + * the hosting metric family as well as the counter lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. + * @param is_sum Indicates whether this metric accumulates something, where + * only the total value is of interest. + */ + template + Counter + CounterSingleton(std::string_view prefix, std::string_view name, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto labels = Span{}; + auto fam = CounterFamily(prefix, name, labels, helptext, + unit, is_sum); + return fam.GetOrAdd({}); + } + + /** + * @return A gauge metric family. Creates the family lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param labels Names for all label dimensions of the metric. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. + * @param is_sum Indicates whether this metric accumulates something, where + * only the total value is of interest. + */ + template + auto GaugeFamily(std::string_view prefix, std::string_view name, + Span labels, + std::string_view helptext, + std::string_view unit = "1", bool is_sum = false) + { + if constexpr (std::is_same::value) + { + return IntGaugeFam(prefix, name, labels, helptext, unit, is_sum); + } + else + { + static_assert(std::is_same::value, + "metrics only support int64_t and double values"); + return DblGaugeFam(prefix, name, labels, helptext, unit, is_sum); + } + } + + /// @copydoc GaugeFamily + template + auto GaugeFamily(std::string_view prefix, std::string_view name, + std::initializer_list labels, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbl_span = Span{labels.begin(), labels.size()}; + return GaugeFamily(prefix, name, lbl_span, helptext, + unit, is_sum); + } + + /** + * Accesses a gauge instance. Creates the hosting metric family as well + * as the gauge lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param labels Values for all label dimensions of the metric. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. + * @param is_sum Indicates whether this metric accumulates something, where + * only the total value is of interest. + */ + template + Gauge + GaugeInstance(std::string_view prefix, std::string_view name, + Span labels, std::string_view helptext, + std::string_view unit = "1", bool is_sum = false) + { + return WithLabelNames(labels, [&, this](auto labelNames) + { + auto family = GaugeFamily(prefix, name, labelNames, + helptext, unit, is_sum); + return family.getOrAdd(labels); + }); + } + + /// @copydoc GaugeInstance + template + Gauge + GaugeInstance(std::string_view prefix, std::string_view name, + std::initializer_list labels, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbl_span = Span{labels.begin(), labels.size()}; + return GaugeInstance(prefix, name, lbl_span, helptext, unit, is_sum); + } + + /** + * Accesses a gauge singleton, i.e., a gauge that belongs to a family + * without label dimensions (which thus only has a single member). Creates + * the hosting metric family as well as the gauge lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. + * @param is_sum Indicates whether this metric accumulates something, where + * only the total value is of interest. + */ + template + Gauge + GaugeSingleton(std::string_view prefix, std::string_view name, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto labels = Span{}; + auto fam = GaugeFamily(prefix, name, labels, helptext, + unit, is_sum); + return fam.GetOrAdd({}); + } + + // Forces the compiler to use the type `Span` instead of trying to + // match paremeters to a `span`. + template + struct ConstSpanOracle + { + using Type = Span; + }; + + // Convenience alias to safe some typing. + template + using ConstSpan = typename ConstSpanOracle::Type; + + /** + * Returns a histogram metric family. Creates the family lazily if + * necessary. + * @param prefix The prefix (namespace) this family belongs to. Usually the + * application or protocol name, e.g., `http`. The prefix `caf` + * as well as prefixes starting with an underscore are + * reserved. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param labels Names for all label dimensions of the metric. + * @param default_upper_bounds Upper bounds for the metric buckets. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. Please use base units such as `bytes` or + * `seconds` (prefer lowercase). The pseudo-unit `1` identifies + * dimensionless counts. + * @param is_sum Setting this to `true` indicates that this metric adds + * something up to a total, where only the total value is of + * interest. For example, the total number of HTTP requests. + * @note The first call wins when calling this function multiple times with + * different bucket settings. Users may also override + * @p default_upper_bounds via run-time configuration. + */ + template + auto HistogramFamily(std::string_view prefix, std::string_view name, + Span labels, + ConstSpan default_upper_bounds, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + if constexpr (std::is_same::value) + { + return IntHistoFam(prefix, name, labels, default_upper_bounds, + helptext, unit, is_sum); + } + else + { + static_assert(std::is_same::value, + "metrics only support int64_t and double values"); + return DblHistoFam(prefix, name, labels, default_upper_bounds, + helptext, unit, is_sum); + } + } + + /// @copydoc HistogramFamily + template + auto HistogramFamily(std::string_view prefix, std::string_view name, + std::initializer_list labels, + ConstSpan default_upper_bounds, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbl_span = Span{labels.begin(), labels.size()}; + return HistogramFamily(prefix, name, lbl_span, + default_upper_bounds, helptext, + unit, is_sum); + } + + /** + * Returns a histogram. Creates the family lazily if necessary. + * @param prefix The prefix (namespace) this family belongs to. Usually the + * application or protocol name, e.g., `http`. The prefix `caf` + * as well as prefixes starting with an underscore are + * reserved. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param labels Names for all label dimensions of the metric. + * @param default_upper_bounds Upper bounds for the metric buckets. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. Please use base units such as `bytes` or + * `seconds` (prefer lowercase). The pseudo-unit `1` identifies + * dimensionless counts. + * @param is_sum Setting this to `true` indicates that this metric adds + * something up to a total, where only the total value is of + * interest. For example, the total number of HTTP requests. + * @note The first call wins when calling this function multiple times with + * different bucket settings. Users may also override + * @p default_upper_bounds via run-time configuration. + */ + template + Histogram + HistogramInstance(std::string_view prefix, std::string_view name, + Span labels, + ConstSpan default_upper_bounds, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + return WithLabelNames(labels, [&, this](auto labelNames) + { + auto family = HistogramFamily(prefix, name, labelNames, + default_upper_bounds, + helptext, unit, is_sum); + return family.getOrAdd(labels); + }); + } + + /// @copdoc HistogramInstance + template + Histogram + HistogramInstance(std::string_view prefix, std::string_view name, + std::initializer_list labels, + ConstSpan default_upper_bounds, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbls = Span{labels.begin(), labels.size()}; + return HistogramInstance(prefix, name, lbls, default_upper_bounds, + helptext, unit, is_sum); + } + + /** + * Returns a histogram metric singleton, i.e., the single instance of a + * family without label dimensions. Creates all objects lazily if necessary, + * but fails if the full name already belongs to a different family. + * @param prefix The prefix (namespace) this family belongs to. Usually the + * application or protocol name, e.g., `http`. The prefix `caf` + * as well as prefixes starting with an underscore are + * reserved. + * @param name The human-readable name of the metric, e.g., `requests`. + * @param default_upper_bounds Upper bounds for the metric buckets. + * @param helptext Short explanation of the metric. + * @param unit Unit of measurement. Please use base units such as `bytes` or + * `seconds` (prefer lowercase). The pseudo-unit `1` identifies + * dimensionless counts. + * @param is_sum Setting this to `true` indicates that this metric adds + * something up to a total, where only the total value is of + * interest. For example, the total number of HTTP requests. + * @note The first call wins when calling this function multiple times with + * different bucket settings. Users may also override + * @p default_upper_bounds via run-time configuration. + */ + template + Histogram + HistogramSingleton(std::string_view prefix, std::string_view name, + ConstSpan default_upper_bounds, + std::string_view helptext, std::string_view unit = "1", + bool is_sum = false) + { + auto lbls = Span{}; + auto fam = HistogramFamily(prefix, name, lbls, + default_upper_bounds, helptext, + unit, is_sum); + return fam.GetOrAdd({}); + } + +protected: + IntCounterFamily + IntCounterFam(std::string_view prefix, std::string_view name, + Span labels, + std::string_view helptext, std::string_view unit, + bool is_sum); + + DblCounterFamily + DblCounterFam(std::string_view prefix, std::string_view name, + Span labels, + std::string_view helptext, std::string_view unit, + bool is_sum); + + IntGaugeFamily + IntGaugeFam(std::string_view prefix, std::string_view name, + Span labels, + std::string_view helptext, std::string_view unit, + bool is_sum); + + DblGaugeFamily + DblGaugeFam(std::string_view prefix, std::string_view name, + Span labels, + std::string_view helptext, std::string_view unit, + bool is_sum); + + IntHistogramFamily + IntHistoFam(std::string_view prefix, std::string_view name, + Span labels, + Span ubounds, std::string_view helptext, + std::string_view unit, bool is_sum); + + DblHistogramFamily + DblHistoFam(std::string_view prefix, std::string_view name, + Span labels, + Span ubounds, std::string_view helptext, + std::string_view unit, bool is_sum); + + template + static void WithLabelNames(Span xs, F continuation) + { + if ( xs.size() <= 10 ) { + std::string_view buf[10]; + for ( size_t index = 0; index < xs.size(); ++index ) + buf[index] = xs[index].first; + return continuation(Span{buf, xs.size()}); + } else { + std::vector buf; + for ( auto x : xs ) + buf.emplace_back(x.first, x.second); + return continuation(Span{buf}); + } + } + + Impl* pimpl; +}; + +} // namespace zeek::telemetry + +namespace zeek { + +extern telemetry::Manager* telemetry_mgr; + +} // namespace zeek diff --git a/src/telemetry/MetricFamily.cc b/src/telemetry/MetricFamily.cc new file mode 100644 index 0000000000..f3660fbdbc --- /dev/null +++ b/src/telemetry/MetricFamily.cc @@ -0,0 +1,52 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "zeek/telemetry/MetricFamily.h" + +#include "caf/telemetry/metric_family.hpp" + +namespace zeek::telemetry { + +namespace { + +namespace ct = caf::telemetry; + +using NativeMetricFamily = ct::metric_family; + +auto& deref(MetricFamily::Impl* ptr) + { + return *reinterpret_cast(ptr); + } + +} // namespace + +std::string_view MetricFamily::Prefix() const noexcept + { + return deref(pimpl).prefix(); + } + +std::string_view MetricFamily::Name() const noexcept + { + return deref(pimpl).name(); + } + +Span MetricFamily::LabelNames() const noexcept + { + return deref(pimpl).label_names(); + } + +std::string_view MetricFamily::Helptext() const noexcept + { + return deref(pimpl).helptext(); + } + +std::string_view MetricFamily::Unit() const noexcept + { + return deref(pimpl).unit(); + } + +bool MetricFamily::IsSum() const noexcept + { + return deref(pimpl).is_sum(); + } + +} // namespace zeek::telemetry diff --git a/src/telemetry/MetricFamily.h b/src/telemetry/MetricFamily.h new file mode 100644 index 0000000000..50e59b22e4 --- /dev/null +++ b/src/telemetry/MetricFamily.h @@ -0,0 +1,76 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include +#include +#include + +#include "Span.h" + +namespace zeek::telemetry { + +/** + * A key-value pair for a single label dimension. + */ +using LabelView = std::pair; + +/** + * Manages a collection (family) of metrics. All members of the family share + * the same prefix (namespace), name, and label dimensions. + */ +class MetricFamily { +public: + struct Impl; + + MetricFamily() = delete; + MetricFamily(const MetricFamily&) noexcept = default; + MetricFamily& operator=(const MetricFamily&) noexcept = default; + + /** + * @return The prefix (namespace) this family belongs to. Builtin metrics + * of Zeek return @c zeek. Custom metrics, e.g., created in a + * script, may use a prefix that represents the application/script + * or protocol (e.g. @c http) name. + */ + std::string_view Prefix() const noexcept; + + /** + * @return The human-readable name of the metric, e.g., + * @p open-connections. + */ + std::string_view Name() const noexcept; + + /** + * @return The names for all label dimensions. + */ + Span LabelNames() const noexcept; + + /** + * @return A short explanation of the metric. + */ + std::string_view Helptext() const noexcept; + + /** + * @return The unit of measurement, preferably a base unit such as + * @c bytes or @c seconds. Dimensionless counts return the + * pseudo-unit @c 1. + */ + std::string_view Unit() const noexcept; + + /** + * @return Whether metrics of this family accumulate values, where only the + * total value is of interest. For example, the total number of + * HTTP requests. + */ + bool IsSum() const noexcept; + +protected: + explicit MetricFamily(Impl* ptr) : pimpl(ptr) + { + } + + Impl* pimpl; +}; + +} // namespace zeek::telemetry diff --git a/src/telemetry/Timer.h b/src/telemetry/Timer.h new file mode 100644 index 0000000000..f711026024 --- /dev/null +++ b/src/telemetry/Timer.h @@ -0,0 +1,57 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#pragma once + +#include + +#include "zeek/telemetry/Histogram.h" + +namespace zeek::telemetry { + +/// Convenience helper for measuring durations such as latency using a histogram +/// with second resolution. The measurement starts when creating the objects and +/// finishes when the Timer goes out of scope. +class [[nodiscard]] Timer { +public: + using Clock = std::chrono::steady_clock; + + explicit Timer(DblHistogram h) : h_(h) + { + start_ = Clock::now(); + } + + Timer(const Timer&) = delete; + + Timer&operator=(const Timer&) = delete; + + ~Timer() + { + Observe(h_, start_); + } + + /// @return The histogram handle. + auto Handle() const noexcept + { + return h_; + } + + /// @return The recorded start time. + auto Started() const noexcept + { + return start_; + } + + /// Calls `h.Observe` with the time passed since `start`. + static void Observe(DblHistogram h, Clock::time_point start) + { + using DblSec = std::chrono::duration; + if ( auto end = Clock::now(); end > start ) + h.Observe(std::chrono::duration_cast(end - start).count()); + } + +private: + DblHistogram h_; + Clock::time_point start_; +}; + +} // namespace zeek::telemetry diff --git a/src/telemetry/telemetry.bif b/src/telemetry/telemetry.bif new file mode 100644 index 0000000000..3c4d6d2d1b --- /dev/null +++ b/src/telemetry/telemetry.bif @@ -0,0 +1,550 @@ + +##! Functions for accessing counter metrics from script land. + +module Telemetry; + +%%{ +#include "zeek/telemetry/Counter.h" +#include "zeek/telemetry/Gauge.h" +#include "zeek/telemetry/Histogram.h" +#include "zeek/telemetry/Manager.h" + +namespace { + +template +auto with(zeek::Val* val, const char* error_msg, Fun fun) + { + if ( auto ptr = dynamic_cast(val) ) + { + fun(ptr->GetHandle()); + return zeek::val_mgr->True(); + } + else + { + zeek::reporter->Error("%s", error_msg); + return zeek::val_mgr->False(); + } + } + +std::string_view sv(const zeek::String* str) noexcept + { + auto len = static_cast(str->Len()); + return {reinterpret_cast(str->Bytes()), len}; + }; + +std::string_view sv(const zeek::StringVal* val) noexcept + { + return sv(val->AsString()); + }; + +std::vector sv_vec(zeek::VectorVal* xs) + { + std::vector result; + if ( xs ) + for ( unsigned index = 0; index < xs->Size(); ++index ) + if ( auto ptr = xs->ValAt(index) ) + if ( auto* str = ptr->AsString() ) + result.emplace_back(sv(str)); + return result; + } + +std::vector sv_tbl(zeek::TableVal* xs) + { + std::vector result; + if ( xs ) + { + for ( auto& val : *xs->Get() ) + { + auto val_ptr = val.GetValue()->GetVal(); + result.emplace_back(std::string_view{val.GetKey(), val.key_size}, + sv(val_ptr->AsStringVal())); + } + } + return result; + } + +bool is_valid(zeek::Span labels, + zeek::Span label_names) + { + auto key_in_label_names = [keys{label_names}](auto x) + { + return std::find(keys.begin(), keys.end(), x.first) != keys.end(); + }; + return labels.size() == label_names.size() + && std::all_of(labels.begin(), labels.end(), key_in_label_names); + } + +template +auto to_std_vec(zeek::VectorVal* xs) + { + std::vector result; + if( xs ) + for ( unsigned index = 0; index < xs->Size() ; ++index ) + { + if constexpr (std::is_same::value) + result.emplace_back(xs->ValAt(index)->AsInt()); + else + result.emplace_back(xs->ValAt(index)->AsDouble()); + } + return result; + } + +template +auto to_std_vec(zeek::Val* xs) + { + return to_std_vec(xs->AsVectorVal()); + } + +} +%%} + +# -- IntCounter ---------------------------------------------------------------- + +function Telemetry::__int_counter_family%(prefix: string, + name: string, + labels: string_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of int_counter_metric_family + %{ + auto lbl_vec = sv_vec(labels->AsVectorVal()); + auto hdl = telemetry_mgr->CounterFamily(sv(prefix), sv(name), lbl_vec, + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__int_counter_metric_get_or_add%(family: opaque of int_counter_metric_family, + labels: table_string_of_string%): opaque of int_counter_metric + %{ + using ResultType = zeek::IntrusivePtr; + if ( auto ptr = dynamic_cast(family) ) + { + auto hdl = ptr->GetHandle(); + auto lbl_map = sv_tbl(labels->AsTableVal()); + if ( is_valid(lbl_map, hdl.LabelNames()) ) + { + auto res = hdl.GetOrAdd(lbl_map); + return zeek::make_intrusive(res); + } + else + { + zeek::reporter->Error("Telemetry::int_counter_metric_get_or_add: invalid label dimensions."); + return ResultType{nullptr}; + } + } + else + { + zeek::reporter->Error("Telemetry::int_counter_metric_get_or_add: invalid handle."); + return ResultType{nullptr}; + } + %} + +function Telemetry::__int_counter_singleton%(prefix: string, + name: string, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of int_counter_metric + %{ + auto hdl = telemetry_mgr->CounterSingleton(sv(prefix), sv(name), + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__int_counter_inc%(val: opaque of int_counter_metric, + amount: int &default = 1%): bool + %{ + return with(val, "Telemetry::int_counter_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); }); + %} + +function Telemetry::__int_counter_value%(val: opaque of int_counter_metric%): int + %{ + if ( auto ptr = dynamic_cast(val) ) + { + return zeek::val_mgr->Int(ptr->GetHandle().Value()); + } + else + { + zeek::reporter->Error("Telemetry::int_counter_value: invalid handle."); + return zeek::val_mgr->Int(0); + } + %} + +# -- DblCounter ---------------------------------------------------------------- + +function Telemetry::__dbl_counter_family%(prefix: string, + name: string, + labels: string_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of dbl_counter_metric_family + %{ + auto lbl_vec = sv_vec(labels->AsVectorVal()); + auto hdl = telemetry_mgr->CounterFamily(sv(prefix), sv(name), lbl_vec, + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__dbl_counter_metric_get_or_add%(family: opaque of dbl_counter_metric_family, + labels: table_string_of_string%): opaque of dbl_counter_metric + %{ + using ResultType = zeek::IntrusivePtr; + if ( auto ptr = dynamic_cast(family) ) + { + auto hdl = ptr->GetHandle(); + auto lbl_map = sv_tbl(labels->AsTableVal()); + if ( is_valid(lbl_map, hdl.LabelNames()) ) + { + auto res = hdl.GetOrAdd(lbl_map); + return zeek::make_intrusive(res); + } + else + { + zeek::reporter->Error("Telemetry::dbl_counter_metric_get_or_add: invalid label dimensions."); + return ResultType{nullptr}; + } + } + else + { + zeek::reporter->Error("Telemetry::dbl_counter_metric_get_or_add: invalid handle."); + return ResultType{nullptr}; + } + %} + +function Telemetry::__dbl_counter_singleton%(prefix: string, + name: string, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of dbl_counter_metric + %{ + auto hdl = telemetry_mgr->CounterSingleton(sv(prefix), sv(name), + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__dbl_counter_inc%(val: opaque of dbl_counter_metric, + amount: double &default = 1.0%): bool + %{ + return with(val, "Telemetry::dbl_counter_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); }); + %} + +function Telemetry::__dbl_counter_value%(val: opaque of dbl_counter_metric%): double + %{ + if ( auto ptr = dynamic_cast(val) ) + { + return zeek::make_intrusive(ptr->GetHandle().Value()); + } + else + { + zeek::reporter->Error("Telemetry::dbl_counter_value: invalid handle."); + return zeek::make_intrusive(0.0); + } + %} + +# -- IntGauge ------------------------------------------------------------------ + +function Telemetry::__int_gauge_family%(prefix: string, + name: string, + labels: string_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of int_gauge_metric_family + %{ + auto lbl_vec = sv_vec(labels->AsVectorVal()); + auto hdl = telemetry_mgr->GaugeFamily(sv(prefix), sv(name), lbl_vec, + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__int_gauge_metric_get_or_add%(family: opaque of int_gauge_metric_family, + labels: table_string_of_string%): opaque of int_gauge_metric + %{ + using ResultType = zeek::IntrusivePtr; + if ( auto ptr = dynamic_cast(family) ) + { + auto hdl = ptr->GetHandle(); + auto lbl_map = sv_tbl(labels->AsTableVal()); + if ( is_valid(lbl_map, hdl.LabelNames()) ) + { + auto res = hdl.GetOrAdd(lbl_map); + return zeek::make_intrusive(res); + } + else + { + zeek::reporter->Error("Telemetry::int_gauge_metric_get_or_add: invalid label dimensions."); + return ResultType{nullptr}; + } + } + else + { + zeek::reporter->Error("Telemetry::int_gauge_metric_get_or_add: invalid handle."); + return ResultType{nullptr}; + } + %} + +function Telemetry::__int_gauge_singleton%(prefix: string, + name: string, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of int_gauge_metric + %{ + auto hdl = telemetry_mgr->GaugeSingleton(sv(prefix), sv(name), + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__int_gauge_inc%(val: opaque of int_gauge_metric, + amount: int &default = 1%): bool + %{ + return with(val, "Telemetry::int_gauge_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); }); + %} + +function Telemetry::__int_gauge_dec%(val: opaque of int_gauge_metric, + amount: int &default = 1%): bool + %{ + return with(val, "Telemetry::int_gauge_dec: invalid handle.", [amount](auto hdl) { hdl.Dec(amount); }); + %} + +function Telemetry::__int_gauge_value%(val: opaque of int_gauge_metric%): int + %{ + if ( auto ptr = dynamic_cast(val) ) + { + return zeek::val_mgr->Int(ptr->GetHandle().Value()); + } + else + { + zeek::reporter->Error("Telemetry::int_gauge_value: invalid handle."); + return zeek::val_mgr->Int(0); + } + %} + +# -- DblGauge ------------------------------------------------------------------ + +function Telemetry::__dbl_gauge_family%(prefix: string, + name: string, + labels: string_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of dbl_gauge_metric_family + %{ + auto lbl_vec = sv_vec(labels->AsVectorVal()); + auto hdl = telemetry_mgr->GaugeFamily(sv(prefix), sv(name), lbl_vec, + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__dbl_gauge_metric_get_or_add%(family: opaque of dbl_gauge_metric_family, + labels: table_string_of_string%): opaque of dbl_gauge_metric + %{ + using ResultType = zeek::IntrusivePtr; + if ( auto ptr = dynamic_cast(family) ) + { + auto hdl = ptr->GetHandle(); + auto lbl_map = sv_tbl(labels->AsTableVal()); + if ( is_valid(lbl_map, hdl.LabelNames()) ) + { + auto res = hdl.GetOrAdd(lbl_map); + return zeek::make_intrusive(res); + } + else + { + zeek::reporter->Error("Telemetry::dbl_gauge_metric_get_or_add: invalid label dimensions."); + return ResultType{nullptr}; + } + } + else + { + zeek::reporter->Error("Telemetry::dbl_gauge_metric_get_or_add: invalid handle."); + return ResultType{nullptr}; + } + %} + +function Telemetry::__dbl_gauge_singleton%(prefix: string, + name: string, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of dbl_gauge_metric + %{ + auto hdl = telemetry_mgr->GaugeSingleton(sv(prefix), sv(name), + sv(helptext), sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__dbl_gauge_inc%(val: opaque of dbl_gauge_metric, + amount: double &default = 1.0%): bool + %{ + return with(val, "Telemetry::dbl_gauge_inc: invalid handle.", [amount](auto hdl) { hdl.Inc(amount); }); + %} + +function Telemetry::__dbl_gauge_dec%(val: opaque of dbl_gauge_metric, + amount: double &default = 1.0%): bool + %{ + return with(val, "Telemetry::dbl_gauge_dec: invalid handle.", [amount](auto hdl) { hdl.Dec(amount); }); + %} + +function Telemetry::__dbl_gauge_value%(val: opaque of dbl_gauge_metric%): double + %{ + if ( auto ptr = dynamic_cast(val) ) + { + return zeek::make_intrusive(ptr->GetHandle().Value()); + } + else + { + zeek::reporter->Error("Telemetry::dbl_gauge_value: invalid handle."); + return zeek::make_intrusive(0.0); + } + %} + +# -- IntHistogram -------------------------------------------------------------- + +function Telemetry::__int_histogram_family%(prefix: string, + name: string, + labels: string_vec, + bounds: int_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of int_histogram_metric_family + %{ + auto lbl_vec = sv_vec(labels->AsVectorVal()); + auto std_bounds = to_std_vec(bounds); + auto hdl = telemetry_mgr->HistogramFamily(sv(prefix), sv(name), lbl_vec, + std_bounds, sv(helptext), + sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__int_histogram_metric_get_or_add%(family: opaque of int_histogram_metric_family, + labels: table_string_of_string%): opaque of int_histogram_metric + %{ + using ResultType = zeek::IntrusivePtr; + if ( auto ptr = dynamic_cast(family) ) + { + auto hdl = ptr->GetHandle(); + auto lbl_map = sv_tbl(labels->AsTableVal()); + if ( is_valid(lbl_map, hdl.LabelNames()) ) + { + auto res = hdl.GetOrAdd(lbl_map); + return zeek::make_intrusive(res); + } + else + { + zeek::reporter->Error("Telemetry::int_histogram_metric_get_or_add: invalid label dimensions."); + return ResultType{nullptr}; + } + } + else + { + zeek::reporter->Error("Telemetry::int_histogram_metric_get_or_add: invalid handle."); + return ResultType{nullptr}; + } + %} + +function Telemetry::__int_histogram_singleton%(prefix: string, + name: string, + bounds: int_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of int_histogram_metric + %{ + auto std_bounds = to_std_vec(bounds); + auto hdl = telemetry_mgr->HistogramSingleton(sv(prefix), sv(name), + std_bounds, sv(helptext), + sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__int_histogram_observe%(val: opaque of int_histogram_metric, + measurement: int%): bool + %{ + return with(val, "Telemetry::int_histogram_inc: invalid handle.", [measurement](auto hdl) { hdl.Observe(measurement); }); + %} + +function Telemetry::__int_histogram_sum%(val: opaque of int_histogram_metric%): int + %{ + if ( auto ptr = dynamic_cast(val) ) + { + return zeek::val_mgr->Int(ptr->GetHandle().Sum()); + } + else + { + zeek::reporter->Error("Telemetry::int_histogram_sum: invalid handle."); + return zeek::val_mgr->Int(0); + } + %} + +# -- DblHistogram -------------------------------------------------------------- + +function Telemetry::__dbl_histogram_family%(prefix: string, + name: string, + labels: string_vec, + bounds: double_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of dbl_histogram_metric_family + %{ + auto lbl_vec = sv_vec(labels->AsVectorVal()); + auto std_bounds = to_std_vec(bounds); + auto hdl = telemetry_mgr->HistogramFamily(sv(prefix), sv(name), + lbl_vec, std_bounds, + sv(helptext), sv(unit), + is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__dbl_histogram_metric_get_or_add%(family: opaque of dbl_histogram_metric_family, + labels: table_string_of_string%): opaque of dbl_histogram_metric + %{ + using ResultType = zeek::IntrusivePtr; + if ( auto ptr = dynamic_cast(family) ) + { + auto hdl = ptr->GetHandle(); + auto lbl_map = sv_tbl(labels->AsTableVal()); + if ( is_valid(lbl_map, hdl.LabelNames()) ) + { + auto res = hdl.GetOrAdd(lbl_map); + return zeek::make_intrusive(res); + } + else + { + zeek::reporter->Error("Telemetry::dbl_histogram_metric_get_or_add: invalid label dimensions."); + return ResultType{nullptr}; + } + } + else + { + zeek::reporter->Error("Telemetry::dbl_histogram_metric_get_or_add: invalid handle."); + return ResultType{nullptr}; + } + %} + +function Telemetry::__dbl_histogram_singleton%(prefix: string, + name: string, + bounds: double_vec, + helptext: string &default = "Zeek Script Metric", + unit: string &default = "1", + is_sum: bool &default = F%): opaque of dbl_histogram_metric + %{ + auto std_bounds = to_std_vec(bounds); + auto hdl = telemetry_mgr->HistogramSingleton(sv(prefix), sv(name), + std_bounds, sv(helptext), + sv(unit), is_sum); + return zeek::make_intrusive(hdl); + %} + +function Telemetry::__dbl_histogram_observe%(val: opaque of dbl_histogram_metric, + measurement: double%): bool + %{ + return with(val, "Telemetry::dbl_histogram_inc: invalid handle.", [measurement](auto hdl) { hdl.Observe(measurement); }); + %} + +function Telemetry::__dbl_histogram_sum%(val: opaque of dbl_histogram_metric%): double + %{ + if ( auto ptr = dynamic_cast(val) ) + { + return zeek::make_intrusive(ptr->GetHandle().Sum()); + } + else + { + zeek::reporter->Error("Telemetry::dbl_histogram_sum: invalid handle."); + return zeek::make_intrusive(0.0); + } + %} diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index a03f9b3696..e349b77de4 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -69,6 +69,7 @@ extern "C" { #include "zeek/zeekygen/Manager.h" #include "zeek/iosource/Manager.h" #include "zeek/broker/Manager.h" +#include "zeek/telemetry/Manager.h" #include "zeek/binpac_zeek.h" #include "zeek/module_util.h" @@ -107,6 +108,7 @@ zeek::file_analysis::Manager* zeek::file_mgr = nullptr; zeek::zeekygen::detail::Manager* zeek::detail::zeekygen_mgr = nullptr; zeek::iosource::Manager* zeek::iosource_mgr = nullptr; zeek::Broker::Manager* zeek::broker_mgr = nullptr; +zeek::telemetry::Manager* zeek::telemetry_mgr = nullptr; zeek::Supervisor* zeek::supervisor_mgr = nullptr; zeek::detail::trigger::Manager* zeek::detail::trigger_mgr = nullptr; @@ -136,6 +138,18 @@ zeek::OpaqueTypePtr bloomfilter_type; zeek::OpaqueTypePtr x509_opaque_type; zeek::OpaqueTypePtr ocsp_resp_opaque_type; zeek::OpaqueTypePtr paraglob_type; +zeek::OpaqueTypePtr int_counter_metric_type; +zeek::OpaqueTypePtr int_counter_metric_family_type; +zeek::OpaqueTypePtr dbl_counter_metric_type; +zeek::OpaqueTypePtr dbl_counter_metric_family_type; +zeek::OpaqueTypePtr int_gauge_metric_type; +zeek::OpaqueTypePtr int_gauge_metric_family_type; +zeek::OpaqueTypePtr dbl_gauge_metric_type; +zeek::OpaqueTypePtr dbl_gauge_metric_family_type; +zeek::OpaqueTypePtr int_histogram_metric_type; +zeek::OpaqueTypePtr int_histogram_metric_family_type; +zeek::OpaqueTypePtr dbl_histogram_metric_type; +zeek::OpaqueTypePtr dbl_histogram_metric_family_type; // Keep copy of command line int zeek::detail::zeek_argc; @@ -327,6 +341,7 @@ static void terminate_bro() delete plugin_mgr; delete val_mgr; delete fragment_mgr; + delete telemetry_mgr; // free the global scope pop_scope(); @@ -576,6 +591,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) file_mgr = new file_analysis::Manager(); auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; broker_mgr = new Broker::Manager(broker_real_time); + telemetry_mgr = broker_mgr->NewTelemetryManager().release(); trigger_mgr = new trigger::Manager(); plugin_mgr->InitPreScript(); @@ -600,6 +616,18 @@ SetupResult setup(int argc, char** argv, Options* zopts) x509_opaque_type = make_intrusive("x509"); ocsp_resp_opaque_type = make_intrusive("ocsp_resp"); paraglob_type = make_intrusive("paraglob"); + int_counter_metric_type = make_intrusive("int_counter_metric"); + int_counter_metric_family_type = make_intrusive("int_counter_metric_family"); + dbl_counter_metric_type = make_intrusive("dbl_counter_metric"); + dbl_counter_metric_family_type = make_intrusive("dbl_counter_metric_family"); + int_gauge_metric_type = make_intrusive("int_gauge_metric"); + int_gauge_metric_family_type = make_intrusive("int_gauge_metric_family"); + dbl_gauge_metric_type = make_intrusive("dbl_gauge_metric"); + dbl_gauge_metric_family_type = make_intrusive("dbl_gauge_metric_family"); + int_histogram_metric_type = make_intrusive("int_histogram_metric"); + int_histogram_metric_family_type = make_intrusive("int_histogram_metric_family"); + dbl_histogram_metric_type = make_intrusive("dbl_histogram_metric"); + dbl_histogram_metric_family_type = make_intrusive("dbl_histogram_metric_family"); // The leak-checker tends to produce some false // positives (memory which had already been @@ -667,6 +695,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) plugin_mgr->InitPostScript(); zeekygen_mgr->InitPostScript(); broker_mgr->InitPostScript(); + telemetry_mgr->InitPostScript(); timer_mgr->InitPostScript(); event_mgr.InitPostScript(); diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 43222b07e4..813ba4354b 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -97,6 +97,7 @@ scripts/base/init-frameworks-and-bifs.zeek scripts/base/utils/patterns.zeek scripts/base/frameworks/files/magic/__load__.zeek build/scripts/base/bif/__load__.zeek + build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index e65e4dd62e..7be8032039 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -97,6 +97,7 @@ scripts/base/init-frameworks-and-bifs.zeek scripts/base/utils/patterns.zeek scripts/base/frameworks/files/magic/__load__.zeek build/scripts/base/bif/__load__.zeek + build/scripts/base/bif/telemetry.bif.zeek build/scripts/base/bif/zeekygen.bif.zeek build/scripts/base/bif/pcap.bif.zeek build/scripts/base/bif/bloom-filter.bif.zeek diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 700c195e39..aafea1436e 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -837,6 +837,7 @@ 0.000000 MetaHookPost LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./sum, <...>/sum.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./thresholds, <...>/thresholds.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./topk, <...>/topk.zeek) -> -1 @@ -1840,6 +1841,7 @@ 0.000000 MetaHookPre LoadFile(0, ./strings.bif.zeek, <...>/strings.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./sum, <...>/sum.zeek) 0.000000 MetaHookPre LoadFile(0, ./supervisor.bif.zeek, <...>/supervisor.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./telemetry.bif.zeek, <...>/telemetry.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./thresholds, <...>/thresholds.zeek) 0.000000 MetaHookPre LoadFile(0, ./top-k.bif.zeek, <...>/top-k.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./topk, <...>/topk.zeek) @@ -2853,6 +2855,7 @@ 0.000000 | HookLoadFile ./strings.bif.zeek <...>/strings.bif.zeek 0.000000 | HookLoadFile ./sum <...>/sum.zeek 0.000000 | HookLoadFile ./supervisor.bif.zeek <...>/supervisor.bif.zeek +0.000000 | HookLoadFile ./telemetry.bif.zeek <...>/telemetry.bif.zeek 0.000000 | HookLoadFile ./thresholds <...>/thresholds.zeek 0.000000 | HookLoadFile ./top-k.bif.zeek <...>/top-k.bif.zeek 0.000000 | HookLoadFile ./topk <...>/topk.zeek diff --git a/testing/btest/Baseline/telemetry.counter/output b/testing/btest/Baseline/telemetry.counter/output new file mode 100644 index 0000000000..72ca34c137 --- /dev/null +++ b/testing/btest/Baseline/telemetry.counter/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +cnt1_bar: 1 +cnt2_bar: 42 +cnt3_bar: 1.000000 +cnt4_bar: 42.000000 diff --git a/testing/btest/Baseline/telemetry.gauge/output b/testing/btest/Baseline/telemetry.gauge/output new file mode 100644 index 0000000000..add597e534 --- /dev/null +++ b/testing/btest/Baseline/telemetry.gauge/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +gg1_bar: 1 +gg2_bar: 23 +gg3_bar: 1.000000 +gg4_bar: 23.000000 diff --git a/testing/btest/Baseline/telemetry.histogram/output b/testing/btest/Baseline/telemetry.histogram/output new file mode 100644 index 0000000000..59a5447d15 --- /dev/null +++ b/testing/btest/Baseline/telemetry.histogram/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +hst1_bar: 12 +hst2_bar: 31337 +hst3_bar: 6.000000 +hst4_bar: 64.000000 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 12c5b9bd4d..48f2aaf929 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,5 +1,5 @@ [btest] -TestDirs = doc bifs language core scripts coverage signatures plugins broker supervisor +TestDirs = doc bifs language core scripts coverage signatures plugins broker supervisor telemetry TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp diff --git a/testing/btest/telemetry/counter.zeek b/testing/btest/telemetry/counter.zeek new file mode 100644 index 0000000000..efe062e46d --- /dev/null +++ b/testing/btest/telemetry/counter.zeek @@ -0,0 +1,25 @@ +# @TEST-GROUP: Telemetry + +# @TEST-EXEC: zeek -b %INPUT >output +# @TEST-EXEC: btest-diff output + +global cnt1 = Telemetry::__int_counter_family("cnt1", "bar", vector("dim1", "dim2")); +global cnt2_bar = Telemetry::__int_counter_singleton("cnt2", "bar"); +global cnt3 = Telemetry::__dbl_counter_family("cnt3", "bar", vector("dim1", "dim2")); +global cnt4_bar = Telemetry::__dbl_counter_singleton("cnt4", "bar"); + +event zeek_init() + { + local cnt1_bar = Telemetry::__int_counter_metric_get_or_add(cnt1, table(["dim1"] = "val1", ["dim2"] = "val2")); + Telemetry::__int_counter_inc(cnt1_bar); + Telemetry::__int_counter_inc(cnt2_bar); + Telemetry::__int_counter_inc(cnt2_bar, 41); + print fmt("cnt1_bar: %d", Telemetry::__int_counter_value(cnt1_bar)); + print fmt("cnt2_bar: %d", Telemetry::__int_counter_value(cnt2_bar)); + local cnt3_bar = Telemetry::__dbl_counter_metric_get_or_add(cnt3, table(["dim1"] = "val1", ["dim2"] = "val2")); + Telemetry::__dbl_counter_inc(cnt3_bar); + Telemetry::__dbl_counter_inc(cnt4_bar); + Telemetry::__dbl_counter_inc(cnt4_bar, 41.0); + print fmt("cnt3_bar: %f", Telemetry::__dbl_counter_value(cnt3_bar)); + print fmt("cnt4_bar: %f", Telemetry::__dbl_counter_value(cnt4_bar)); + } diff --git a/testing/btest/telemetry/gauge.zeek b/testing/btest/telemetry/gauge.zeek new file mode 100644 index 0000000000..5966571dbc --- /dev/null +++ b/testing/btest/telemetry/gauge.zeek @@ -0,0 +1,30 @@ +# @TEST-GROUP: Telemetry + +# @TEST-EXEC: zeek -b %INPUT >output +# @TEST-EXEC: btest-diff output + +global gg1 = Telemetry::__int_gauge_family("gg1", "bar", vector("dim1", "dim2")); +global gg2_bar = Telemetry::__int_gauge_singleton("gg2", "bar"); +global gg3 = Telemetry::__dbl_gauge_family("gg3", "bar", vector("dim1", "dim2")); +global gg4_bar = Telemetry::__dbl_gauge_singleton("gg4", "bar"); + +event zeek_init() + { + local gg1_bar = Telemetry::__int_gauge_metric_get_or_add(gg1, table(["dim1"] = "val1", ["dim2"] = "val2")); + Telemetry::__int_gauge_inc(gg1_bar); + Telemetry::__int_gauge_inc(gg2_bar); + Telemetry::__int_gauge_inc(gg2_bar, 41); + Telemetry::__int_gauge_dec(gg2_bar); + Telemetry::__int_gauge_dec(gg2_bar, 18); + print fmt("gg1_bar: %d", Telemetry::__int_gauge_value(gg1_bar)); + print fmt("gg2_bar: %d", Telemetry::__int_gauge_value(gg2_bar)); + local gg3_bar = Telemetry::__dbl_gauge_metric_get_or_add(gg3, table(["dim1"] = "val1", ["dim2"] = "val2")); + Telemetry::__dbl_gauge_inc(gg3_bar); + Telemetry::__dbl_gauge_inc(gg4_bar); + Telemetry::__dbl_gauge_inc(gg4_bar, 41.0); + Telemetry::__dbl_gauge_dec(gg4_bar); + Telemetry::__dbl_gauge_dec(gg4_bar, 18.0); + print fmt("gg3_bar: %f", Telemetry::__dbl_gauge_value(gg3_bar)); + print fmt("gg4_bar: %f", Telemetry::__dbl_gauge_value(gg4_bar)); + } + diff --git a/testing/btest/telemetry/histogram.zeek b/testing/btest/telemetry/histogram.zeek new file mode 100644 index 0000000000..de183872b0 --- /dev/null +++ b/testing/btest/telemetry/histogram.zeek @@ -0,0 +1,28 @@ +# @TEST-GROUP: Telemetry + +# @TEST-EXEC: zeek -b %INPUT >output +# @TEST-EXEC: btest-diff output + +const int_bounds = vector(+10, +20); +const dbl_bounds = vector(10.0, 20.0); + +global hst1 = Telemetry::__int_histogram_family("hst1", "bar", vector("dim1", "dim2"), int_bounds); +global hst2_bar = Telemetry::__int_histogram_singleton("hst2", "bar", int_bounds); +global hst3 = Telemetry::__dbl_histogram_family("hst3", "bar", vector("dim1", "dim2"), dbl_bounds); +global hst4_bar = Telemetry::__dbl_histogram_singleton("hst4", "bar", dbl_bounds); + +event zeek_init() + { + local hst1_bar = Telemetry::__int_histogram_metric_get_or_add(hst1, table(["dim1"] = "val1", ["dim2"] = "val2")); + Telemetry::__int_histogram_observe(hst1_bar, 1); + Telemetry::__int_histogram_observe(hst1_bar, 11); + Telemetry::__int_histogram_observe(hst2_bar, 31337); + print fmt("hst1_bar: %d", Telemetry::__int_histogram_sum(hst1_bar)); + print fmt("hst2_bar: %d", Telemetry::__int_histogram_sum(hst2_bar)); + local hst3_bar = Telemetry::__dbl_histogram_metric_get_or_add(hst3, table(["dim1"] = "val1", ["dim2"] = "val2")); + Telemetry::__dbl_histogram_observe(hst3_bar, 2.0); + Telemetry::__dbl_histogram_observe(hst3_bar, 4.0); + Telemetry::__dbl_histogram_observe(hst4_bar, 64.0); + print fmt("hst3_bar: %f", Telemetry::__dbl_histogram_sum(hst3_bar)); + print fmt("hst4_bar: %f", Telemetry::__dbl_histogram_sum(hst4_bar)); + }