From 56f30b500aa23cde7edb041cf6f21776ded74cd8 Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Mon, 20 Dec 2021 08:16:21 +0100 Subject: [PATCH 1/2] Update to latest Broker without public CAF dep --- CMakeLists.txt | 46 +---- auxil/broker | 2 +- src/Func.h | 7 +- src/OpaqueVal.cc | 44 ++-- src/Val.cc | 10 +- src/broker/Data.cc | 66 +++--- src/broker/Data.h | 6 +- src/broker/Manager.cc | 107 +++------- src/broker/Manager.h | 5 - src/broker/Store.h | 4 +- src/broker/data.bif | 4 +- src/file_analysis/analyzer/x509/X509.cc | 2 +- src/logging/WriterBackend.cc | 20 +- src/probabilistic/BitVector.cc | 8 +- src/probabilistic/BloomFilter.cc | 4 +- src/probabilistic/CardinalityCounter.cc | 10 +- src/probabilistic/CounterVector.cc | 4 +- src/probabilistic/Hasher.cc | 10 +- src/probabilistic/Topk.cc | 16 +- src/telemetry/CMakeLists.txt | 4 - src/telemetry/Counter.cc | 77 ------- src/telemetry/Counter.h | 58 +++--- src/telemetry/Detail.h | 259 ------------------------ src/telemetry/Gauge.cc | 102 ---------- src/telemetry/Gauge.h | 73 ++++--- src/telemetry/Histogram.cc | 102 ---------- src/telemetry/Histogram.h | 67 +++--- src/telemetry/Manager.cc | 125 +++--------- src/telemetry/Manager.h | 69 ++++--- src/telemetry/MetricFamily.cc | 54 ----- src/telemetry/MetricFamily.h | 25 ++- src/zeek-setup.cc | 2 +- 32 files changed, 328 insertions(+), 1064 deletions(-) delete mode 100644 src/telemetry/Counter.cc delete mode 100644 src/telemetry/Detail.h delete mode 100644 src/telemetry/Gauge.cc delete mode 100644 src/telemetry/Histogram.cc delete mode 100644 src/telemetry/MetricFamily.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 494cb2efa2..e934481023 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -321,36 +321,15 @@ if ( PYTHON_VERSION_STRING VERSION_LESS ${ZEEK_PYTHON_MIN} ) message(FATAL_ERROR "Python ${ZEEK_PYTHON_MIN} or greater is required.") endif () -if ( CAF_ROOT OR BROKER_ROOT_DIR ) - # TODO: drop < 3.12 compatibility check when raising the minimum CMake version - if ( CAF_ROOT AND CMAKE_VERSION VERSION_LESS 3.12 ) - find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED - COMPONENTS openssl test io core - PATHS "${CAF_ROOT}") - else () - find_package(CAF ${CAF_VERSION_MIN_REQUIRED} REQUIRED - COMPONENTS openssl test io core) - endif () - message(STATUS "Using system CAF version ${CAF_VERSION}") - # TODO: drop these legacy variables and simply use the targets consistently - set(CAF_LIBRARIES CAF::core CAF::io CAF::openssl CACHE INTERNAL "") - set(caf_dirs "") - foreach (caf_lib IN LISTS CAF_LIBRARIES ITEMS CAF::test) - get_target_property(dirs ${caf_lib} INTERFACE_INCLUDE_DIRECTORIES) - if ( dirs ) - list(APPEND caf_dirs ${dirs}) - endif () - endforeach () - list(REMOVE_DUPLICATES caf_dirs) - list(GET caf_dirs 0 caf_dir) - set(CAF_INCLUDE_DIRS "${caf_dirs}" CACHE INTERNAL "") -endif () - add_subdirectory(auxil/paraglob) set(zeekdeps ${zeekdeps} paraglob) -if ( BROKER_ROOT_DIR ) - find_package(Broker REQUIRED) +if ( Broker_ROOT ) + find_package(Broker REQUIRED PATHS "${Broker_ROOT}") + set(zeekdeps ${zeekdeps} ${BROKER_LIBRARY}) + set(broker_includes ${BROKER_INCLUDE_DIR}) +elseif ( BROKER_ROOT_DIR ) + find_package(Broker REQUIRED PATHS "${BROKER_ROOT_DIR}") set(zeekdeps ${zeekdeps} ${BROKER_LIBRARY}) set(broker_includes ${BROKER_INCLUDE_DIR}) else () @@ -375,11 +354,6 @@ else () set(broker_includes ${CMAKE_CURRENT_SOURCE_DIR}/auxil/broker/include ${CMAKE_CURRENT_BINARY_DIR}/auxil/broker/include) endif () -# CAF_LIBRARIES and CAF_INCLUDE_DIRS are defined either by calling -# find_package(CAF) or by calling add_subdirectory(auxil/broker). In either case, -# we have to care about CAF here because Broker headers can pull in CAF -# headers. -set(zeekdeps ${zeekdeps} ${CAF_LIBRARIES}) include_directories(BEFORE ${PCAP_INCLUDE_DIR} ${BIND_INCLUDE_DIR} @@ -452,7 +426,7 @@ endif () # Any headers that are possibly bundled in the Zeek source-tree and that are supposed # to have priority over any pre-existing/system-wide headers need to appear early in # compiler search path. -include_directories(BEFORE ${broker_includes} ${CAF_INCLUDE_DIRS}) +include_directories(BEFORE ${broker_includes}) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/highwayhash) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/paraglob/include) include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/auxil/rapidjson/include) @@ -512,12 +486,6 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink "." "${CMAKE_CURRENT_BINARY_DIR}/zeek") -if ( CAF_ROOT ) - set(ZEEK_CONFIG_CAF_ROOT_DIR ${CAF_ROOT}) -else () - set(ZEEK_CONFIG_CAF_ROOT_DIR ${ZEEK_ROOT_DIR}) -endif () - if ( BinPAC_ROOT_DIR ) set(ZEEK_CONFIG_BINPAC_ROOT_DIR ${BinPAC_ROOT_DIR}) else () diff --git a/auxil/broker b/auxil/broker index 36339ad8c9..ce321c0a90 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 36339ad8c9844f1758f8e9d0da4645047f480c34 +Subproject commit ce321c0a90f1d4176796c996fd27fe69ea520316 diff --git a/src/Func.h b/src/Func.h index 713a34fb67..d3bae45d01 100644 --- a/src/Func.h +++ b/src/Func.h @@ -18,16 +18,11 @@ #include "zeek/ZeekArgs.h" #include "zeek/ZeekList.h" -namespace caf - { -template class expected; - } - namespace broker { class data; using vector = std::vector; -using caf::expected; +template class expected; } namespace zeek diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index e18d6e7e28..f39c1933a4 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -33,7 +33,7 @@ inline bool get_vector_idx(const V& v, unsigned int i, D* dst) if ( i >= v.size() ) return false; - auto x = caf::get_if(&v[i]); + auto x = broker::get_if(&v[i]); if ( ! x ) return false; @@ -81,12 +81,12 @@ broker::expected OpaqueVal::Serialize() const OpaqueValPtr OpaqueVal::Unserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() == 2) ) return nullptr; - auto type = caf::get_if(&(*v)[0]); + auto type = broker::get_if(&(*v)[0]); if ( ! type ) return nullptr; @@ -118,17 +118,17 @@ broker::expected OpaqueVal::SerializeType(const TypePtr& t) TypePtr OpaqueVal::UnserializeType(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() == 2) ) return nullptr; - auto by_name = caf::get_if(&(*v)[0]); + auto by_name = broker::get_if(&(*v)[0]); if ( ! by_name ) return nullptr; if ( *by_name ) { - auto name = caf::get_if(&(*v)[1]); + auto name = broker::get_if(&(*v)[1]); if ( ! name ) return nullptr; @@ -142,7 +142,7 @@ TypePtr OpaqueVal::UnserializeType(const broker::data& data) return id->GetType(); } - auto tag = caf::get_if(&(*v)[1]); + auto tag = broker::get_if(&(*v)[1]); if ( ! tag ) return nullptr; @@ -295,11 +295,11 @@ broker::expected MD5Val::DoSerialize() const bool MD5Val::DoUnserialize(const broker::data& data) { - auto d = caf::get_if(&data); + auto d = broker::get_if(&data); if ( ! d ) return false; - auto valid = caf::get_if(&(*d)[0]); + auto valid = broker::get_if(&(*d)[0]); if ( ! valid ) return false; @@ -312,7 +312,7 @@ bool MD5Val::DoUnserialize(const broker::data& data) if ( (*d).size() != 2 ) return false; - auto s = caf::get_if(&(*d)[1]); + auto s = broker::get_if(&(*d)[1]); if ( ! s ) return false; @@ -385,11 +385,11 @@ broker::expected SHA1Val::DoSerialize() const bool SHA1Val::DoUnserialize(const broker::data& data) { - auto d = caf::get_if(&data); + auto d = broker::get_if(&data); if ( ! d ) return false; - auto valid = caf::get_if(&(*d)[0]); + auto valid = broker::get_if(&(*d)[0]); if ( ! valid ) return false; @@ -402,7 +402,7 @@ bool SHA1Val::DoUnserialize(const broker::data& data) if ( (*d).size() != 2 ) return false; - auto s = caf::get_if(&(*d)[1]); + auto s = broker::get_if(&(*d)[1]); if ( ! s ) return false; @@ -475,11 +475,11 @@ broker::expected SHA256Val::DoSerialize() const bool SHA256Val::DoUnserialize(const broker::data& data) { - auto d = caf::get_if(&data); + auto d = broker::get_if(&data); if ( ! d ) return false; - auto valid = caf::get_if(&(*d)[0]); + auto valid = broker::get_if(&(*d)[0]); if ( ! valid ) return false; @@ -492,7 +492,7 @@ bool SHA256Val::DoUnserialize(const broker::data& data) if ( (*d).size() != 2 ) return false; - auto s = caf::get_if(&(*d)[1]); + auto s = broker::get_if(&(*d)[1]); if ( ! s ) return false; @@ -546,7 +546,7 @@ broker::expected EntropyVal::DoSerialize() const bool EntropyVal::DoUnserialize(const broker::data& data) { - auto d = caf::get_if(&data); + auto d = broker::get_if(&data); if ( ! d ) return false; @@ -728,12 +728,12 @@ broker::expected BloomFilterVal::DoSerialize() const bool BloomFilterVal::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() == 2) ) return false; - auto no_type = caf::get_if(&(*v)[0]); + auto no_type = broker::get_if(&(*v)[0]); if ( ! no_type ) { auto t = UnserializeType((*v)[0]); @@ -822,12 +822,12 @@ broker::expected CardinalityVal::DoSerialize() const bool CardinalityVal::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() == 2) ) return false; - auto no_type = caf::get_if(&(*v)[0]); + auto no_type = broker::get_if(&(*v)[0]); if ( ! no_type ) { auto t = UnserializeType((*v)[0]); @@ -879,7 +879,7 @@ broker::expected ParaglobVal::DoSerialize() const bool ParaglobVal::DoUnserialize(const broker::data& data) { - auto d = caf::get_if(&data); + auto d = broker::get_if(&data); if ( ! d ) return false; diff --git a/src/Val.cc b/src/Val.cc index 3ad6169149..7c89f28c41 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2145,15 +2145,7 @@ void TableVal::SendToStore(const Val* index, const TableEntryVal* new_entry_val, case ELEMENT_NEW: case ELEMENT_CHANGED: { -#ifndef __clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#endif - broker::optional expiry; -#ifndef __clang__ -#pragma GCC diagnostic pop -#endif - + std::optional expiry; auto expire_time = GetExpireTime(); if ( expire_time == 0 ) // Entry is set to immediately expire. Let's not forward it. diff --git a/src/broker/Data.cc b/src/broker/Data.cc index 80651fc822..84c59d3700 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -226,7 +226,7 @@ struct val_converter { const auto& expected_index_types = tt->GetIndices()->GetTypes(); broker::vector composite_key; - auto indices = caf::get_if(&item); + auto indices = get_if(&item); if ( indices ) { @@ -281,7 +281,7 @@ struct val_converter { const auto& expected_index_types = tt->GetIndices()->GetTypes(); broker::vector composite_key; - auto indices = caf::get_if(&item.first); + auto indices = get_if(&item.first); if ( indices ) { @@ -446,7 +446,7 @@ struct val_converter if ( idx >= a.size() ) return nullptr; - if ( caf::get_if(&a[idx]) != nullptr ) + if ( get_if(&a[idx]) != nullptr ) { rval->Remove(i); ++idx; @@ -469,8 +469,8 @@ struct val_converter if ( a.size() != 2 ) return nullptr; - auto exact_text = caf::get_if(&a[0]); - auto anywhere_text = caf::get_if(&a[1]); + auto exact_text = get_if(&a[0]); + auto anywhere_text = get_if(&a[1]); if ( ! exact_text || ! anywhere_text ) return nullptr; @@ -606,7 +606,7 @@ struct type_checker for ( const auto& item : a ) { const auto& expected_index_types = tt->GetIndices()->GetTypes(); - auto indices = caf::get_if(&item); + auto indices = get_if(&item); vector indices_to_check; if ( indices ) @@ -663,7 +663,7 @@ struct type_checker for ( auto& item : a ) { const auto& expected_index_types = tt->GetIndices()->GetTypes(); - auto indices = caf::get_if(&item.first); + auto indices = get_if(&item.first); vector indices_to_check; if ( indices ) @@ -765,7 +765,7 @@ struct type_checker if ( idx >= a.size() ) return false; - if ( caf::get_if(&a[idx]) != nullptr ) + if ( get_if(&a[idx]) != nullptr ) { ++idx; continue; @@ -784,8 +784,8 @@ struct type_checker if ( a.size() != 2 ) return false; - auto exact_text = caf::get_if(&a[0]); - auto anywhere_text = caf::get_if(&a[1]); + auto exact_text = get_if(&a[0]); + auto anywhere_text = get_if(&a[1]); if ( ! exact_text || ! anywhere_text ) return false; @@ -820,7 +820,7 @@ static bool data_type_check(const broker::data& d, Type* t) if ( t->Tag() == TYPE_ANY ) return true; - return caf::visit(type_checker{t}, d); + return visit(type_checker{t}, d); } ValPtr data_to_val(broker::data d, Type* type) @@ -828,7 +828,7 @@ ValPtr data_to_val(broker::data d, Type* type) if ( type->Tag() == TYPE_ANY ) return make_data_val(move(d)); - return caf::visit(val_converter{type}, std::move(d)); + return visit(val_converter{type}, d); } broker::expected val_to_data(const Val* v) @@ -961,7 +961,7 @@ broker::expected val_to_data(const Val* v) key = move(composite_key); if ( is_set ) - caf::get(rval).emplace(move(key)); + get(rval).emplace(move(key)); else { auto val = val_to_data(entry->GetVal().get()); @@ -969,7 +969,7 @@ broker::expected val_to_data(const Val* v) if ( ! val ) return broker::ec::invalid_data; - caf::get(rval).emplace(move(key), move(*val)); + get(rval).emplace(move(key), move(*val)); } } @@ -1180,7 +1180,7 @@ struct data_type_getter EnumValPtr get_data_type(RecordVal* v, zeek::detail::Frame* frame) { - return caf::visit(data_type_getter{}, opaque_field_to_data(v, frame)); + return visit(data_type_getter{}, opaque_field_to_data(v, frame)); } broker::data& opaque_field_to_data(RecordVal* v, zeek::detail::Frame* f) @@ -1240,11 +1240,11 @@ broker::expected SetIterator::DoSerialize() const bool SetIterator::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = get_if(&data); if ( ! (v && v->size() == 2) ) return false; - auto x = caf::get_if(&(*v)[0]); + auto x = get_if(&(*v)[0]); // We set the iterator by finding the element it used to point to. // This is not perfect, as there's no guarantee that the restored @@ -1267,11 +1267,11 @@ broker::expected TableIterator::DoSerialize() const bool TableIterator::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = get_if(&data); if ( ! (v && v->size() == 2) ) return false; - auto x = caf::get_if(&(*v)[0]); + auto x = get_if(&(*v)[0]); // We set the iterator by finding the element it used to point to. // This is not perfect, as there's no guarantee that the restored @@ -1295,12 +1295,12 @@ broker::expected VectorIterator::DoSerialize() const bool VectorIterator::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = get_if(&data); if ( ! (v && v->size() == 2) ) return false; - auto x = caf::get_if(&(*v)[0]); - auto y = caf::get_if(&(*v)[1]); + auto x = get_if(&(*v)[0]); + auto y = get_if(&(*v)[1]); if ( ! (x && y) ) return false; @@ -1320,12 +1320,12 @@ broker::expected RecordIterator::DoSerialize() const bool RecordIterator::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = get_if(&data); if ( ! (v && v->size() == 2) ) return false; - auto x = caf::get_if(&(*v)[0]); - auto y = caf::get_if(&(*v)[1]); + auto x = get_if(&(*v)[0]); + auto y = get_if(&(*v)[1]); if ( ! (x && y) ) return false; @@ -1352,25 +1352,25 @@ broker::data threading_field_to_data(const threading::Field* f) threading::Field* data_to_threading_field(broker::data d) { - if ( ! caf::holds_alternative(d) ) + if ( ! holds_alternative(d) ) return nullptr; - auto& v = caf::get(d); - auto name = caf::get_if(&v[0]); + auto& v = get(d); + auto name = get_if(&v[0]); auto secondary = v[1]; - auto type = caf::get_if(&v[2]); - auto subtype = caf::get_if(&v[3]); - auto optional = caf::get_if(&v[4]); + auto type = get_if(&v[2]); + auto subtype = get_if(&v[3]); + auto optional = get_if(&v[4]); if ( ! (name && type && subtype && optional) ) return nullptr; - if ( secondary != broker::nil && ! caf::holds_alternative(secondary) ) + if ( secondary != broker::nil && ! holds_alternative(secondary) ) return nullptr; return new threading::Field( name->c_str(), - secondary != broker::nil ? caf::get(secondary).c_str() : nullptr, + secondary != broker::nil ? get(secondary).c_str() : nullptr, static_cast(*type), static_cast(*subtype), *optional); } diff --git a/src/broker/Data.h b/src/broker/Data.h index b948f3744c..3335adf4b2 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -5,6 +5,8 @@ #include "zeek/OpaqueVal.h" #include "zeek/Reporter.h" +#include "broker/data.hh" + namespace zeek { @@ -181,10 +183,10 @@ broker::data& opaque_field_to_data(zeek::RecordVal* v, zeek::detail::Frame* f); template T& require_data_type(broker::data& d, zeek::TypeTag tag, zeek::detail::Frame* f) { - auto ptr = caf::get_if(&d); + auto ptr = broker::get_if(&d); if ( ! ptr ) zeek::reporter->RuntimeError(f->GetCallLocation(), "data is of type '%s' not of type '%s'", - caf::visit(type_name_getter{tag}, d), zeek::type_name(tag)); + visit(type_name_getter{tag}, d), zeek::type_name(tag)); return *ptr; } diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index d4a8445c37..babf719d3e 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -1,6 +1,7 @@ #include "zeek/broker/Manager.h" #include +#include #include #include #include @@ -90,10 +91,10 @@ namespace struct opt_mapping { broker::configuration* cfg; - std::string_view broker_name; + std::string broker_name; const char* zeek_name; - template auto broker_read() { return caf::get_as(*cfg, broker_name); } + template auto broker_read() { return broker::get_as(*cfg, broker_name); } template auto broker_write(T&& val) { cfg->set(broker_name, std::forward(val)); } @@ -107,23 +108,10 @@ struct opt_mapping } // namespace -class BrokerConfig : public broker::configuration - { -public: - BrokerConfig(broker::broker_options options) : broker::configuration(options) - { - openssl_cafile = get_option("Broker::ssl_cafile")->AsString()->CheckString(); - openssl_capath = get_option("Broker::ssl_capath")->AsString()->CheckString(); - openssl_certificate = get_option("Broker::ssl_certificate")->AsString()->CheckString(); - openssl_key = get_option("Broker::ssl_keyfile")->AsString()->CheckString(); - openssl_passphrase = get_option("Broker::ssl_passphrase")->AsString()->CheckString(); - } - }; - class BrokerState { public: - BrokerState(BrokerConfig config, size_t congestion_queue_size) + BrokerState(broker::configuration config, size_t congestion_queue_size) : endpoint(std::move(config)), subscriber(endpoint.make_subscriber({broker::topic::statuses(), broker::topic::errors()}, congestion_queue_size)) @@ -238,7 +226,13 @@ void Manager::InitPostScript() options.forward = get_option("Broker::forward_messages")->AsBool(); options.use_real_time = use_real_time; - BrokerConfig config{std::move(options)}; + broker::configuration config{std::move(options)}; + + config.openssl_cafile(get_option("Broker::ssl_cafile")->AsString()->CheckString()); + config.openssl_capath(get_option("Broker::ssl_capath")->AsString()->CheckString()); + config.openssl_certificate(get_option("Broker::ssl_certificate")->AsString()->CheckString()); + config.openssl_key(get_option("Broker::ssl_keyfile")->AsString()->CheckString()); + config.openssl_passphrase(get_option("Broker::ssl_passphrase")->AsString()->CheckString()); auto scheduler_policy = get_option("Broker::scheduler_policy")->AsString()->CheckString(); @@ -257,11 +251,11 @@ void Manager::InitPostScript() config.set("caf.scheduler.max-threads", get_option("Broker::max_threads")->AsCount()); config.set("caf.work-stealing.moderate-sleep-duration", - caf::timespan(static_cast( + broker::timespan(static_cast( get_option("Broker::moderate_sleep")->AsInterval() * 1e9))); config.set("caf.work-stealing.relaxed-sleep-duration", - caf::timespan( + broker::timespan( static_cast(get_option("Broker::relaxed_sleep")->AsInterval() * 1e9))); config.set("caf.work-stealing.aggressive-poll-attempts", @@ -363,6 +357,8 @@ void Manager::InitPostScript() bstate->subscriber.add_topic(broker::topic::store_events(), true); + telemetry_mgr->InitPostBrokerSetup(bstate->endpoint); + InitializeBrokerStoreForwarding(); } @@ -1030,7 +1026,7 @@ void Manager::DispatchMessage(const broker::topic& topic, broker::data msg) if ( ! batch.valid() ) { reporter->Warning("received invalid broker Batch: %s", - broker::to_string(batch).data()); + broker::to_string(batch.as_data()).data()); return; } @@ -1376,7 +1372,8 @@ bool Manager::ProcessLogCreate(broker::zeek::LogCreate lc) DBG_LOG(DBG_BROKER, "Received log-create: %s", RenderMessage(lc.as_data()).c_str()); if ( ! lc.valid() ) { - reporter->Warning("received invalid broker LogCreate: %s", broker::to_string(lc).data()); + reporter->Warning("received invalid broker LogCreate: %s", + broker::to_string(lc.as_data()).data()); return false; } @@ -1402,7 +1399,7 @@ bool Manager::ProcessLogCreate(broker::zeek::LogCreate lc) } // Get log fields. - auto fields_data = caf::get_if(&lc.fields_data()); + auto fields_data = get_if(&lc.fields_data()); if ( ! fields_data ) { @@ -1442,7 +1439,8 @@ bool Manager::ProcessLogWrite(broker::zeek::LogWrite lw) if ( ! lw.valid() ) { - reporter->Warning("received invalid broker LogWrite: %s", broker::to_string(lw).data()); + reporter->Warning("received invalid broker LogWrite: %s", + broker::to_string(lw.as_data()).data()); return false; } @@ -1467,7 +1465,7 @@ bool Manager::ProcessLogWrite(broker::zeek::LogWrite lw) return false; } - auto path = caf::get_if(&lw.path()); + auto path = get_if(&lw.path()); if ( ! path ) { @@ -1476,7 +1474,7 @@ bool Manager::ProcessLogWrite(broker::zeek::LogWrite lw) return false; } - auto serial_data = caf::get_if(&lw.serial_data()); + auto serial_data = get_if(&lw.serial_data()); if ( ! serial_data ) { @@ -1531,7 +1529,7 @@ bool Manager::ProcessIdentifierUpdate(broker::zeek::IdentifierUpdate iu) if ( ! iu.valid() ) { reporter->Warning("received invalid broker IdentifierUpdate: %s", - broker::to_string(iu).data()); + broker::to_string(iu.as_data()).data()); return false; } @@ -1655,12 +1653,12 @@ void Manager::ProcessError(broker::error_view err) if ( auto ctx = err.context() ) { msg += '('; - msg += to_string(ctx->node); + msg += broker::to_string(ctx->node); msg += ", "; - msg += caf::deep_to_string(ctx->network); + msg += broker::to_string(ctx->network); msg += ", "; if ( auto what = err.message() ) - msg += caf::deep_to_string(*what); + msg += *what; else msg += R"_("")_"; msg += ')'; @@ -1783,7 +1781,7 @@ void Manager::BrokerStoreToZeekTable(const std::string& name, const detail::Stor if ( ! keys ) return; - auto set = caf::get_if(&(keys->get_data())); + auto set = get_if(&(keys->get_data())); auto table = handle->forward_to; const auto& its = table->GetType()->AsTableType()->GetIndexTypes(); bool is_set = table->GetType()->IsSet(); @@ -1982,53 +1980,4 @@ void Manager::SetMetricsExportPrefixes(std::vector filter) bstate->endpoint.metrics_exporter().set_prefixes(std::move(filter)); } -std::unique_ptr Manager::NewTelemetryManager() - { - // The telemetry Manager actually only has a dependency on the actor system, - // not to the Broker Manager. By having the telemetry Manager hold on to a - // shared_ptr to our Broker state, we make sure the Broker endpoint, which - // owns the CAF actor system, lives for as long as necessary. This also - // makes sure that the Broker Manager may even get destroyed before the - // telemetry Manager. - struct TM final : public telemetry::Manager - { - using MetricRegistryPtr = std::unique_ptr; - - static auto getPimpl(BrokerState& st) - { - auto registry = std::addressof(st.endpoint.system().metrics()); - return reinterpret_cast(registry); - } - - static auto getPimpl(MetricRegistryPtr& ptr) - { - return reinterpret_cast(ptr.get()); - } - - explicit TM(Broker::Manager* parent, MetricRegistryPtr ptr) - : telemetry::Manager(getPimpl(ptr)), parent(parent), tmp(std::move(ptr)) - { - assert(tmp != nullptr); - assert(parent != nullptr); - } - - void InitPostScript() override - { - assert(parent->bstate != nullptr); - ptr = parent->bstate; - auto registry = std::addressof(ptr->endpoint.system().metrics()); - registry->merge(*tmp); - tmp.reset(); - pimpl = reinterpret_cast(registry); - } - - Broker::Manager* parent; - MetricRegistryPtr tmp; - std::shared_ptr ptr; - }; - - auto tmp = std::make_unique(); - return std::make_unique(this, std::move(tmp)); - } - } // namespace zeek::Broker diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 867180e7e9..0c1bcd0232 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -396,11 +396,6 @@ public: */ void SetMetricsExportPrefixes(std::vector filter); - /** - * Allocates a new manager for telemetry data. - */ - std::unique_ptr NewTelemetryManager(); - private: void DispatchMessage(const broker::topic& topic, broker::data msg); // Process events used for Broker store backed zeek tables diff --git a/src/broker/Store.h b/src/broker/Store.h index b81eec89cc..70cba682b6 100644 --- a/src/broker/Store.h +++ b/src/broker/Store.h @@ -52,9 +52,9 @@ inline RecordValPtr query_result(RecordValPtr data) * @param e: expire interval as double; 0 if no expiry * @return expire interval in Broker format */ -static broker::optional convert_expiry(double e) +static std::optional convert_expiry(double e) { - broker::optional ts; + std::optional ts; if ( e ) { diff --git a/src/broker/data.bif b/src/broker/data.bif index f671702bf2..85b34aa991 100644 --- a/src/broker/data.bif +++ b/src/broker/data.bif @@ -471,7 +471,7 @@ function Broker::__record_lookup%(r: Broker::Data, idx: count%): Broker::Data auto& v = zeek::Broker::detail::require_data_type(r->AsRecordVal(), zeek::TYPE_RECORD, frame); - if ( idx >= v.size() || caf::get_if(&v[idx]) ) + if ( idx >= v.size() || broker::get_if(&v[idx]) ) return zeek::make_intrusive(zeek::BifType::Record::Broker::Data); return zeek::Broker::detail::make_data_val(v[idx]); @@ -510,7 +510,7 @@ function Broker::__record_iterator_value%(it: opaque of Broker::RecordIterator%) return rval; } - if ( caf::get_if(&(*ri->it)) ) + if ( broker::get_if(&(*ri->it)) ) return rval; // field isn't set rval->Assign(0, zeek::make_intrusive(*ri->it)); diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc index e5841cca33..29e5d951a8 100644 --- a/src/file_analysis/analyzer/x509/X509.cc +++ b/src/file_analysis/analyzer/x509/X509.cc @@ -623,7 +623,7 @@ broker::expected X509Val::DoSerialize() const bool X509Val::DoUnserialize(const broker::data& data) { - auto s = caf::get_if(&data); + auto s = broker::get_if(&data); if ( ! s ) return false; diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 33c8d5e23c..e0aae0db8f 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -100,16 +100,16 @@ broker::data WriterBackend::WriterInfo::ToBroker() const bool WriterBackend::WriterInfo::FromBroker(broker::data d) { - if ( ! caf::holds_alternative(d) ) + if ( ! broker::is(d) ) return false; - auto v = caf::get(d); - auto bpath = caf::get_if(&v[0]); - auto brotation_base = caf::get_if(&v[1]); - auto brotation_interval = caf::get_if(&v[2]); - auto bnetwork_time = caf::get_if(&v[3]); - auto bconfig = caf::get_if(&v[4]); - auto bppf = caf::get_if(&v[5]); + auto v = broker::get(d); + auto bpath = broker::get_if(&v[0]); + auto brotation_base = broker::get_if(&v[1]); + auto brotation_interval = broker::get_if(&v[2]); + auto bnetwork_time = broker::get_if(&v[3]); + auto bconfig = broker::get_if(&v[4]); + auto bppf = broker::get_if(&v[5]); if ( ! (bpath && brotation_base && brotation_interval && bnetwork_time && bconfig && bppf) ) return false; @@ -122,8 +122,8 @@ bool WriterBackend::WriterInfo::FromBroker(broker::data d) for ( auto i : *bconfig ) { - auto k = caf::get_if(&i.first); - auto v = caf::get_if(&i.second); + auto k = broker::get_if(&i.first); + auto v = broker::get_if(&i.second); if ( ! (k && v) ) return false; diff --git a/src/probabilistic/BitVector.cc b/src/probabilistic/BitVector.cc index 2f2175a594..f18915a77f 100644 --- a/src/probabilistic/BitVector.cc +++ b/src/probabilistic/BitVector.cc @@ -512,12 +512,12 @@ broker::expected BitVector::Serialize() const std::unique_ptr BitVector::Unserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() >= 2) ) return nullptr; - auto num_bits = caf::get_if(&(*v)[0]); - auto size = caf::get_if(&(*v)[1]); + auto num_bits = broker::get_if(&(*v)[0]); + auto size = broker::get_if(&(*v)[1]); if ( ! (num_bits && size) ) return nullptr; @@ -530,7 +530,7 @@ std::unique_ptr BitVector::Unserialize(const broker::data& data) for ( size_t i = 0; i < *size; ++i ) { - auto x = caf::get_if(&(*v)[2 + i]); + auto x = broker::get_if(&(*v)[2 + i]); if ( ! x ) return nullptr; diff --git a/src/probabilistic/BloomFilter.cc b/src/probabilistic/BloomFilter.cc index 4c046c3511..d3f27695d9 100644 --- a/src/probabilistic/BloomFilter.cc +++ b/src/probabilistic/BloomFilter.cc @@ -46,12 +46,12 @@ broker::expected BloomFilter::Serialize() const std::unique_ptr BloomFilter::Unserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() == 3) ) return nullptr; - auto type = caf::get_if(&(*v)[0]); + auto type = broker::get_if(&(*v)[0]); if ( ! type ) return nullptr; diff --git a/src/probabilistic/CardinalityCounter.cc b/src/probabilistic/CardinalityCounter.cc index 0e9365b4d3..b203fa6642 100644 --- a/src/probabilistic/CardinalityCounter.cc +++ b/src/probabilistic/CardinalityCounter.cc @@ -212,13 +212,13 @@ broker::expected CardinalityCounter::Serialize() const std::unique_ptr CardinalityCounter::Unserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() >= 3) ) return nullptr; - auto m = caf::get_if(&(*v)[0]); - auto V = caf::get_if(&(*v)[1]); - auto alpha_m = caf::get_if(&(*v)[2]); + auto m = broker::get_if(&(*v)[0]); + auto V = broker::get_if(&(*v)[1]); + auto alpha_m = broker::get_if(&(*v)[2]); if ( ! (m && V && alpha_m) ) return nullptr; @@ -233,7 +233,7 @@ std::unique_ptr CardinalityCounter::Unserialize(const broker for ( size_t i = 0; i < *m; ++i ) { - auto x = caf::get_if(&(*v)[3 + i]); + auto x = broker::get_if(&(*v)[3 + i]); if ( ! x ) return nullptr; diff --git a/src/probabilistic/CounterVector.cc b/src/probabilistic/CounterVector.cc index 1272487ee9..7224411f62 100644 --- a/src/probabilistic/CounterVector.cc +++ b/src/probabilistic/CounterVector.cc @@ -169,11 +169,11 @@ broker::expected CounterVector::Serialize() const std::unique_ptr CounterVector::Unserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() >= 2) ) return nullptr; - auto width = caf::get_if(&(*v)[0]); + auto width = broker::get_if(&(*v)[0]); auto bits = BitVector::Unserialize((*v)[1]); if ( ! (width && bits) ) diff --git a/src/probabilistic/Hasher.cc b/src/probabilistic/Hasher.cc index 9935f31896..5b20222d10 100644 --- a/src/probabilistic/Hasher.cc +++ b/src/probabilistic/Hasher.cc @@ -60,15 +60,15 @@ broker::expected Hasher::Serialize() const std::unique_ptr Hasher::Unserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() == 4) ) return nullptr; - auto type = caf::get_if(&(*v)[0]); - auto k = caf::get_if(&(*v)[1]); - auto h1 = caf::get_if(&(*v)[2]); - auto h2 = caf::get_if(&(*v)[3]); + auto type = broker::get_if(&(*v)[0]); + auto k = broker::get_if(&(*v)[1]); + auto h1 = broker::get_if(&(*v)[2]); + auto h2 = broker::get_if(&(*v)[3]); if ( ! (type && k && h1 && h2) ) return nullptr; diff --git a/src/probabilistic/Topk.cc b/src/probabilistic/Topk.cc index 481499630e..bc2232335d 100644 --- a/src/probabilistic/Topk.cc +++ b/src/probabilistic/Topk.cc @@ -447,14 +447,14 @@ broker::expected TopkVal::DoSerialize() const bool TopkVal::DoUnserialize(const broker::data& data) { - auto v = caf::get_if(&data); + auto v = broker::get_if(&data); if ( ! (v && v->size() >= 4) ) return false; - auto size_ = caf::get_if(&(*v)[0]); - auto numElements_ = caf::get_if(&(*v)[1]); - auto pruned_ = caf::get_if(&(*v)[2]); + auto size_ = broker::get_if(&(*v)[0]); + auto numElements_ = broker::get_if(&(*v)[1]); + auto pruned_ = broker::get_if(&(*v)[2]); if ( ! (size_ && numElements_ && pruned_) ) return false; @@ -463,7 +463,7 @@ bool TopkVal::DoUnserialize(const broker::data& data) numElements = *numElements_; pruned = *pruned_; - auto no_type = caf::get_if(&(*v)[3]); + auto no_type = broker::get_if(&(*v)[3]); if ( ! no_type ) { auto t = UnserializeType((*v)[3]); @@ -479,8 +479,8 @@ bool TopkVal::DoUnserialize(const broker::data& data) while ( i < numElements ) { - auto elements_count = caf::get_if(&(*v)[idx++]); - auto count = caf::get_if(&(*v)[idx++]); + auto elements_count = broker::get_if(&(*v)[idx++]); + auto count = broker::get_if(&(*v)[idx++]); if ( ! (elements_count && count) ) return false; @@ -491,7 +491,7 @@ bool TopkVal::DoUnserialize(const broker::data& data) for ( uint64_t j = 0; j < *elements_count; j++ ) { - auto epsilon = caf::get_if(&(*v)[idx++]); + auto epsilon = broker::get_if(&(*v)[idx++]); auto val = Broker::detail::data_to_val((*v)[idx++], type.get()); if ( ! (epsilon && val) ) diff --git a/src/telemetry/CMakeLists.txt b/src/telemetry/CMakeLists.txt index c0c2153387..542b7a8528 100644 --- a/src/telemetry/CMakeLists.txt +++ b/src/telemetry/CMakeLists.txt @@ -6,11 +6,7 @@ include_directories(BEFORE ) set(telemetry_SRCS - Counter.cc - Gauge.cc - Histogram.cc Manager.cc - MetricFamily.cc ) bif_target(telemetry.bif) diff --git a/src/telemetry/Counter.cc b/src/telemetry/Counter.cc deleted file mode 100644 index 6ddc7ece5a..0000000000 --- a/src/telemetry/Counter.cc +++ /dev/null @@ -1,77 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek/telemetry/Counter.h" - -#include "zeek/telemetry/Detail.h" - -#include "caf/telemetry/counter.hpp" -#include "caf/telemetry/metric_family.hpp" -#include "caf/telemetry/metric_family_impl.hpp" - -namespace zeek::telemetry - { - -// -- IntCounter --------------------------------------------------------------- - -void IntCounter::Inc() noexcept - { - deref(pimpl).inc(); - } - -void IntCounter::Inc(int64_t amount) noexcept - { - deref(pimpl).inc(amount); - } - -int64_t IntCounter::operator++() noexcept - { - return ++deref(pimpl); - } - -int64_t IntCounter::Value() const noexcept - { - return deref(pimpl).value(); - } - -IntCounterFamily::IntCounterFamily(Impl* ptr) : MetricFamily(upcast(ptr)) { } - -IntCounter IntCounterFamily::GetOrAdd(Span labels) - { - return with_native_labels(labels, - [this](auto nativeLabels) - { - auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); - return IntCounter{hdl}; - }); - } - -// -- DblCounter --------------------------------------------------------------- - -void DblCounter::Inc() noexcept - { - deref(pimpl).inc(); - } - -void DblCounter::Inc(double amount) noexcept - { - deref(pimpl).inc(amount); - } - -double DblCounter::Value() const noexcept - { - return deref(pimpl).value(); - } - -DblCounterFamily::DblCounterFamily(Impl* ptr) : MetricFamily(upcast(ptr)) { } - -DblCounter DblCounterFamily::GetOrAdd(Span labels) - { - return with_native_labels(labels, - [this](auto nativeLabels) - { - auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); - return DblCounter{hdl}; - }); - } - - } // namespace zeek::telemetry diff --git a/src/telemetry/Counter.h b/src/telemetry/Counter.h index 126816b523..6f5e360a89 100644 --- a/src/telemetry/Counter.h +++ b/src/telemetry/Counter.h @@ -9,6 +9,8 @@ #include "zeek/Span.h" #include "zeek/telemetry/MetricFamily.h" +#include "broker/telemetry/fwd.hh" + namespace zeek::telemetry { @@ -24,8 +26,6 @@ class IntCounter public: friend class IntCounterFamily; - struct Impl; - static inline const char* OpaqueName = "IntCounterMetricVal"; IntCounter() = delete; @@ -35,34 +35,36 @@ public: /** * Increments the value by 1. */ - void Inc() noexcept; + void Inc() noexcept { broker::telemetry::inc(hdl); } /** * Increments the value by @p amount. * @pre `amount >= 0` */ - void Inc(int64_t amount) noexcept; + void Inc(int64_t amount) noexcept { broker::telemetry::inc(hdl, amount); } /** * Increments the value by 1. * @return The new value. */ - int64_t operator++() noexcept; + int64_t operator++() noexcept { return broker::telemetry::inc(hdl); } /** * @return The current value. */ - int64_t Value() const noexcept; + int64_t Value() const noexcept { return broker::telemetry::value(hdl); } /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(IntCounter other) const noexcept { return pimpl == other.pimpl; } + constexpr bool IsSameAs(IntCounter other) const noexcept { return hdl == other.hdl; } private: - explicit IntCounter(Impl* ptr) noexcept : pimpl(ptr) { } + using Handle = broker::telemetry::int_counter_hdl*; - Impl* pimpl; + explicit IntCounter(Handle hdl) noexcept : hdl(hdl) { } + + Handle hdl; }; /** @@ -89,8 +91,6 @@ class IntCounterFamily : public MetricFamily public: friend class Manager; - class Impl; - static inline const char* OpaqueName = "IntCounterMetricFamilyVal"; using InstanceType = IntCounter; @@ -102,7 +102,10 @@ public: * Returns the metrics handle for given labels, creating a new instance * lazily if necessary. */ - IntCounter GetOrAdd(Span labels); + IntCounter GetOrAdd(Span labels) + { + return IntCounter{int_counter_get_or_add(hdl, labels)}; + } /** * @copydoc GetOrAdd @@ -113,7 +116,9 @@ public: } private: - explicit IntCounterFamily(Impl* ptr); + using Handle = broker::telemetry::int_counter_family_hdl*; + + explicit IntCounterFamily(Handle hdl) : MetricFamily(upcast(hdl)) { } }; /** @@ -125,8 +130,6 @@ class DblCounter public: friend class DblCounterFamily; - struct Impl; - static inline const char* OpaqueName = "DblCounterMetricVal"; DblCounter() = delete; @@ -136,28 +139,30 @@ public: /** * Increments the value by 1. */ - void Inc() noexcept; + void Inc() noexcept { broker::telemetry::inc(hdl); } /** * Increments the value by @p amount. * @pre `amount >= 0` */ - void Inc(double amount) noexcept; + void Inc(double amount) noexcept { broker::telemetry::inc(hdl, amount); } /** * @return The current value. */ - double Value() const noexcept; + double Value() const noexcept { return broker::telemetry::value(hdl); } /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(DblCounter other) const noexcept { return pimpl == other.pimpl; } + constexpr bool IsSameAs(DblCounter other) const noexcept { return hdl == other.hdl; } private: - explicit DblCounter(Impl* ptr) noexcept : pimpl(ptr) { } + using Handle = broker::telemetry::dbl_counter_hdl*; - Impl* pimpl; + explicit DblCounter(Handle hdl) noexcept : hdl(hdl) { } + + Handle hdl; }; /** @@ -184,8 +189,6 @@ class DblCounterFamily : public MetricFamily public: friend class Manager; - class Impl; - static inline const char* OpaqueName = "DblCounterMetricFamilyVal"; using InstanceType = DblCounter; @@ -197,7 +200,10 @@ public: * Returns the metrics handle for given labels, creating a new instance * lazily if necessary. */ - DblCounter GetOrAdd(Span labels); + DblCounter GetOrAdd(Span labels) + { + return DblCounter{dbl_counter_get_or_add(hdl, labels)}; + } /** * @copydoc GetOrAdd @@ -208,7 +214,9 @@ public: } private: - explicit DblCounterFamily(Impl* ptr); + using Handle = broker::telemetry::dbl_counter_family_hdl*; + + explicit DblCounterFamily(Handle hdl) : MetricFamily(upcast(hdl)) { } }; namespace detail diff --git a/src/telemetry/Detail.h b/src/telemetry/Detail.h deleted file mode 100644 index 829041d7ad..0000000000 --- a/src/telemetry/Detail.h +++ /dev/null @@ -1,259 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -// This header contains private implementation details for telemetry classes -// and should not get included outside of .cc files. - -#pragma once - -#include -#include -#include - -#include "zeek/Span.h" -#include "zeek/telemetry/Counter.h" -#include "zeek/telemetry/Gauge.h" -#include "zeek/telemetry/Histogram.h" -#include "zeek/telemetry/Manager.h" -#include "zeek/telemetry/MetricFamily.h" - -#include "caf/telemetry/label_view.hpp" -#include "caf/telemetry/metric_family.hpp" - -namespace zeek::telemetry - { - -// -- traits for converting between opaque handles and native pointers --------- - -/** - * This trait must provide the member types @c Native for referring to the CAF - * type, @c Opaque for referring to the @c Impl type. For instance types such as - * @c IntCounter, the trait must also provide the member type @c NativeFamily. - */ -template struct PimplTrait; - -template <> struct PimplTrait - { - using Native = caf::telemetry::int_counter; - using Oqaque = IntCounter::Impl; - using NativeFamily = caf::telemetry::metric_family_impl; - }; - -template <> struct PimplTrait : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = typename PimplTrait::NativeFamily; - using Oqaque = IntCounterFamily::Impl; - }; - -template <> -struct PimplTrait::NativeFamily> - : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = caf::telemetry::dbl_counter; - using Oqaque = DblCounter::Impl; - using NativeFamily = caf::telemetry::metric_family_impl; - }; - -template <> struct PimplTrait : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = typename PimplTrait::NativeFamily; - using Oqaque = DblCounterFamily::Impl; - }; - -template <> -struct PimplTrait::NativeFamily> - : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = caf::telemetry::int_gauge; - using Oqaque = IntGauge::Impl; - using NativeFamily = caf::telemetry::metric_family_impl; - }; - -template <> struct PimplTrait : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = typename PimplTrait::NativeFamily; - using Oqaque = IntGaugeFamily::Impl; - }; - -template <> -struct PimplTrait::NativeFamily> - : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = caf::telemetry::dbl_gauge; - using Oqaque = DblGauge::Impl; - using NativeFamily = caf::telemetry::metric_family_impl; - }; - -template <> struct PimplTrait : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = typename PimplTrait::NativeFamily; - using Oqaque = DblGaugeFamily::Impl; - }; - -template <> -struct PimplTrait::NativeFamily> - : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = caf::telemetry::int_histogram; - using Oqaque = IntHistogram::Impl; - using NativeFamily = caf::telemetry::metric_family_impl; - }; - -template <> struct PimplTrait : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = typename PimplTrait::NativeFamily; - using Oqaque = IntHistogramFamily::Impl; - }; - -template <> -struct PimplTrait::NativeFamily> - : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = caf::telemetry::dbl_histogram; - using Oqaque = DblHistogram::Impl; - using NativeFamily = caf::telemetry::metric_family_impl; - }; - -template <> struct PimplTrait : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = typename PimplTrait::NativeFamily; - using Oqaque = DblHistogramFamily::Impl; - }; - -template <> -struct PimplTrait::NativeFamily> - : PimplTrait - { - }; - -template <> struct PimplTrait - { - using Native = caf::telemetry::metric_registry; - using Oqaque = Manager::Impl; - }; - -template <> -struct PimplTrait::Native> : PimplTrait - { - }; - -// -- free functions ----------------------------------------------------------- - -template ::Native> auto& deref(T* ptr) - { - return *reinterpret_cast(ptr); - } - -template auto& deref(Family*, MetricFamily::Impl* ptr) - { - using InstanceType = typename Family::InstanceType; - using ImplType = typename InstanceType::Impl; - using NativeType = typename PimplTrait::NativeFamily; - return *reinterpret_cast(ptr); - } - -template ::Oqaque> auto opaque(T* ptr) - { - return reinterpret_cast(ptr); - } - -template auto opaque(const Family*, MetricFamily::Impl* ptr) - { - using InstanceType = typename Family::InstanceType; - using ImplType = typename InstanceType::Impl; - using OpaqueType = typename PimplTrait::NativeFamily; - return reinterpret_cast(ptr); - } - -template ::Native> auto upcast(T* ptr) - { - auto native = reinterpret_cast(ptr); - auto base_ptr = static_cast(native); - return reinterpret_cast(base_ptr); - } - -template auto with_native_labels(Span xs, F continuation) - { - namespace ct = caf::telemetry; - - if ( xs.size() <= 10 ) - { - ct::label_view buf[10] = { - {{}, {}}, {{}, {}}, {{}, {}}, {{}, {}}, {{}, {}}, - {{}, {}}, {{}, {}}, {{}, {}}, {{}, {}}, {{}, {}}, - }; - for ( size_t index = 0; index < xs.size(); ++index ) - buf[index] = ct::label_view{xs[index].first, xs[index].second}; - return continuation(Span{buf, xs.size()}); - } - else - { - std::vector buf; - for ( auto x : xs ) - buf.emplace_back(x.first, x.second); - return continuation(Span{buf}); - } - } - -template auto with_native_labels(Span xs, F continuation) - { - if ( xs.size() <= 10 ) - { - caf::string_view buf[10]; - for ( size_t index = 0; index < xs.size(); ++index ) - buf[index] = xs[index]; - return continuation(Span{buf, xs.size()}); - } - else - { - std::vector buf; - for ( auto x : xs ) - buf.emplace_back(x); - return continuation(Span{buf}); - } - } - - } // namespace zeek::telemetry diff --git a/src/telemetry/Gauge.cc b/src/telemetry/Gauge.cc deleted file mode 100644 index ad6ed5d6ea..0000000000 --- a/src/telemetry/Gauge.cc +++ /dev/null @@ -1,102 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek/telemetry/Gauge.h" - -#include "zeek/telemetry/Detail.h" - -#include "caf/telemetry/gauge.hpp" -#include "caf/telemetry/metric_family.hpp" -#include "caf/telemetry/metric_family_impl.hpp" - -namespace zeek::telemetry - { - -// -- IntGauge --------------------------------------------------------------- - -void IntGauge::Inc() noexcept - { - deref(pimpl).inc(); - } - -void IntGauge::Inc(int64_t amount) noexcept - { - deref(pimpl).inc(amount); - } - -void IntGauge::Dec() noexcept - { - deref(pimpl).dec(); - } - -void IntGauge::Dec(int64_t amount) noexcept - { - deref(pimpl).dec(amount); - } - -int64_t IntGauge::operator++() noexcept - { - return ++deref(pimpl); - } - -int64_t IntGauge::operator--() noexcept - { - return --deref(pimpl); - } - -int64_t IntGauge::Value() const noexcept - { - return deref(pimpl).value(); - } - -IntGaugeFamily::IntGaugeFamily(Impl* ptr) : MetricFamily(upcast(ptr)) { } - -IntGauge IntGaugeFamily::GetOrAdd(Span labels) - { - return with_native_labels(labels, - [this](auto nativeLabels) - { - auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); - return IntGauge{hdl}; - }); - } - -// -- DblGauge --------------------------------------------------------------- - -void DblGauge::Inc() noexcept - { - deref(pimpl).inc(); - } - -void DblGauge::Inc(double amount) noexcept - { - deref(pimpl).inc(amount); - } - -void DblGauge::Dec() noexcept - { - deref(pimpl).dec(); - } - -void DblGauge::Dec(double amount) noexcept - { - deref(pimpl).dec(amount); - } - -double DblGauge::Value() const noexcept - { - return deref(pimpl).value(); - } - -DblGaugeFamily::DblGaugeFamily(Impl* ptr) : MetricFamily(upcast(ptr)) { } - -DblGauge DblGaugeFamily::GetOrAdd(Span labels) - { - return with_native_labels(labels, - [this](auto nativeLabels) - { - auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); - return DblGauge{hdl}; - }); - } - - } // namespace zeek::telemetry diff --git a/src/telemetry/Gauge.h b/src/telemetry/Gauge.h index f072178d98..e0af13f2c5 100644 --- a/src/telemetry/Gauge.h +++ b/src/telemetry/Gauge.h @@ -9,6 +9,8 @@ #include "zeek/Span.h" #include "zeek/telemetry/MetricFamily.h" +#include "broker/telemetry/fwd.hh" + namespace zeek::telemetry { @@ -17,7 +19,7 @@ class IntGaugeFamily; class Manager; /** - * A handle to a metric that represents an integer value. Gauges are less + * A handle to a metric that represents an integer value. Gauges are more * permissive than counters and also allow decrementing the value. */ class IntGauge @@ -25,8 +27,6 @@ class IntGauge public: friend class IntGaugeFamily; - struct Impl; - static inline const char* OpaqueName = "IntGaugeMetricVal"; IntGauge() = delete; @@ -36,49 +36,51 @@ public: /** * Increments the value by 1. */ - void Inc() noexcept; + void Inc() noexcept { broker::telemetry::inc(hdl); } /** * Increments the value by @p amount. */ - void Inc(int64_t amount) noexcept; + void Inc(int64_t amount) noexcept { broker::telemetry::inc(hdl, amount); } /** * Increments the value by 1. * @return The new value. */ - int64_t operator++() noexcept; + int64_t operator++() noexcept { return broker::telemetry::inc(hdl); } /** * Decrements the value by 1. */ - void Dec() noexcept; + void Dec() noexcept { broker::telemetry::dec(hdl); } /** * Decrements the value by @p amount. */ - void Dec(int64_t amount) noexcept; + void Dec(int64_t amount) noexcept { broker::telemetry::dec(hdl, amount); } /** * Decrements the value by 1. * @return The new value. */ - int64_t operator--() noexcept; + int64_t operator--() noexcept { return broker::telemetry::dec(hdl); } /** * @return The current value. */ - int64_t Value() const noexcept; + int64_t Value() const noexcept { return broker::telemetry::value(hdl); } /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(IntGauge other) const noexcept { return pimpl == other.pimpl; } + constexpr bool IsSameAs(IntGauge other) const noexcept { return hdl == other.hdl; } private: - explicit IntGauge(Impl* ptr) noexcept : pimpl(ptr) { } + using Handle = broker::telemetry::int_gauge_hdl*; - Impl* pimpl; + explicit IntGauge(Handle hdl) noexcept : hdl(hdl) { } + + Handle hdl; }; /** @@ -105,8 +107,6 @@ class IntGaugeFamily : public MetricFamily public: friend class Manager; - class Impl; - static inline const char* OpaqueName = "IntGaugeMetricFamilyVal"; using InstanceType = IntGauge; @@ -118,7 +118,10 @@ public: * Returns the metrics handle for given labels, creating a new instance * lazily if necessary. */ - IntGauge GetOrAdd(Span labels); + IntGauge GetOrAdd(Span labels) + { + return IntGauge{int_gauge_get_or_add(hdl, labels)}; + } /** * @copydoc GetOrAdd @@ -129,21 +132,20 @@ public: } private: - explicit IntGaugeFamily(Impl* ptr); + using Handle = broker::telemetry::int_gauge_family_hdl*; + + explicit IntGaugeFamily(Handle hdl) : MetricFamily(upcast(hdl)) { } }; /** - * A handle to a metric that represents a floating point value. Gauges are less + * A handle to a metric that represents a floating point value. Gauges are more * permissive than counters and also allow decrementing the value. - * up. */ class DblGauge { public: friend class DblGaugeFamily; - struct Impl; - static inline const char* OpaqueName = "DblGaugeMetricVal"; DblGauge() = delete; @@ -153,37 +155,39 @@ public: /** * Increments the value by 1. */ - void Inc() noexcept; + void Inc() noexcept { broker::telemetry::inc(hdl); } /** * Increments the value by @p amount. */ - void Inc(double amount) noexcept; + void Inc(double amount) noexcept { broker::telemetry::inc(hdl, amount); } /** * Increments the value by 1. */ - void Dec() noexcept; + void Dec() noexcept { broker::telemetry::dec(hdl); } /** * Increments the value by @p amount. */ - void Dec(double amount) noexcept; + void Dec(double amount) noexcept { broker::telemetry::dec(hdl, amount); } /** * @return The current value. */ - double Value() const noexcept; + double Value() const noexcept { return broker::telemetry::value(hdl); } /** * @return Whether @c this and @p other refer to the same counter. */ - constexpr bool IsSameAs(DblGauge other) const noexcept { return pimpl == other.pimpl; } + constexpr bool IsSameAs(DblGauge other) const noexcept { return hdl == other.hdl; } private: - explicit DblGauge(Impl* ptr) noexcept : pimpl(ptr) { } + using Handle = broker::telemetry::dbl_gauge_hdl*; - Impl* pimpl; + explicit DblGauge(Handle hdl) noexcept : hdl(hdl) { } + + Handle hdl; }; /** @@ -210,8 +214,6 @@ class DblGaugeFamily : public MetricFamily public: friend class Manager; - class Impl; - static inline const char* OpaqueName = "DblGaugeMetricFamilyVal"; using InstanceType = DblGauge; @@ -223,7 +225,10 @@ public: * Returns the metrics handle for given labels, creating a new instance * lazily if necessary. */ - DblGauge GetOrAdd(Span labels); + DblGauge GetOrAdd(Span labels) + { + return DblGauge{dbl_gauge_get_or_add(hdl, labels)}; + } /** * @copydoc GetOrAdd @@ -234,7 +239,9 @@ public: } private: - explicit DblGaugeFamily(Impl* ptr); + using Handle = broker::telemetry::dbl_gauge_family_hdl*; + + explicit DblGaugeFamily(Handle hdl) : MetricFamily(upcast(hdl)) { } }; namespace detail diff --git a/src/telemetry/Histogram.cc b/src/telemetry/Histogram.cc deleted file mode 100644 index e6f95a55f3..0000000000 --- a/src/telemetry/Histogram.cc +++ /dev/null @@ -1,102 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek/telemetry/Histogram.h" - -#include - -#include "zeek/telemetry/Detail.h" - -#include "caf/telemetry/histogram.hpp" -#include "caf/telemetry/metric_family.hpp" -#include "caf/telemetry/metric_family_impl.hpp" - -namespace zeek::telemetry - { - -// -- IntHistogram --------------------------------------------------------------- - -void IntHistogram::Observe(int64_t value) noexcept - { - deref(pimpl).observe(value); - } - -int64_t IntHistogram::Sum() const noexcept - { - return deref(pimpl).sum(); - } - -size_t IntHistogram::NumBuckets() const noexcept - { - return deref(pimpl).buckets().size(); - } - -int64_t IntHistogram::CountAt(size_t index) const noexcept - { - auto xs = deref(pimpl).buckets(); - assert(index < xs.size()); - return xs[index].count.value(); - } - -int64_t IntHistogram::UpperBoundAt(size_t index) const noexcept - { - auto xs = deref(pimpl).buckets(); - assert(index < xs.size()); - return xs[index].upper_bound; - } - -IntHistogramFamily::IntHistogramFamily(Impl* ptr) : MetricFamily(upcast(ptr)) { } - -IntHistogram IntHistogramFamily::GetOrAdd(Span labels) - { - return with_native_labels(labels, - [this](auto nativeLabels) - { - auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); - return IntHistogram{hdl}; - }); - } - -// -- DblHistogram --------------------------------------------------------------- - -void DblHistogram::Observe(double amount) noexcept - { - deref(pimpl).observe(amount); - } - -double DblHistogram::Sum() const noexcept - { - return deref(pimpl).sum(); - } - -size_t DblHistogram::NumBuckets() const noexcept - { - return deref(pimpl).buckets().size(); - } - -int64_t DblHistogram::CountAt(size_t index) const noexcept - { - auto xs = deref(pimpl).buckets(); - assert(index < xs.size()); - return xs[index].count.value(); - } - -double DblHistogram::UpperBoundAt(size_t index) const noexcept - { - auto xs = deref(pimpl).buckets(); - assert(index < xs.size()); - return xs[index].upper_bound; - } - -DblHistogramFamily::DblHistogramFamily(Impl* ptr) : MetricFamily(upcast(ptr)) { } - -DblHistogram DblHistogramFamily::GetOrAdd(Span labels) - { - return with_native_labels(labels, - [this](auto nativeLabels) - { - auto hdl = opaque(deref(this, pimpl).get_or_add(nativeLabels)); - return DblHistogram{hdl}; - }); - } - - } // namespace zeek::telemetry diff --git a/src/telemetry/Histogram.h b/src/telemetry/Histogram.h index 0097142bc2..3b69ef3af3 100644 --- a/src/telemetry/Histogram.h +++ b/src/telemetry/Histogram.h @@ -9,6 +9,8 @@ #include "zeek/Span.h" #include "zeek/telemetry/MetricFamily.h" +#include "broker/telemetry/fwd.hh" + namespace zeek::telemetry { @@ -26,8 +28,6 @@ class IntHistogram public: friend class IntHistogramFamily; - struct Impl; - static inline const char* OpaqueName = "IntHistogramMetricVal"; IntHistogram() = delete; @@ -38,31 +38,33 @@ public: * Increments all buckets with an upper bound less than or equal to @p value * by one and adds @p value to the total sum of all observed values. */ - void Observe(int64_t value) noexcept; + void Observe(int64_t value) noexcept { return broker::telemetry::observe(hdl, value); } /// @return The sum of all observed values. - int64_t Sum() const noexcept; + int64_t Sum() const noexcept { return broker::telemetry::sum(hdl); } /// @return The number of buckets, including the implicit "infinite" bucket. - size_t NumBuckets() const noexcept; + size_t NumBuckets() const noexcept { return broker::telemetry::num_buckets(hdl); } /// @return The number of observations in the bucket at @p index. /// @pre index < NumBuckets() - int64_t CountAt(size_t index) const noexcept; + int64_t CountAt(size_t index) const noexcept { return broker::telemetry::count_at(hdl, index); } /// @return The upper bound of the bucket at @p index. /// @pre index < NumBuckets() - int64_t UpperBoundAt(size_t index) const noexcept; + int64_t UpperBoundAt(size_t index) const noexcept { return broker::telemetry::upper_bound_at(hdl, index); } /** * @return Whether @c this and @p other refer to the same histogram. */ - constexpr bool IsSameAs(IntHistogram other) const noexcept { return pimpl == other.pimpl; } + constexpr bool IsSameAs(IntHistogram other) const noexcept { return hdl == other.hdl; } private: - explicit IntHistogram(Impl* ptr) noexcept : pimpl(ptr) { } + using Handle = broker::telemetry::int_histogram_hdl*; - Impl* pimpl; + explicit IntHistogram(Handle hdl) noexcept : hdl(hdl) { } + + Handle hdl; }; /** @@ -88,8 +90,6 @@ class IntHistogramFamily : public MetricFamily public: friend class Manager; - class Impl; - static inline const char* OpaqueName = "IntHistogramMetricFamilyVal"; using InstanceType = IntHistogram; @@ -101,7 +101,10 @@ public: * Returns the metrics handle for given labels, creating a new instance * lazily if necessary. */ - IntHistogram GetOrAdd(Span labels); + IntHistogram GetOrAdd(Span labels) + { + return IntHistogram{int_histogram_get_or_add(hdl, labels)}; + } /** * @copydoc GetOrAdd @@ -112,7 +115,9 @@ public: } private: - explicit IntHistogramFamily(Impl* ptr); + using Handle = broker::telemetry::int_histogram_family_hdl*; + + explicit IntHistogramFamily(Handle hdl) : MetricFamily(upcast(hdl)) { } }; /** @@ -125,8 +130,6 @@ class DblHistogram public: friend class DblHistogramFamily; - struct Impl; - static inline const char* OpaqueName = "DblHistogramMetricVal"; DblHistogram() = delete; @@ -137,31 +140,36 @@ public: * Increments all buckets with an upper bound less than or equal to @p value * by one and adds @p value to the total sum of all observed values. */ - void Observe(double value) noexcept; + void Observe(double value) noexcept { broker::telemetry::observe(hdl, value); } /// @return The sum of all observed values. - double Sum() const noexcept; + double Sum() const noexcept { return broker::telemetry::sum(hdl); } /// @return The number of buckets, including the implicit "infinite" bucket. - size_t NumBuckets() const noexcept; + size_t NumBuckets() const noexcept { return broker::telemetry::num_buckets(hdl); } /// @return The number of observations in the bucket at @p index. /// @pre index < NumBuckets() - int64_t CountAt(size_t index) const noexcept; + int64_t CountAt(size_t index) const noexcept { return broker::telemetry::count_at(hdl, index); } /// @return The upper bound of the bucket at @p index. /// @pre index < NumBuckets() - double UpperBoundAt(size_t index) const noexcept; + double UpperBoundAt(size_t index) const noexcept + { + return broker::telemetry::upper_bound_at(hdl, index); + } /** * @return Whether @c this and @p other refer to the same histogram. */ - constexpr bool IsSameAs(DblHistogram other) const noexcept { return pimpl == other.pimpl; } + constexpr bool IsSameAs(DblHistogram other) const noexcept { return hdl == other.hdl; } private: - explicit DblHistogram(Impl* ptr) noexcept : pimpl(ptr) { } + using Handle = broker::telemetry::dbl_histogram_hdl*; - Impl* pimpl; + explicit DblHistogram(Handle hdl) noexcept : hdl(hdl) { } + + Handle hdl; }; /** @@ -187,8 +195,6 @@ class DblHistogramFamily : public MetricFamily public: friend class Manager; - class Impl; - static inline const char* OpaqueName = "DblHistogramMetricFamilyVal"; using InstanceType = DblHistogram; @@ -200,7 +206,10 @@ public: * Returns the metrics handle for given labels, creating a new instance * lazily if necessary. */ - DblHistogram GetOrAdd(Span labels); + DblHistogram GetOrAdd(Span labels) + { + return DblHistogram{dbl_histogram_get_or_add(hdl, labels)}; + } /** * @copydoc GetOrAdd @@ -211,7 +220,9 @@ public: } private: - explicit DblHistogramFamily(Impl* ptr); + using Handle = broker::telemetry::dbl_histogram_family_hdl*; + + explicit DblHistogramFamily(Handle hdl) : MetricFamily(upcast(hdl)) { } }; namespace detail diff --git a/src/telemetry/Manager.cc b/src/telemetry/Manager.cc index d7eac6904e..66fb3d5b94 100644 --- a/src/telemetry/Manager.cc +++ b/src/telemetry/Manager.cc @@ -5,102 +5,37 @@ #include #include "zeek/3rdparty/doctest.h" -#include "zeek/telemetry/Detail.h" +#include "zeek/broker/Manager.h" #include "zeek/telemetry/Timer.h" -#include "caf/telemetry/metric_registry.hpp" +#include "broker/telemetry/metric_registry.hh" + +namespace + { +using NativeManager = broker::telemetry::metric_registry; +using NativeManagerImpl = broker::telemetry::metric_registry_impl; +using NativeManagerImplPtr = zeek::IntrusivePtr; + } namespace zeek::telemetry { +Manager::Manager() + { + auto reg = NativeManager::pre_init_instance(); + NativeManagerImplPtr ptr{NewRef{}, reg.pimpl()}; + pimpl.swap(ptr); + } + Manager::~Manager() { } void Manager::InitPostScript() { } -IntCounterFamily Manager::IntCounterFam(std::string_view prefix, std::string_view name, - Span labels, - std::string_view helptext, std::string_view unit, - bool is_sum) +void Manager::InitPostBrokerSetup(broker::endpoint& ep) { - return with_native_labels(labels, - [&, this](auto xs) - { - auto ptr = deref(pimpl).counter_family(prefix, name, xs, helptext, - unit, is_sum); - return IntCounterFamily{opaque(ptr)}; - }); - } - -DblCounterFamily Manager::DblCounterFam(std::string_view prefix, std::string_view name, - Span labels, - std::string_view helptext, std::string_view unit, - bool is_sum) - { - return with_native_labels(labels, - [&, this](auto xs) - { - auto ptr = deref(pimpl).counter_family( - prefix, name, xs, helptext, unit, is_sum); - return DblCounterFamily{opaque(ptr)}; - }); - } - -IntGaugeFamily Manager::IntGaugeFam(std::string_view prefix, std::string_view name, - Span labels, std::string_view helptext, - std::string_view unit, bool is_sum) - { - return with_native_labels(labels, - [&, this](auto xs) - { - auto ptr = deref(pimpl).gauge_family(prefix, name, xs, helptext, - unit, is_sum); - return IntGaugeFamily{opaque(ptr)}; - }); - } - -DblGaugeFamily Manager::DblGaugeFam(std::string_view prefix, std::string_view name, - Span labels, std::string_view helptext, - std::string_view unit, bool is_sum) - { - return with_native_labels(labels, - [&, this](auto xs) - { - auto ptr = deref(pimpl).gauge_family( - prefix, name, xs, helptext, unit, is_sum); - return DblGaugeFamily{opaque(ptr)}; - }); - } - -IntHistogramFamily Manager::IntHistoFam(std::string_view prefix, std::string_view name, - Span labels, - Span ubounds, std::string_view helptext, - std::string_view unit, bool is_sum) - { - return with_native_labels( - labels, - [&, this](auto xs) - { - auto bounds = caf::span{ubounds.data(), ubounds.size()}; - auto ptr = deref(pimpl).histogram_family(prefix, name, xs, bounds, helptext, unit, - is_sum); - return IntHistogramFamily{opaque(ptr)}; - }); - } - -DblHistogramFamily Manager::DblHistoFam(std::string_view prefix, std::string_view name, - Span labels, - Span ubounds, std::string_view helptext, - std::string_view unit, bool is_sum) - { - return with_native_labels( - labels, - [&, this](auto xs) - { - auto bounds = caf::span{ubounds.data(), ubounds.size()}; - auto ptr = deref(pimpl).histogram_family(prefix, name, xs, bounds, helptext, - unit, is_sum); - return DblHistogramFamily{opaque(ptr)}; - }); + auto reg = NativeManager::merge(NativeManager{pimpl.get()}, ep); + NativeManagerImplPtr ptr{NewRef{}, reg.pimpl()}; + pimpl.swap(ptr); } } // namespace zeek::telemetry @@ -110,8 +45,6 @@ DblHistogramFamily Manager::DblHistoFam(std::string_view prefix, std::string_vie using namespace std::literals; using namespace zeek::telemetry; -using NativeManager = caf::telemetry::metric_registry; - namespace { @@ -129,8 +62,7 @@ SCENARIO("telemetry managers provide access to counter singletons") { GIVEN("a telemetry manager") { - NativeManager native_mgr; - Manager mgr{opaque(&native_mgr)}; + Manager mgr; WHEN("retrieving an IntCounter singleton") { auto first = mgr.CounterSingleton("zeek", "int-count", "test"); @@ -184,8 +116,7 @@ SCENARIO("telemetry managers provide access to counter families") { GIVEN("a telemetry manager") { - NativeManager native_mgr; - Manager mgr{opaque(&native_mgr)}; + Manager mgr; WHEN("retrieving an IntCounter family") { auto family = mgr.CounterFamily("zeek", "requests", {"method"}, "test", "1", true); @@ -244,8 +175,7 @@ SCENARIO("telemetry managers provide access to gauge singletons") { GIVEN("a telemetry manager") { - NativeManager native_mgr; - Manager mgr{opaque(&native_mgr)}; + Manager mgr; WHEN("retrieving an IntGauge singleton") { auto first = mgr.GaugeSingleton("zeek", "int-gauge", "test"); @@ -309,8 +239,7 @@ SCENARIO("telemetry managers provide access to gauge families") { GIVEN("a telemetry manager") { - NativeManager native_mgr; - Manager mgr{opaque(&native_mgr)}; + Manager mgr; WHEN("retrieving an IntGauge family") { auto family = mgr.GaugeFamily("zeek", "open-connections", {"protocol"}, "test"); @@ -369,8 +298,7 @@ SCENARIO("telemetry managers provide access to histogram singletons") { GIVEN("a telemetry manager") { - NativeManager native_mgr; - Manager mgr{opaque(&native_mgr)}; + Manager mgr; WHEN("retrieving an IntHistogram singleton") { const auto max_int = std::numeric_limits::max(); @@ -456,8 +384,7 @@ SCENARIO("telemetry managers provide access to histogram families") { GIVEN("a telemetry manager") { - NativeManager native_mgr; - Manager mgr{opaque(&native_mgr)}; + Manager mgr; WHEN("retrieving an IntHistogram family") { int64_t buckets[] = {10, 20}; diff --git a/src/telemetry/Manager.h b/src/telemetry/Manager.h index 1643fe0fa7..9158ab667c 100644 --- a/src/telemetry/Manager.h +++ b/src/telemetry/Manager.h @@ -7,11 +7,24 @@ #include #include +#include "zeek/IntrusivePtr.h" #include "zeek/Span.h" #include "zeek/telemetry/Counter.h" #include "zeek/telemetry/Gauge.h" #include "zeek/telemetry/Histogram.h" +#include "broker/telemetry/fwd.hh" + +namespace broker + { +class endpoint; + } + +namespace zeek::Broker + { +class Manager; + } + namespace zeek::telemetry { @@ -21,9 +34,9 @@ namespace zeek::telemetry class Manager { public: - class Impl; + friend class Broker::Manager; - explicit Manager(Impl* ptr) : pimpl(ptr) { } + Manager(); Manager(const Manager&) = delete; @@ -54,13 +67,15 @@ public: { if constexpr ( std::is_same::value ) { - return IntCounterFam(prefix, name, labels, helptext, unit, is_sum); + auto fam = int_counter_fam(ptr(), prefix, name, labels, helptext, unit, is_sum); + return IntCounterFamily{fam}; } else { static_assert(std::is_same::value, "metrics only support int64_t and double values"); - return DblCounterFam(prefix, name, labels, helptext, unit, is_sum); + auto fam = dbl_counter_fam(ptr(), prefix, name, labels, helptext, unit, is_sum); + return DblCounterFamily{fam}; } } @@ -148,13 +163,15 @@ public: { if constexpr ( std::is_same::value ) { - return IntGaugeFam(prefix, name, labels, helptext, unit, is_sum); + auto fam = int_gauge_fam(ptr(), prefix, name, labels, helptext, unit, is_sum); + return IntGaugeFamily{fam}; } else { static_assert(std::is_same::value, "metrics only support int64_t and double values"); - return DblGaugeFam(prefix, name, labels, helptext, unit, is_sum); + auto fam = dbl_gauge_fam(ptr(), prefix, name, labels, helptext, unit, is_sum); + return DblGaugeFamily{fam}; } } @@ -264,13 +281,17 @@ public: { if constexpr ( std::is_same::value ) { - return IntHistoFam(prefix, name, labels, default_upper_bounds, helptext, unit, is_sum); + auto fam = int_histogram_fam(ptr(), prefix, name, labels, default_upper_bounds, + helptext, unit, is_sum); + return IntHistogramFamily{fam}; } else { static_assert(std::is_same::value, "metrics only support int64_t and double values"); - return DblHistoFam(prefix, name, labels, default_upper_bounds, helptext, unit, is_sum); + auto fam = dbl_histogram_fam(ptr(), prefix, name, labels, default_upper_bounds, + helptext, unit, is_sum); + return DblHistogramFamily{fam}; } } @@ -368,30 +389,6 @@ public: } protected: - IntCounterFamily IntCounterFam(std::string_view prefix, std::string_view name, - Span labels, std::string_view helptext, - std::string_view unit, bool is_sum); - - DblCounterFamily DblCounterFam(std::string_view prefix, std::string_view name, - Span labels, std::string_view helptext, - std::string_view unit, bool is_sum); - - IntGaugeFamily IntGaugeFam(std::string_view prefix, std::string_view name, - Span labels, std::string_view helptext, - std::string_view unit, bool is_sum); - - DblGaugeFamily DblGaugeFam(std::string_view prefix, std::string_view name, - Span labels, std::string_view helptext, - std::string_view unit, bool is_sum); - - IntHistogramFamily IntHistoFam(std::string_view prefix, std::string_view name, - Span labels, Span ubounds, - std::string_view helptext, std::string_view unit, bool is_sum); - - DblHistogramFamily DblHistoFam(std::string_view prefix, std::string_view name, - Span labels, Span ubounds, - std::string_view helptext, std::string_view unit, bool is_sum); - template static void WithLabelNames(Span xs, F continuation) { if ( xs.size() <= 10 ) @@ -410,7 +407,13 @@ protected: } } - Impl* pimpl; + broker::telemetry::metric_registry_impl* ptr() { return pimpl.get(); } + + // Connects all the dots after the Broker Manager constructed the endpoint + // for this Zeek instance. Called from Broker::Manager::InitPostScript(). + void InitPostBrokerSetup(broker::endpoint&); + + IntrusivePtr pimpl; }; } // namespace zeek::telemetry diff --git a/src/telemetry/MetricFamily.cc b/src/telemetry/MetricFamily.cc deleted file mode 100644 index cdd7f52e4e..0000000000 --- a/src/telemetry/MetricFamily.cc +++ /dev/null @@ -1,54 +0,0 @@ -// See the file "COPYING" in the main distribution directory for copyright. - -#include "zeek/telemetry/MetricFamily.h" - -#include "caf/telemetry/metric_family.hpp" - -namespace zeek::telemetry - { - -namespace - { - -namespace ct = caf::telemetry; - -using NativeMetricFamily = ct::metric_family; - -auto& deref(MetricFamily::Impl* ptr) - { - return *reinterpret_cast(ptr); - } - - } // namespace - -std::string_view MetricFamily::Prefix() const noexcept - { - return deref(pimpl).prefix(); - } - -std::string_view MetricFamily::Name() const noexcept - { - return deref(pimpl).name(); - } - -Span MetricFamily::LabelNames() const noexcept - { - return deref(pimpl).label_names(); - } - -std::string_view MetricFamily::Helptext() const noexcept - { - return deref(pimpl).helptext(); - } - -std::string_view MetricFamily::Unit() const noexcept - { - return deref(pimpl).unit(); - } - -bool MetricFamily::IsSum() const noexcept - { - return deref(pimpl).is_sum(); - } - - } // namespace zeek::telemetry diff --git a/src/telemetry/MetricFamily.h b/src/telemetry/MetricFamily.h index 6ea546d5d3..7fdf65fd43 100644 --- a/src/telemetry/MetricFamily.h +++ b/src/telemetry/MetricFamily.h @@ -8,6 +8,8 @@ #include "zeek/Span.h" +#include "broker/telemetry/metric_family.hh" + namespace zeek::telemetry { @@ -23,8 +25,6 @@ using LabelView = std::pair; class MetricFamily { public: - struct Impl; - MetricFamily() = delete; MetricFamily(const MetricFamily&) noexcept = default; MetricFamily& operator=(const MetricFamily&) noexcept = default; @@ -35,42 +35,47 @@ public: * script, may use a prefix that represents the application/script * or protocol (e.g. @c http) name. */ - std::string_view Prefix() const noexcept; + std::string_view Prefix() const noexcept { return broker::telemetry::prefix(hdl); } /** * @return The human-readable name of the metric, e.g., * @p open-connections. */ - std::string_view Name() const noexcept; + std::string_view Name() const noexcept { return broker::telemetry::name(hdl); } /** * @return The names for all label dimensions. */ - Span LabelNames() const noexcept; + Span LabelNames() const noexcept + { + return broker::telemetry::label_names(hdl); + } /** * @return A short explanation of the metric. */ - std::string_view Helptext() const noexcept; + std::string_view Helptext() const noexcept { return broker::telemetry::helptext(hdl); } /** * @return The unit of measurement, preferably a base unit such as * @c bytes or @c seconds. Dimensionless counts return the * pseudo-unit @c 1. */ - std::string_view Unit() const noexcept; + std::string_view Unit() const noexcept { return broker::telemetry::unit(hdl); } /** * @return Whether metrics of this family accumulate values, where only the * total value is of interest. For example, the total number of * HTTP requests. */ - bool IsSum() const noexcept; + bool IsSum() const noexcept { return broker::telemetry::is_sum(hdl); } protected: - explicit MetricFamily(Impl* ptr) : pimpl(ptr) { } + using Handle = broker::telemetry::metric_family_hdl*; - Impl* pimpl; + explicit MetricFamily(Handle hdl) : hdl(hdl) { } + + Handle hdl; }; } // namespace zeek::telemetry diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 871ced8e53..7a5fd82d2d 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -579,7 +579,7 @@ SetupResult setup(int argc, char** argv, Options* zopts) file_mgr = new file_analysis::Manager(); auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; broker_mgr = new Broker::Manager(broker_real_time); - telemetry_mgr = broker_mgr->NewTelemetryManager().release(); + telemetry_mgr = new telemetry::Manager; trigger_mgr = new trigger::Manager(); plugin_mgr->InitPreScript(); From da2a5ef4558b870a394ea5470fdc932011deb1d3 Mon Sep 17 00:00:00 2001 From: Dominik Charousset Date: Tue, 21 Dec 2021 11:02:26 +0100 Subject: [PATCH 2/2] Fix GCC builds and string output for Broker errors --- auxil/broker | 2 +- src/broker/Manager.cc | 53 +++++++++++++++++++++++++++++++++- src/probabilistic/BitVector.cc | 1 + 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/auxil/broker b/auxil/broker index ce321c0a90..0b2691392f 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit ce321c0a90f1d4176796c996fd27fe69ea520316 +Subproject commit 0b2691392f335d409781257f210c7b1681530c59 diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index babf719d3e..6fc8327d51 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -28,6 +28,57 @@ using namespace std; +namespace { + +void print_escaped(std::string& buf, std::string_view str) + { + buf.push_back('"'); + for ( auto c : str ) + { + switch ( c ) + { + default: + buf.push_back(c); + break; + case '\\': + buf.push_back('\\'); + buf.push_back('\\'); + break; + case '\b': + buf.push_back('\\'); + buf.push_back('b'); + break; + case '\f': + buf.push_back('\\'); + buf.push_back('f'); + break; + case '\n': + buf.push_back('\\'); + buf.push_back('n'); + break; + case '\r': + buf.push_back('\\'); + buf.push_back('r'); + break; + case '\t': + buf.push_back('\\'); + buf.push_back('t'); + break; + case '\v': + buf.push_back('\\'); + buf.push_back('v'); + break; + case '"': + buf.push_back('\\'); + buf.push_back('"'); + break; + } + } + buf.push_back('"'); + } + + } // namespace + namespace zeek::Broker { @@ -1658,7 +1709,7 @@ void Manager::ProcessError(broker::error_view err) msg += broker::to_string(ctx->network); msg += ", "; if ( auto what = err.message() ) - msg += *what; + print_escaped(msg, *what); else msg += R"_("")_"; msg += ')'; diff --git a/src/probabilistic/BitVector.cc b/src/probabilistic/BitVector.cc index f18915a77f..d724a54e7f 100644 --- a/src/probabilistic/BitVector.cc +++ b/src/probabilistic/BitVector.cc @@ -5,6 +5,7 @@ #include #include #include +#include #include #include "zeek/digest.h"