From fc96c81c707b430901369a932bee2ad27e24628c Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 7 Apr 2025 10:08:05 +0200 Subject: [PATCH 1/9] EventMgr: Add CurrentEvent() accessor Avoid proliferation of accessors on EventMgr. --- src/Event.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Event.h b/src/Event.h index 5639fea96f..a5229b3e10 100644 --- a/src/Event.h +++ b/src/Event.h @@ -116,6 +116,11 @@ public: // by returning a zero-timeout. double GetNextTimeout() override { return head ? 0.0 : -1.0; } + /** + * @return A pointer to the currently dispatched event or nullptr. + */ + const Event* CurrentEvent() const { return current; } + void Process() override; const char* Tag() override { return "EventManager"; } void InitPostScript(); From cc7dc60c1ecd2d1d19f898ef78dfd565bd0bee0a Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 7 Apr 2025 10:03:26 +0200 Subject: [PATCH 2/9] EventRegistry/zeek.bif/init-bare: Add event metadata infrastructure Introduce a new EventMetadata module and members on EventMgr to register event metadata types. --- scripts/base/init-bare.zeek | 18 +++++ src/Event.cc | 16 ++++- src/EventRegistry.cc | 68 +++++++++++++++++++ src/EventRegistry.h | 58 ++++++++++++++++ src/zeek.bif | 27 ++++++++ .../.stderr | 1 + .../.stdout | 1 + .../event-metadata/register-type-errors.zeek | 30 ++++++++ 8 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr create mode 100644 testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout create mode 100644 testing/btest/core/event-metadata/register-type-errors.zeek diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index e80262b125..a6a3154f68 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -579,6 +579,21 @@ const io_poll_interval_live = 10 &redef; ## while testing, but should be used sparingly. const running_under_test: bool = F &redef; +module EventMetadata; + +export { + ## Enum type for metadata identifiers. + type ID: enum { + NETWORK_TIMESTAMP = 1, + }; + + ## A event metadata entry. + type Entry: record { + id: EventMetadata::ID; ##< The registered :zeek:see:`EventMetadata::ID` value. + val: any; ##< The value. Its type matches what was passed to :zeek:see:`EventMetadata::register`. + }; +} + module FTP; export { @@ -751,6 +766,9 @@ module GLOBAL; ## directly and then remove this alias. type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn; +## A type alias for event metadata. +type event_metadata_vec: vector of EventMetadata::Entry; + ## Statistics about a :zeek:type:`connection` endpoint. ## ## .. zeek:see:: connection diff --git a/src/Event.cc b/src/Event.cc index d113e475cf..8f9cc76315 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -3,6 +3,7 @@ #include "zeek/Event.h" #include "zeek/Desc.h" +#include "zeek/EventRegistry.h" #include "zeek/Trigger.h" #include "zeek/Val.h" #include "zeek/iosource/Manager.h" @@ -182,6 +183,19 @@ void EventMgr::Process() { // and had the opportunity to spawn new events. } -void EventMgr::InitPostScript() { iosource_mgr->Register(this, true, false); } +void EventMgr::InitPostScript() { + // Check if expected types and identifiers are available. + const auto& et = zeek::id::find_type("EventMetadata::ID"); + if ( ! et ) + zeek::reporter->FatalError("Failed to find EventMetadata::ID"); + const auto& net_ts_val = et->GetEnumVal(et->Lookup("EventMetadata::NETWORK_TIMESTAMP")); + if ( ! net_ts_val ) + zeek::reporter->FatalError("Failed to lookup EventMetadata::NETWORK_TIMESTAMP"); + + if ( ! zeek::event_registry->RegisterMetadata(net_ts_val, zeek::base_type(zeek::TYPE_TIME)) ) + zeek::reporter->FatalError("Failed to register NETWORK_TIMESTAMP metadata"); + + iosource_mgr->Register(this, true, false); +} } // namespace zeek diff --git a/src/EventRegistry.cc b/src/EventRegistry.cc index 0585752af1..ee52e5e62b 100644 --- a/src/EventRegistry.cc +++ b/src/EventRegistry.cc @@ -3,11 +3,16 @@ #include "zeek/EventRegistry.h" #include +#include +#include "zeek/Desc.h" #include "zeek/EventHandler.h" #include "zeek/Func.h" #include "zeek/RE.h" #include "zeek/Reporter.h" +#include "zeek/Traverse.h" +#include "zeek/TraverseTypes.h" +#include "zeek/Type.h" namespace zeek { @@ -164,4 +169,67 @@ void EventGroup::Disable() { void EventGroup::AddFunc(detail::ScriptFuncPtr f) { funcs.insert(f); } +namespace { + +class EventMetadataTypeRejector : public detail::TraversalCallback { +public: + detail::TraversalCode PreType(const Type* t) override { + if ( visited.count(t) > 0 ) + return detail::TC_ABORTSTMT; + + visited.insert(t); + + if ( reject.count(t->Tag()) ) + rejected.push_back(t); + + return detail::TC_CONTINUE; + }; + + std::set visited; + std::vector rejected; + + std::set reject = {TYPE_ANY, TYPE_FUNC, TYPE_FILE, TYPE_OPAQUE}; +}; + +} // namespace + +bool EventRegistry::RegisterMetadata(EnumValPtr id, TypePtr type) { + static const auto& metadata_id_type = id::find_type("EventMetadata::ID"); + + if ( metadata_id_type != id->GetType() ) + return false; + + auto id_int = id->Get(); + if ( id_int < 0 ) { + zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int); + } + + zeek_uint_t id_uint = static_cast(id_int); + + if ( auto it = event_metadata_types.find(id_uint); it != event_metadata_types.end() ) + return same_type(it->second.Type(), type); + + EventMetadataTypeRejector cb; + type->Traverse(&cb); + + if ( cb.rejected.size() > 0 ) + return false; + + event_metadata_types.insert({id_uint, EventMetadataDescriptor{id_uint, std::move(id), std::move(type)}}); + + return true; +} + +const EventMetadataDescriptor* EventRegistry::LookupMetadata(zeek_uint_t id) const { + const auto it = event_metadata_types.find(id); + if ( it == event_metadata_types.end() ) + return nullptr; + + if ( it->second.Id() != id ) { + zeek::reporter->InternalError("inconsistent metadata descriptor: %" PRIu64 " vs %" PRId64, it->second.Id(), id); + } + + return &(it->second); +} + } // namespace zeek diff --git a/src/EventRegistry.h b/src/EventRegistry.h index 3db64960ea..d4bfb5aa30 100644 --- a/src/EventRegistry.h +++ b/src/EventRegistry.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -22,18 +23,55 @@ enum class EventGroupKind { Module, }; +class EnumVal; class EventGroup; class EventHandler; class EventHandlerPtr; class RE_Matcher; +class RecordVal; +class Type; +using EnumValPtr = IntrusivePtr; using EventGroupPtr = std::shared_ptr; +using RecordValPtr = IntrusivePtr; +using TypePtr = IntrusivePtr; namespace detail { class ScriptFunc; using ScriptFuncPtr = zeek::IntrusivePtr; + +/** + * Well-known event metadata identifiers. + */ +enum class MetadataType : uint8_t { + NetworkTimestamp = 1, +}; + } // namespace detail +/** + * Descriptor for event metadata. + * + * Event metadata is registered via @ref EventRegistry::RegisterMetadata. The descriptor + * holds the metadata identifier and registered type. For the identifier, + * *id* is the unsigned int representation, while *id_val* holds the + * script-layer zeek::EnumVal. + */ +class EventMetadataDescriptor { +public: + EventMetadataDescriptor(zeek_uint_t id, EnumValPtr id_val, TypePtr type) + : id(id), id_val(std::move(id_val)), type(std::move(type)) {} + + zeek_uint_t Id() const { return id; } + const EnumValPtr& IdVal() const { return id_val; } + const TypePtr& Type() const { return type; } + +private: + zeek_uint_t id; + EnumValPtr id_val; + TypePtr type; +}; + // The registry keeps track of all events that we provide or handle. class EventRegistry final { public: @@ -98,6 +136,23 @@ public: */ EventGroupPtr LookupGroup(EventGroupKind kind, std::string_view name); + /** + * Register a script-layer metadata identifier *id* with type *type*. + * + * @param id The script-level ``EventMetadata::ID`` enum value. + * @param type The type to expect for the given metadata identifier. + */ + bool RegisterMetadata(EnumValPtr id, TypePtr type); + + /** + * Lookup the MetadataDescriptor for metadata identifier *id* + * + * @param id The metadata identifier as unsigned int. + * @return A pointer to a MetadataDescriptor or nullptr. + */ + const EventMetadataDescriptor* LookupMetadata(zeek_uint_t id) const; + + private: std::map, std::less<>> handlers; // Tracks whether a given event handler was registered in a @@ -106,6 +161,9 @@ private: // Map event groups identified by kind and name to their instances. std::map, std::shared_ptr, std::less<>> event_groups; + + // Map for event metadata identifier to their descriptors types. + std::unordered_map event_metadata_types; }; /** diff --git a/src/zeek.bif b/src/zeek.bif index f21d9fa065..5807ee80c7 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -401,6 +401,33 @@ function current_event_time%(%): time return zeek::make_intrusive(zeek::event_mgr.CurrentEventTime()); %} +## Register the expected Zeek type for event metadata. +## +## id: The event metadata identifier. +## +## t: A type expression or type alias. The type cannot be ``any``, ``func``, +## ``file``, ``opaque`` or a composite type containing one of these types. +## +## Returns: true if the registration was successful, false if *id* is +## registered with a different type already, or type is invalid. +## +## .. zeek:see:: EventMetadata::current EventMetadata::current_all +function EventMetadata::register%(id: EventMetadata::ID, t: any%): bool + %{ + const auto& tty = t->GetType(); + + if ( tty->Tag() != zeek::TYPE_TYPE ) + { + zeek::emit_builtin_error("register_event_metadata() expects type"); + return zeek::val_mgr->False(); + } + + EnumValPtr idvp = {zeek::NewRef{}, id->AsEnumVal()}; + + bool r = zeek::event_registry->RegisterMetadata(idvp, tty->AsTypeType()->GetType()); + return zeek::val_mgr->Bool(r); + %} + ## Returns a system environment variable. ## ## var: The name of the variable whose value to request. diff --git a/testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/core/event-metadata/register-type-errors.zeek b/testing/btest/core/event-metadata/register-type-errors.zeek new file mode 100644 index 0000000000..440d7161c8 --- /dev/null +++ b/testing/btest/core/event-metadata/register-type-errors.zeek @@ -0,0 +1,30 @@ +# @TEST-DOC: Verify rejection of certain metadata types. +# +# @TEST-EXEC: unset ZEEK_ALLOW_INIT_ERRORS; zeek -b %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +module App; + +export { + redef enum EventMetadata::ID += { + MY_METADATA = 1000, + MY_TABLE = 1002, + MY_VECTOR = 1003, + }; +} + +type R: record { + f: file; + a: any; + l: function(x: count): bool; +}; + +event zeek_init() + { + assert ! EventMetadata::register(MY_METADATA, any); + assert ! EventMetadata::register(MY_METADATA, table[count] of any); + assert ! EventMetadata::register(MY_METADATA, table[count] of function(x: count): bool); + assert ! EventMetadata::register(MY_METADATA, R); + assert ! EventMetadata::register(MY_METADATA, vector of R); + } From 12c523f3f79cc7afa3407ad29327f21eef003584 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 21 May 2025 15:01:08 +0200 Subject: [PATCH 3/9] Event: Store timestamp in metadata vector This removes the ts attribute from Event and instead allocates a vector for storing metadata. By default, adds the network time as a TimeVal. Later patches will make the allocation of the vector optional by introducing a different constructor so that users that are not interested in network timestamp metadata do not take the allocation hit. Moving the explicit ``ts`` out of the event is done in order to treat it just as generic metadata, too. However, the Time() accessor is adapted to lookup the value from the metadata vector instead. --- src/Event.cc | 48 +++++++++++++++++++++++++++++++++++++++++++++--- src/Event.h | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/src/Event.cc b/src/Event.cc index 8f9cc76315..7f61e062dd 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -2,9 +2,12 @@ #include "zeek/Event.h" +#include + #include "zeek/Desc.h" #include "zeek/EventRegistry.h" #include "zeek/Trigger.h" +#include "zeek/Type.h" #include "zeek/Val.h" #include "zeek/iosource/Manager.h" #include "zeek/plugin/Manager.h" @@ -15,19 +18,58 @@ zeek::EventMgr zeek::event_mgr; namespace zeek { +detail::EventMetadataVectorPtr detail::MakeEventMetadataVector(double t) { + auto tv = make_intrusive(t); + auto entry = detail::MetadataEntry{static_cast(detail::MetadataType::NetworkTimestamp), std::move(tv)}; + return std::make_unique(std::vector{std::move(entry)}); +} + +RecordValPtr detail::MetadataEntry::BuildVal() const { + static const auto rt = id::find_type("EventMetadata::Entry"); + auto rv = make_intrusive(rt); + const auto* desc = event_registry->LookupMetadata(id); + if ( ! desc ) { + zeek::reporter->InternalWarning("unable to find metadata descriptor for id %" PRIu64, id); + return rv; + } + + rv->Assign(0, desc->IdVal()); + rv->Assign(1, val); + + return rv; +} + Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj, double arg_ts) : handler(arg_handler), args(std::move(arg_args)), src(arg_src), aid(arg_aid), - ts(arg_ts), obj(arg_obj), - next_event(nullptr) { + next_event(nullptr), + meta(detail::MakeEventMetadataVector(arg_ts)) { if ( obj ) Ref(obj); } +double Event::Time() const { + if ( ! meta ) + return 0.0; + + for ( const auto& m : *meta ) + if ( m.Id() == static_cast(detail::MetadataType::NetworkTimestamp) ) { + if ( m.Val()->GetType()->Tag() != TYPE_TIME ) { + // This should've been caught during parsing. + zeek::reporter->InternalError("event metadata timestamp has wrong type: %s", + obj_desc_short(m.Val()->GetType().get()).c_str()); + } + + return m.Val()->AsTime(); + } + + return 0.0; +} + void Event::Describe(ODesc* d) const { if ( d->IsReadable() ) d->AddSP("event"); @@ -53,7 +95,7 @@ void Event::Dispatch(bool no_remote) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" // Replace in v8.1 with handler->Call(&args). - handler->Call(&args, no_remote, ts); + handler->Call(&args, no_remote, Time()); #pragma GCC diagnostic pop } diff --git a/src/Event.h b/src/Event.h index a5229b3e10..68c235d6ee 100644 --- a/src/Event.h +++ b/src/Event.h @@ -4,6 +4,7 @@ #include #include +#include #include "zeek/ZeekArgs.h" #include "zeek/analyzer/Analyzer.h" @@ -18,6 +19,38 @@ extern double network_time; class EventMgr; +namespace detail { + +/** + * An event metadata entry as stored in Event or @ref zeek::cluster::detail::Event. + */ +class MetadataEntry { +public: + MetadataEntry(zeek_uint_t id, zeek::ValPtr val) : id(id), val(std::move(val)) {} + + zeek_uint_t Id() const { return id; } + const zeek::ValPtr& Val() const { return val; } + + /** + * @return Pointer to a script-layer ``EventMetadata::Entry`` zeek::RecordVal representing this metadata entry. + */ + RecordValPtr BuildVal() const; + +private: + zeek_uint_t id; + zeek::ValPtr val; +}; + +using EventMetadataVector = std::vector; +using EventMetadataVectorPtr = std::unique_ptr; + +/** + * @return A new event metadata vector containing network timestamp value set to \a t; + */ +EventMetadataVectorPtr MakeEventMetadataVector(double t); + +} // namespace detail + class Event final : public Obj { public: Event(const EventHandlerPtr& handler, zeek::Args args, util::detail::SourceID src = util::detail::SOURCE_LOCAL, @@ -30,7 +63,7 @@ public: analyzer::ID Analyzer() const { return aid; } EventHandlerPtr Handler() const { return handler; } const zeek::Args& Args() const { return args; } - double Time() const { return ts; } + double Time() const; void Describe(ODesc* d) const override; @@ -45,9 +78,9 @@ private: zeek::Args args; util::detail::SourceID src; analyzer::ID aid; - double ts; Obj* obj; Event* next_event; + detail::EventMetadataVectorPtr meta; }; class EventMgr final : public Obj, public iosource::IOSource { From 53b0f0ad64ac4033d18431275f545fb81a56c91c Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 21 May 2025 16:20:31 +0200 Subject: [PATCH 4/9] Event: Deprecate default network timestamp metadata This deprecates the Event constructor and the ``ts`` parameter of Enqueue() Instead, versions are introduced that take a detail::MetadataVectorPtr which can hold the network timestamp metadata and is meant to be allocated by the caller instead of automatically during Enqueue() or within the Event constructor. This also introduces a BifConst ``EventMetadata::add_network_timestamp`` to opt-in adding network timestamps to events globally. It's disabled by default as there are not a lot of known use cases that need this. --- NEWS | 16 ++++ scripts/base/init-bare.zeek | 9 +++ src/Event.cc | 78 ++++++++++++++++++- src/Event.h | 42 +++++++++- src/Expr.cc | 18 +++-- src/const.bif | 2 + .../btest/broker/remote_event_auto_ts.zeek | 2 + .../broker/remote_event_schedule_ts.zeek | 2 + testing/btest/broker/remote_event_ts.zeek | 2 + .../btest/broker/remote_event_ts_compat.zeek | 1 + .../broker/web-socket-events-metadata.zeek | 1 + .../btest/language/event-ts-scheduled.zeek | 2 + testing/btest/language/event-ts.zeek | 2 + 13 files changed, 167 insertions(+), 10 deletions(-) diff --git a/NEWS b/NEWS index fbb9fc89a8..c713bcb14a 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,15 @@ Breaking Changes files. We tested builds of all of the existing third-party packages and only noticed one or two failures, but there is a possibility for breakage related to this cleanup. +- Network timestamps are not added to events by default anymore. Use the following + redef line to enable them: + + redef EventMetadata::add_network_timestamp = T; + + The background is that event metadata has become more generic and may incur + a small overhead when enabled. There's not enough users of network timestamp + metadata to justify the complexity of treating it separate. + New Functionality ----------------- @@ -65,6 +74,13 @@ Deprecated Functionality The replacement script doesn't populate the ``email_body_sections`` anymore either. +- The ``zeek::Event()`` constructor was deprecated. Use ``event_mgr::Enqueue()`` + or ``event_mgr::Dispatch()`` instead. + +- Passing ``ts`` as the last argument to ``EventMgr::Enqueue()`` has been deprecated + and will lead to compile time warnings. Use ``EventMgr::Enqueue(detail::MetadataVectorPtr meta, ...)`` + for populating ``meta`` accordingly. + Zeek 7.2.0 ========== diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index a6a3154f68..9fd71dae1a 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -592,6 +592,15 @@ export { id: EventMetadata::ID; ##< The registered :zeek:see:`EventMetadata::ID` value. val: any; ##< The value. Its type matches what was passed to :zeek:see:`EventMetadata::register`. }; + + ## Add network timestamp metadata to all events. + ## + ## Adding network timestamp metadata affects local and + ## remote events. Events scheduled have a network timestamp + ## of when the scheduled timer was supposed to expire, which + ## might be a value before the network_time() when the event + ## was actually dispatched. + const add_network_timestamp: bool = F &redef; } module FTP; diff --git a/src/Event.cc b/src/Event.cc index 7f61e062dd..7465fff410 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -12,6 +12,7 @@ #include "zeek/iosource/Manager.h" #include "zeek/plugin/Manager.h" +#include "const.bif.netvar_h" #include "event.bif.netvar_h" zeek::EventMgr zeek::event_mgr; @@ -52,6 +53,19 @@ Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::deta Ref(obj); } +Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args, + util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj) + : handler(arg_handler), + args(std::move(arg_args)), + src(arg_src), + aid(arg_aid), + obj(arg_obj), + next_event(nullptr), + meta(std::move(arg_meta)) { + if ( obj ) + Ref(obj); +} + double Event::Time() const { if ( ! meta ) return 0.0; @@ -120,8 +134,60 @@ EventMgr::~EventMgr() { } void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID src, analyzer::ID aid, Obj* obj, - double ts) { - QueueEvent(new Event(h, std::move(vl), src, aid, obj, ts)); + DeprecatedTimestamp deprecated_ts) { + detail::EventMetadataVectorPtr meta; + + double ts = double(deprecated_ts); + if ( BifConst::EventMetadata::add_network_timestamp ) { + if ( ts < 0.0 ) // default -1.0, modify to current network_time + ts = run_state::network_time; + + // In v8.1 when the deprecated_ts parameters is gone: Just use run_state::network_time directly here. + meta = detail::MakeEventMetadataVector(ts); + } + else if ( ts >= 0.0 ) { + // EventMetadata::add_network_timestamp is false, but EventMgr::Enqueue() + // with an explicit (non-negative) timestamp is used. That's a deprecated + // API, but we continue to support it until v8.1. + meta = detail::MakeEventMetadataVector(ts); + } + + QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj)); +} + +void EventMgr::Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, Args vl, + util::detail::SourceID src, analyzer::ID aid, Obj* obj) { + if ( BifConst::EventMetadata::add_network_timestamp ) { + // If all events are supposed to have a network time attached, ensure + // that the meta vector was passed *and* contains a network timestamp. + bool has_time = false; + + if ( ! meta ) { + // No metadata vector at all, make one with a timestamp. + meta = detail::MakeEventMetadataVector(run_state::network_time); + } + else { + // Check all entries for a network timestamp + for ( const auto& m : *meta ) { + if ( m.Id() == static_cast(detail::MetadataType::NetworkTimestamp) ) { + has_time = true; + + if ( m.Val()->GetType()->Tag() != TYPE_TIME ) { + // This should've been caught during parsing. + zeek::reporter->InternalError("event metadata timestamp has wrong type: %s", + obj_desc_short(m.Val()->GetType().get()).c_str()); + } + } + } + + if ( ! has_time ) { + auto tv = zeek::make_intrusive(run_state::network_time); + meta->push_back({static_cast(detail::MetadataType::NetworkTimestamp), std::move(tv)}); + } + } + } + + QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj)); } void EventMgr::QueueEvent(Event* event) { @@ -150,7 +216,13 @@ void EventMgr::Dispatch(Event* event, bool no_remote) { } void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) { - auto* ev = new Event(h, std::move(vl)); + detail::EventMetadataVectorPtr meta; + + // If all events should have network timestamps, create the vector holding one. + if ( BifConst::EventMetadata::add_network_timestamp ) + meta = detail::MakeEventMetadataVector(run_state::network_time); + + auto* ev = new Event(std::move(meta), h, std::move(vl), util::detail::SOURCE_LOCAL, 0, nullptr); // Technically this isn't queued, but still give plugins a chance to // intercept the event and cancel or modify it if really wanted. diff --git a/src/Event.h b/src/Event.h index 68c235d6ee..1edda36bc3 100644 --- a/src/Event.h +++ b/src/Event.h @@ -53,6 +53,7 @@ EventMetadataVectorPtr MakeEventMetadataVector(double t); class Event final : public Obj { public: + [[deprecated("Remove in v8.1: Do not instantiate raw events. Use EventMgr::Dispatch() or EventMgr::Enqueue().")]] Event(const EventHandlerPtr& handler, zeek::Args args, util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time); @@ -70,6 +71,10 @@ public: private: friend class EventMgr; + // Construct an event with a metadata vector. Passing arg_meta as nullptr is explicitly allowed. + Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args, + util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj); + // This method is protected to make sure that everybody goes through // EventMgr::Dispatch(). void Dispatch(bool no_remote = false); @@ -84,6 +89,8 @@ private: }; class EventMgr final : public Obj, public iosource::IOSource { + class DeprecatedTimestamp; + public: ~EventMgr() override; @@ -99,10 +106,10 @@ public: * @param obj an arbitrary object to use as a "cookie" or just hold a * reference to until dispatching the event. * @param ts timestamp at which the event is intended to be executed - * (defaults to current network time). + * (defaults to current network time - deprecated). */ void Enqueue(const EventHandlerPtr& h, zeek::Args vl, util::detail::SourceID src = util::detail::SOURCE_LOCAL, - analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time); + analyzer::ID aid = 0, Obj* obj = nullptr, DeprecatedTimestamp ts = {}); /** * A version of Enqueue() taking a variable number of arguments. @@ -113,6 +120,19 @@ public: return Enqueue(h, zeek::Args{std::forward(args)...}); } + /** + * Enqueue() with metadata vector support. + * @param meta Metadata to attach to the event, can be nullptr. + * @param h reference to the event handler to later call. + * @param vl the argument list to the event handler call. + * @param src indicates the origin of the event (local versus remote). + * @param aid identifies the protocol analyzer generating the event. + * @param obj an arbitrary object to use as a "cookie" or just hold a + * reference to until dispatching the event. + */ + void Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, zeek::Args vl, + util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr); + [[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]] void Dispatch(Event* event, bool no_remote = false); @@ -162,6 +182,24 @@ public: uint64_t num_events_dispatched = 0; private: + /** + * Helper class to produce a compile time warning if Enqueue() is called with an explicit timestamp. + * + * Remove in v8.1. + */ + class DeprecatedTimestamp { + public: + DeprecatedTimestamp() : d(-1.0) {} + [[deprecated("Use overload EventMgr::Enqueue(EventMetadataVectorPtr meta, ...) to pass timestamp metadata")]] + /*implicit*/ DeprecatedTimestamp(double d) + : d(d) {} + + explicit operator double() const { return d; } + + private: + double d; + }; + void QueueEvent(Event* event); Event* current = nullptr; diff --git a/src/Expr.cc b/src/Expr.cc index 53405ba198..cddc9d800a 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -24,6 +24,8 @@ #include "zeek/script_opt/Expr.h" #include "zeek/script_opt/ScriptOpt.h" +#include "const.bif.netvar_h" + namespace zeek::detail { const char* expr_name(ExprTag t) { @@ -3827,11 +3829,17 @@ ScheduleTimer::ScheduleTimer(const EventHandlerPtr& arg_event, Args arg_args, do void ScheduleTimer::Dispatch(double /* t */, bool /* is_expire */) { if ( event ) { - // An event's intended timestamp might be in the past as timer expiration is driven by - // network time. Guarantee that the intended timestamp is never in the future (e.g., - // when all timers are expired on shutdown). - auto ts = std::min(this->Time(), run_state::network_time); - event_mgr.Enqueue(event, std::move(args), util::detail::SOURCE_LOCAL, 0, nullptr, ts); + detail::EventMetadataVectorPtr meta; + + if ( BifConst::EventMetadata::add_network_timestamp ) { + // An event's intended timestamp might be in the past as timer expiration is driven by + // network time. Guarantee that the intended timestamp is never in the future (e.g., + // when all timers are expired on shutdown). + auto ts = std::min(this->Time(), run_state::network_time); + meta = detail::MakeEventMetadataVector(ts); + } + + event_mgr.Enqueue(std::move(meta), event, std::move(args)); } } diff --git a/src/const.bif b/src/const.bif index 822b0d30a6..482a325588 100644 --- a/src/const.bif +++ b/src/const.bif @@ -34,3 +34,5 @@ const Log::flush_interval: interval; const Log::write_buffer_size: count; const Storage::expire_interval: interval; + +const EventMetadata::add_network_timestamp: bool; diff --git a/testing/btest/broker/remote_event_auto_ts.zeek b/testing/btest/broker/remote_event_auto_ts.zeek index 49b26d846b..42050ae458 100644 --- a/testing/btest/broker/remote_event_auto_ts.zeek +++ b/testing/btest/broker/remote_event_auto_ts.zeek @@ -17,6 +17,7 @@ # @TEST-START-FILE send.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global runs = 0; global ping: event(msg: string, intended_ts: time); @@ -58,6 +59,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) # @TEST-START-FILE recv.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global msg_count = 0; diff --git a/testing/btest/broker/remote_event_schedule_ts.zeek b/testing/btest/broker/remote_event_schedule_ts.zeek index fe90f2b0cb..4b631ed95b 100644 --- a/testing/btest/broker/remote_event_schedule_ts.zeek +++ b/testing/btest/broker/remote_event_schedule_ts.zeek @@ -17,6 +17,7 @@ # @TEST-START-FILE send.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global runs = 0; global ping: event(msg: string, intended_ts: time, publish_ts: time); @@ -64,6 +65,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) # @TEST-START-FILE recv.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global msg_count = 0; diff --git a/testing/btest/broker/remote_event_ts.zeek b/testing/btest/broker/remote_event_ts.zeek index f822fde332..d9b8c2fadd 100644 --- a/testing/btest/broker/remote_event_ts.zeek +++ b/testing/btest/broker/remote_event_ts.zeek @@ -15,6 +15,7 @@ # @TEST-START-FILE send.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global runs = 0; global ping: event(msg: string, intended_ts: time); @@ -54,6 +55,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) # @TEST-START-FILE recv.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global msg_count = 0; diff --git a/testing/btest/broker/remote_event_ts_compat.zeek b/testing/btest/broker/remote_event_ts_compat.zeek index 7c457f9911..991e425759 100644 --- a/testing/btest/broker/remote_event_ts_compat.zeek +++ b/testing/btest/broker/remote_event_ts_compat.zeek @@ -17,6 +17,7 @@ redef exit_only_after_terminate = T; redef allow_network_time_forward = F; +redef EventMetadata::add_network_timestamp = T; event zeek_init() { diff --git a/testing/btest/broker/web-socket-events-metadata.zeek b/testing/btest/broker/web-socket-events-metadata.zeek index 34ab366061..5581e12a0d 100644 --- a/testing/btest/broker/web-socket-events-metadata.zeek +++ b/testing/btest/broker/web-socket-events-metadata.zeek @@ -16,6 +16,7 @@ redef allow_network_time_forward = F; redef exit_only_after_terminate = T; redef Broker::disable_ssl = T; +redef EventMetadata::add_network_timestamp = T; global event_count = 0; diff --git a/testing/btest/language/event-ts-scheduled.zeek b/testing/btest/language/event-ts-scheduled.zeek index bc0088845f..2a0ba07d55 100644 --- a/testing/btest/language/event-ts-scheduled.zeek +++ b/testing/btest/language/event-ts-scheduled.zeek @@ -1,6 +1,8 @@ # @TEST-EXEC: zeek -b -r $TRACES/ticks-dns-1hr.pcap %INPUT > out # @TEST-EXEC: btest-diff out +redef EventMetadata::add_network_timestamp = T; + global runs = 0; event test(schedule_time: time) diff --git a/testing/btest/language/event-ts.zeek b/testing/btest/language/event-ts.zeek index f35c3e5994..b4a8ff39ab 100644 --- a/testing/btest/language/event-ts.zeek +++ b/testing/btest/language/event-ts.zeek @@ -4,6 +4,8 @@ # Note: We use a PCAP with DNS queries only so that we have a single packet per # time step. Thus the run loop will be executed only once per time step. +redef EventMetadata::add_network_timestamp = T; + global runs = -1; event test(depth: count) From 75aa6588fe659b9e6e1a8b88a6e6fe1ff568d767 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 21 May 2025 16:48:47 +0200 Subject: [PATCH 5/9] Event/zeek.bif: Add EventMetadata current() and current_values() accessors ...and basic smoke testing. --- src/Event.cc | 34 ++++++++++++++ src/Event.h | 12 +++++ src/zeek.bif | 44 +++++++++++++++++++ .../.stderr | 1 + .../.stdout | 1 + .../with_ts.out | 2 + .../without_ts.out | 2 + .../core.event-metadata.non-event/.stderr | 1 + .../core.event-metadata.non-event/.stdout | 1 + .../core.event-metadata.register/.stderr | 1 + .../core.event-metadata.register/.stdout | 1 + .../event-metadata/network-timestamp.zeek | 17 +++++++ .../btest/core/event-metadata/non-event.zeek | 8 ++++ .../btest/core/event-metadata/register.zeek | 30 +++++++++++++ 14 files changed, 155 insertions(+) create mode 100644 testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr create mode 100644 testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout create mode 100644 testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out create mode 100644 testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out create mode 100644 testing/btest/Baseline/core.event-metadata.non-event/.stderr create mode 100644 testing/btest/Baseline/core.event-metadata.non-event/.stdout create mode 100644 testing/btest/Baseline/core.event-metadata.register/.stderr create mode 100644 testing/btest/Baseline/core.event-metadata.register/.stdout create mode 100644 testing/btest/core/event-metadata/network-timestamp.zeek create mode 100644 testing/btest/core/event-metadata/non-event.zeek create mode 100644 testing/btest/core/event-metadata/register.zeek diff --git a/src/Event.cc b/src/Event.cc index 7465fff410..e74936b1c8 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -66,6 +66,40 @@ Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg Ref(obj); } +zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const { + static const auto& any_vec_t = zeek::id::find_type("any_vec"); + auto result = zeek::make_intrusive(any_vec_t); + + if ( ! meta ) + return result; + + auto id_int = id->Get(); + if ( id_int < 0 ) + zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int); + + zeek_uint_t uintid = static_cast(id_int); + const auto* desc = event_registry->LookupMetadata(uintid); + if ( ! desc ) + return result; + + for ( const auto& entry : *meta ) { + if ( entry.Id() != uintid ) + continue; + + // Sanity check the type. + if ( ! same_type(desc->Type(), entry.Val()->GetType()) ) { + zeek::reporter->InternalWarning("metadata has unexpected type %s, wanted %s", + obj_desc_short(entry.Val()->GetType().get()).c_str(), + obj_desc_short(desc->Type().get()).c_str()); + continue; + } + + result->Append(entry.Val()); + } + + return result; +} + double Event::Time() const { if ( ! meta ) return 0.0; diff --git a/src/Event.h b/src/Event.h index 1edda36bc3..0826b251c6 100644 --- a/src/Event.h +++ b/src/Event.h @@ -66,6 +66,18 @@ public: const zeek::Args& Args() const { return args; } double Time() const; + /** + * @return a pointer to the MetadataVector of this event or a nullptr. + */ + const detail::EventMetadataVector* Metadata() const { return meta.get(); } + + /** + * @return a vector of values for metadata matching identifier \a id. + * + * @param id The metadata identifier as an enum value. + */ + VectorValPtr MetadataValues(const EnumValPtr& id) const; + void Describe(ODesc* d) const override; private: diff --git a/src/zeek.bif b/src/zeek.bif index 5807ee80c7..93f9569850 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -428,6 +428,50 @@ function EventMetadata::register%(id: EventMetadata::ID, t: any%): bool return zeek::val_mgr->Bool(r); %} +## Query the current event's metadata with identifier *id*. +## +## id: The metadata identifier, e.g. ``EventMetadata::NETWORK_TIMESTAMP``. +## +## Returns: A vector of values. The vector is empty if no metadata with +## the given identifier is attached to this event, otherwise a +## vector whose elements are of the type used during registration. +## +## .. zeek:see:: EventMetadata::register EventMetadata::current_all +function EventMetadata::current%(id: EventMetadata::ID%): any_vec + %{ + static const auto& vt = zeek::id::find_type("any_vec"); + + const auto* event = zeek::event_mgr.CurrentEvent(); + if ( ! event ) + return zeek::make_intrusive(vt); + + return event->MetadataValues({zeek::NewRef{}, id->AsEnumVal()}); + %} + +## Query all of the current event's metadata. +## +## Returns: A vector :zeek:see:`EventMetadata::Entry` elements holding all +## the metadata attached to this event. +## +## .. zeek:see:: EventMetadata::register EventMetadata::current +function EventMetadata::current_all%(%): event_metadata_vec + %{ + static const auto& vt = zeek::id::find_type("event_metadata_vec"); + auto result = zeek::make_intrusive(vt); + + const auto* event = zeek::event_mgr.CurrentEvent(); + if ( ! event ) + return result; + + if ( const auto* mdv = event->Metadata() ) { + for ( const auto& entry : *mdv ) + result->Append(entry.BuildVal()); + + } + + return result; + %} + ## Returns a system environment variable. ## ## var: The name of the variable whose value to request. diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out b/testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out new file mode 100644 index 0000000000..01ede881c4 --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +new_connection CHhAvVGS1DHFjwGM9 all=[[id=EventMetadata::NETWORK_TIMESTAMP, val=1362692526.869344]] network_timestamp=[1362692526.869344] diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out b/testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out new file mode 100644 index 0000000000..9c9414d530 --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +new_connection CHhAvVGS1DHFjwGM9 all=[] network_timestamp=[] diff --git a/testing/btest/Baseline/core.event-metadata.non-event/.stderr b/testing/btest/Baseline/core.event-metadata.non-event/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.non-event/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.non-event/.stdout b/testing/btest/Baseline/core.event-metadata.non-event/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.non-event/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register/.stderr b/testing/btest/Baseline/core.event-metadata.register/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register/.stdout b/testing/btest/Baseline/core.event-metadata.register/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/core/event-metadata/network-timestamp.zeek b/testing/btest/core/event-metadata/network-timestamp.zeek new file mode 100644 index 0000000000..2d90f3c71f --- /dev/null +++ b/testing/btest/core/event-metadata/network-timestamp.zeek @@ -0,0 +1,17 @@ +# @TEST-DOC: Check network timestamp available if opt-in. +# +# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=T > with_ts.out +# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=F > without_ts.out +# +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff with_ts.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff without_ts.out +# @TEST-EXEC: btest-diff .stderr + + +event new_connection(c: connection) + { + print fmt("new_connection %s all=%s network_timestamp=%s", + c$uid, + EventMetadata::current_all(), + EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP)); + } diff --git a/testing/btest/core/event-metadata/non-event.zeek b/testing/btest/core/event-metadata/non-event.zeek new file mode 100644 index 0000000000..acf80e941c --- /dev/null +++ b/testing/btest/core/event-metadata/non-event.zeek @@ -0,0 +1,8 @@ +# @TEST-DOC: Ensure EventMetadata::current() and EventMetadata::current_all() in non-event context returns empty vectors. +# +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +assert |EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP)| == 0; +assert |EventMetadata::current_all()| == 0; diff --git a/testing/btest/core/event-metadata/register.zeek b/testing/btest/core/event-metadata/register.zeek new file mode 100644 index 0000000000..ae90712b6f --- /dev/null +++ b/testing/btest/core/event-metadata/register.zeek @@ -0,0 +1,30 @@ +# @TEST-DOC: Very basic registration of event metadata identifiers. +# +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +module App; + +export { + redef enum EventMetadata::ID += { + MY_STRING = 1000, + MY_COUNT = 1001, + MY_TABLE = 1002, + }; +} + +event zeek_init() + { + assert EventMetadata::register(MY_STRING, string); + assert EventMetadata::register(MY_STRING, string); # double register is okay + assert EventMetadata::register(MY_COUNT, count); + assert EventMetadata::register(MY_COUNT, count); + assert EventMetadata::register(MY_TABLE, table[string] of count); + assert EventMetadata::register(MY_TABLE, table[string] of count); + + # Type mismatch all return F, but no output on stderr. + assert ! EventMetadata::register(MY_STRING, count); + assert ! EventMetadata::register(MY_COUNT, string); + assert ! EventMetadata::register(MY_TABLE, table[count] of string); + } From 506fea333520e8f257800ce6dc363db9ad27d044 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 21 May 2025 17:47:55 +0200 Subject: [PATCH 6/9] broker and cluster: Switch to new Enqueue() API This is a bit intermediary. In part 2 this will deal with any metadata, not just timestamps. --- src/broker/Manager.cc | 9 +++++++-- src/cluster/Backend.cc | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 6885a1c3f0..e42bed29ab 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -1655,8 +1655,13 @@ void Manager::ProcessMessage(std::string_view topic, broker::zeek::Event& ev) { } } - if ( vl.size() == args.size() ) - event_mgr.Enqueue(handler, std::move(vl), util::detail::SOURCE_BROKER, 0, nullptr, ts); + if ( vl.size() == args.size() ) { + zeek::detail::EventMetadataVectorPtr meta; + if ( ts > 0.0 ) + meta = zeek::detail::MakeEventMetadataVector(ts); + + event_mgr.Enqueue(std::move(meta), handler, std::move(vl), util::detail::SOURCE_BROKER); + } } bool Manager::ProcessMessage(std::string_view, broker::zeek::LogCreate& lc) { diff --git a/src/cluster/Backend.cc b/src/cluster/Backend.cc index dc0cff5af0..f7300e22d6 100644 --- a/src/cluster/Backend.cc +++ b/src/cluster/Backend.cc @@ -23,7 +23,11 @@ using namespace zeek::cluster; bool detail::LocalEventHandlingStrategy::DoProcessEvent(std::string_view topic, detail::Event e) { - zeek::event_mgr.Enqueue(e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER, 0, nullptr, e.Timestamp()); + zeek::detail::EventMetadataVectorPtr meta; + if ( auto ts = e.Timestamp(); ts > 0.0 ) + meta = zeek::detail::MakeEventMetadataVector(e.Timestamp()); + + zeek::event_mgr.Enqueue(std::move(meta), e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER); return true; } From e4e9ec3e803a366d347b20274d6a07b77f04b3f4 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 21 May 2025 17:53:11 +0200 Subject: [PATCH 7/9] btest/zam: Update for new EventMetadata bifs --- src/script_opt/FuncInfo.cc | 3 +++ testing/btest/Baseline/opt.ZAM-bif-tracking/output | 2 +- testing/btest/opt/ZAM-bif-tracking.zeek | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 49bce2d028..a06ef289ea 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -79,6 +79,9 @@ static std::unordered_map func_attrs = { {"Cluster::__unsubscribe", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::make_event", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::publish", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"EventMetadata::current", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"EventMetadata::current_all", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"EventMetadata::register", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"FileExtract::__set_limit", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Files::__add_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Files::__analyzer_enabled", ATTR_NO_ZEEK_SIDE_EFFECTS}, diff --git a/testing/btest/Baseline/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output index 546f4ca5f9..3294385ed4 100644 --- a/testing/btest/Baseline/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -560 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +563 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index dc43f6d5fa..7a30869274 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -106,6 +106,9 @@ global known_BiFs = set( "Cluster::publish", "Cluster::publish_hrw", "Cluster::publish_rr", + "EventMetadata::current", + "EventMetadata::current_all", + "EventMetadata::register", "FileExtract::__set_limit", "Files::__add_analyzer", "Files::__analyzer_enabled", From 89402fcc1b8e50ee106abc14fb3d5faf35bc1e47 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Thu, 22 May 2025 17:00:38 +0200 Subject: [PATCH 8/9] Event: Use IntrusivePtr to manage obj refcount Fly-by cleanup. --- src/Event.cc | 19 ++++++------------- src/Event.h | 2 +- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/Event.cc b/src/Event.cc index e74936b1c8..51bcc00a15 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -46,12 +46,9 @@ Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::deta args(std::move(arg_args)), src(arg_src), aid(arg_aid), - obj(arg_obj), + obj(zeek::NewRef{}, arg_obj), next_event(nullptr), - meta(detail::MakeEventMetadataVector(arg_ts)) { - if ( obj ) - Ref(obj); -} + meta(detail::MakeEventMetadataVector(arg_ts)) {} Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj) @@ -59,12 +56,9 @@ Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg args(std::move(arg_args)), src(arg_src), aid(arg_aid), - obj(arg_obj), + obj(zeek::NewRef{}, arg_obj), next_event(nullptr), - meta(std::move(arg_meta)) { - if ( obj ) - Ref(obj); -} + meta(std::move(arg_meta)) {} zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const { static const auto& any_vec_t = zeek::id::find_type("any_vec"); @@ -151,9 +145,8 @@ void Event::Dispatch(bool no_remote) { // Already reported. } - if ( obj ) - // obj->EventDone(); - Unref(obj); + // Unref obj + obj.reset(); if ( handler->ErrorHandler() ) reporter->EndErrorHandler(); diff --git a/src/Event.h b/src/Event.h index 0826b251c6..109a01bbfe 100644 --- a/src/Event.h +++ b/src/Event.h @@ -95,7 +95,7 @@ private: zeek::Args args; util::detail::SourceID src; analyzer::ID aid; - Obj* obj; + zeek::IntrusivePtr obj; Event* next_event; detail::EventMetadataVectorPtr meta; }; From bfcb68f8e7afc2c5788b3f58db4f48c21e9350de Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 23 May 2025 16:16:11 +0200 Subject: [PATCH 9/9] Event: Move meta after args --- src/Event.cc | 8 ++++---- src/Event.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Event.cc b/src/Event.cc index 51bcc00a15..c6af18be0d 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -44,21 +44,21 @@ Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::deta analyzer::ID arg_aid, Obj* arg_obj, double arg_ts) : handler(arg_handler), args(std::move(arg_args)), + meta(detail::MakeEventMetadataVector(arg_ts)), src(arg_src), aid(arg_aid), obj(zeek::NewRef{}, arg_obj), - next_event(nullptr), - meta(detail::MakeEventMetadataVector(arg_ts)) {} + next_event(nullptr) {} Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj) : handler(arg_handler), args(std::move(arg_args)), + meta(std::move(arg_meta)), src(arg_src), aid(arg_aid), obj(zeek::NewRef{}, arg_obj), - next_event(nullptr), - meta(std::move(arg_meta)) {} + next_event(nullptr) {} zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const { static const auto& any_vec_t = zeek::id::find_type("any_vec"); diff --git a/src/Event.h b/src/Event.h index 109a01bbfe..49c78dae6f 100644 --- a/src/Event.h +++ b/src/Event.h @@ -93,11 +93,11 @@ private: EventHandlerPtr handler; zeek::Args args; + detail::EventMetadataVectorPtr meta; util::detail::SourceID src; analyzer::ID aid; zeek::IntrusivePtr obj; Event* next_event; - detail::EventMetadataVectorPtr meta; }; class EventMgr final : public Obj, public iosource::IOSource {