diff --git a/CHANGES b/CHANGES index 84943a78a8..879189d9cc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,14 @@ +8.0.0-dev.201 | 2025-05-23 20:57:54 +0200 + + * Add generic event metadata, part 1 (Arne Welzel, Corelight) + + The ``ts`` member of ``Event`` was removed in favor of a more generic + approach that can hold any event metadata. Further, the Zeek script-layer + API for registering event metadata types was added. + + Event instances do not network timestamp metadata by default anymore. + This feature can be re-enabled via EventMetadata::add_network_timestamp=T. + 8.0.0-dev.191 | 2025-05-23 08:32:00 -0700 * Remove FindClangTidy.cmake, update cmake submodule (Tim Wojtulewicz, Corelight) diff --git a/NEWS b/NEWS index fbb9fc89a8..c713bcb14a 100644 --- a/NEWS +++ b/NEWS @@ -15,6 +15,15 @@ Breaking Changes files. We tested builds of all of the existing third-party packages and only noticed one or two failures, but there is a possibility for breakage related to this cleanup. +- Network timestamps are not added to events by default anymore. Use the following + redef line to enable them: + + redef EventMetadata::add_network_timestamp = T; + + The background is that event metadata has become more generic and may incur + a small overhead when enabled. There's not enough users of network timestamp + metadata to justify the complexity of treating it separate. + New Functionality ----------------- @@ -65,6 +74,13 @@ Deprecated Functionality The replacement script doesn't populate the ``email_body_sections`` anymore either. +- The ``zeek::Event()`` constructor was deprecated. Use ``event_mgr::Enqueue()`` + or ``event_mgr::Dispatch()`` instead. + +- Passing ``ts`` as the last argument to ``EventMgr::Enqueue()`` has been deprecated + and will lead to compile time warnings. Use ``EventMgr::Enqueue(detail::MetadataVectorPtr meta, ...)`` + for populating ``meta`` accordingly. + Zeek 7.2.0 ========== diff --git a/VERSION b/VERSION index a7f3602eda..4dda64675f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -8.0.0-dev.191 +8.0.0-dev.201 diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index e80262b125..9fd71dae1a 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -579,6 +579,30 @@ const io_poll_interval_live = 10 &redef; ## while testing, but should be used sparingly. const running_under_test: bool = F &redef; +module EventMetadata; + +export { + ## Enum type for metadata identifiers. + type ID: enum { + NETWORK_TIMESTAMP = 1, + }; + + ## A event metadata entry. + type Entry: record { + id: EventMetadata::ID; ##< The registered :zeek:see:`EventMetadata::ID` value. + val: any; ##< The value. Its type matches what was passed to :zeek:see:`EventMetadata::register`. + }; + + ## Add network timestamp metadata to all events. + ## + ## Adding network timestamp metadata affects local and + ## remote events. Events scheduled have a network timestamp + ## of when the scheduled timer was supposed to expire, which + ## might be a value before the network_time() when the event + ## was actually dispatched. + const add_network_timestamp: bool = F &redef; +} + module FTP; export { @@ -751,6 +775,9 @@ module GLOBAL; ## directly and then remove this alias. type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn; +## A type alias for event metadata. +type event_metadata_vec: vector of EventMetadata::Entry; + ## Statistics about a :zeek:type:`connection` endpoint. ## ## .. zeek:see:: connection diff --git a/src/Event.cc b/src/Event.cc index d113e475cf..c6af18be0d 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -2,29 +2,114 @@ #include "zeek/Event.h" +#include + #include "zeek/Desc.h" +#include "zeek/EventRegistry.h" #include "zeek/Trigger.h" +#include "zeek/Type.h" #include "zeek/Val.h" #include "zeek/iosource/Manager.h" #include "zeek/plugin/Manager.h" +#include "const.bif.netvar_h" #include "event.bif.netvar_h" zeek::EventMgr zeek::event_mgr; namespace zeek { +detail::EventMetadataVectorPtr detail::MakeEventMetadataVector(double t) { + auto tv = make_intrusive(t); + auto entry = detail::MetadataEntry{static_cast(detail::MetadataType::NetworkTimestamp), std::move(tv)}; + return std::make_unique(std::vector{std::move(entry)}); +} + +RecordValPtr detail::MetadataEntry::BuildVal() const { + static const auto rt = id::find_type("EventMetadata::Entry"); + auto rv = make_intrusive(rt); + const auto* desc = event_registry->LookupMetadata(id); + if ( ! desc ) { + zeek::reporter->InternalWarning("unable to find metadata descriptor for id %" PRIu64, id); + return rv; + } + + rv->Assign(0, desc->IdVal()); + rv->Assign(1, val); + + return rv; +} + Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj, double arg_ts) : handler(arg_handler), args(std::move(arg_args)), + meta(detail::MakeEventMetadataVector(arg_ts)), src(arg_src), aid(arg_aid), - ts(arg_ts), - obj(arg_obj), - next_event(nullptr) { - if ( obj ) - Ref(obj); + obj(zeek::NewRef{}, arg_obj), + next_event(nullptr) {} + +Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args, + util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj) + : handler(arg_handler), + args(std::move(arg_args)), + meta(std::move(arg_meta)), + src(arg_src), + aid(arg_aid), + obj(zeek::NewRef{}, arg_obj), + next_event(nullptr) {} + +zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const { + static const auto& any_vec_t = zeek::id::find_type("any_vec"); + auto result = zeek::make_intrusive(any_vec_t); + + if ( ! meta ) + return result; + + auto id_int = id->Get(); + if ( id_int < 0 ) + zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int); + + zeek_uint_t uintid = static_cast(id_int); + const auto* desc = event_registry->LookupMetadata(uintid); + if ( ! desc ) + return result; + + for ( const auto& entry : *meta ) { + if ( entry.Id() != uintid ) + continue; + + // Sanity check the type. + if ( ! same_type(desc->Type(), entry.Val()->GetType()) ) { + zeek::reporter->InternalWarning("metadata has unexpected type %s, wanted %s", + obj_desc_short(entry.Val()->GetType().get()).c_str(), + obj_desc_short(desc->Type().get()).c_str()); + continue; + } + + result->Append(entry.Val()); + } + + return result; +} + +double Event::Time() const { + if ( ! meta ) + return 0.0; + + for ( const auto& m : *meta ) + if ( m.Id() == static_cast(detail::MetadataType::NetworkTimestamp) ) { + if ( m.Val()->GetType()->Tag() != TYPE_TIME ) { + // This should've been caught during parsing. + zeek::reporter->InternalError("event metadata timestamp has wrong type: %s", + obj_desc_short(m.Val()->GetType().get()).c_str()); + } + + return m.Val()->AsTime(); + } + + return 0.0; } void Event::Describe(ODesc* d) const { @@ -52,7 +137,7 @@ void Event::Dispatch(bool no_remote) { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wdeprecated-declarations" // Replace in v8.1 with handler->Call(&args). - handler->Call(&args, no_remote, ts); + handler->Call(&args, no_remote, Time()); #pragma GCC diagnostic pop } @@ -60,9 +145,8 @@ void Event::Dispatch(bool no_remote) { // Already reported. } - if ( obj ) - // obj->EventDone(); - Unref(obj); + // Unref obj + obj.reset(); if ( handler->ErrorHandler() ) reporter->EndErrorHandler(); @@ -77,8 +161,60 @@ EventMgr::~EventMgr() { } void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID src, analyzer::ID aid, Obj* obj, - double ts) { - QueueEvent(new Event(h, std::move(vl), src, aid, obj, ts)); + DeprecatedTimestamp deprecated_ts) { + detail::EventMetadataVectorPtr meta; + + double ts = double(deprecated_ts); + if ( BifConst::EventMetadata::add_network_timestamp ) { + if ( ts < 0.0 ) // default -1.0, modify to current network_time + ts = run_state::network_time; + + // In v8.1 when the deprecated_ts parameters is gone: Just use run_state::network_time directly here. + meta = detail::MakeEventMetadataVector(ts); + } + else if ( ts >= 0.0 ) { + // EventMetadata::add_network_timestamp is false, but EventMgr::Enqueue() + // with an explicit (non-negative) timestamp is used. That's a deprecated + // API, but we continue to support it until v8.1. + meta = detail::MakeEventMetadataVector(ts); + } + + QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj)); +} + +void EventMgr::Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, Args vl, + util::detail::SourceID src, analyzer::ID aid, Obj* obj) { + if ( BifConst::EventMetadata::add_network_timestamp ) { + // If all events are supposed to have a network time attached, ensure + // that the meta vector was passed *and* contains a network timestamp. + bool has_time = false; + + if ( ! meta ) { + // No metadata vector at all, make one with a timestamp. + meta = detail::MakeEventMetadataVector(run_state::network_time); + } + else { + // Check all entries for a network timestamp + for ( const auto& m : *meta ) { + if ( m.Id() == static_cast(detail::MetadataType::NetworkTimestamp) ) { + has_time = true; + + if ( m.Val()->GetType()->Tag() != TYPE_TIME ) { + // This should've been caught during parsing. + zeek::reporter->InternalError("event metadata timestamp has wrong type: %s", + obj_desc_short(m.Val()->GetType().get()).c_str()); + } + } + } + + if ( ! has_time ) { + auto tv = zeek::make_intrusive(run_state::network_time); + meta->push_back({static_cast(detail::MetadataType::NetworkTimestamp), std::move(tv)}); + } + } + } + + QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj)); } void EventMgr::QueueEvent(Event* event) { @@ -107,7 +243,13 @@ void EventMgr::Dispatch(Event* event, bool no_remote) { } void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) { - auto* ev = new Event(h, std::move(vl)); + detail::EventMetadataVectorPtr meta; + + // If all events should have network timestamps, create the vector holding one. + if ( BifConst::EventMetadata::add_network_timestamp ) + meta = detail::MakeEventMetadataVector(run_state::network_time); + + auto* ev = new Event(std::move(meta), h, std::move(vl), util::detail::SOURCE_LOCAL, 0, nullptr); // Technically this isn't queued, but still give plugins a chance to // intercept the event and cancel or modify it if really wanted. @@ -182,6 +324,19 @@ void EventMgr::Process() { // and had the opportunity to spawn new events. } -void EventMgr::InitPostScript() { iosource_mgr->Register(this, true, false); } +void EventMgr::InitPostScript() { + // Check if expected types and identifiers are available. + const auto& et = zeek::id::find_type("EventMetadata::ID"); + if ( ! et ) + zeek::reporter->FatalError("Failed to find EventMetadata::ID"); + const auto& net_ts_val = et->GetEnumVal(et->Lookup("EventMetadata::NETWORK_TIMESTAMP")); + if ( ! net_ts_val ) + zeek::reporter->FatalError("Failed to lookup EventMetadata::NETWORK_TIMESTAMP"); + + if ( ! zeek::event_registry->RegisterMetadata(net_ts_val, zeek::base_type(zeek::TYPE_TIME)) ) + zeek::reporter->FatalError("Failed to register NETWORK_TIMESTAMP metadata"); + + iosource_mgr->Register(this, true, false); +} } // namespace zeek diff --git a/src/Event.h b/src/Event.h index 5639fea96f..49c78dae6f 100644 --- a/src/Event.h +++ b/src/Event.h @@ -4,6 +4,7 @@ #include #include +#include #include "zeek/ZeekArgs.h" #include "zeek/analyzer/Analyzer.h" @@ -18,8 +19,41 @@ extern double network_time; class EventMgr; +namespace detail { + +/** + * An event metadata entry as stored in Event or @ref zeek::cluster::detail::Event. + */ +class MetadataEntry { +public: + MetadataEntry(zeek_uint_t id, zeek::ValPtr val) : id(id), val(std::move(val)) {} + + zeek_uint_t Id() const { return id; } + const zeek::ValPtr& Val() const { return val; } + + /** + * @return Pointer to a script-layer ``EventMetadata::Entry`` zeek::RecordVal representing this metadata entry. + */ + RecordValPtr BuildVal() const; + +private: + zeek_uint_t id; + zeek::ValPtr val; +}; + +using EventMetadataVector = std::vector; +using EventMetadataVectorPtr = std::unique_ptr; + +/** + * @return A new event metadata vector containing network timestamp value set to \a t; + */ +EventMetadataVectorPtr MakeEventMetadataVector(double t); + +} // namespace detail + class Event final : public Obj { public: + [[deprecated("Remove in v8.1: Do not instantiate raw events. Use EventMgr::Dispatch() or EventMgr::Enqueue().")]] Event(const EventHandlerPtr& handler, zeek::Args args, util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time); @@ -30,27 +64,45 @@ public: analyzer::ID Analyzer() const { return aid; } EventHandlerPtr Handler() const { return handler; } const zeek::Args& Args() const { return args; } - double Time() const { return ts; } + double Time() const; + + /** + * @return a pointer to the MetadataVector of this event or a nullptr. + */ + const detail::EventMetadataVector* Metadata() const { return meta.get(); } + + /** + * @return a vector of values for metadata matching identifier \a id. + * + * @param id The metadata identifier as an enum value. + */ + VectorValPtr MetadataValues(const EnumValPtr& id) const; void Describe(ODesc* d) const override; private: friend class EventMgr; + // Construct an event with a metadata vector. Passing arg_meta as nullptr is explicitly allowed. + Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args, + util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj); + // This method is protected to make sure that everybody goes through // EventMgr::Dispatch(). void Dispatch(bool no_remote = false); EventHandlerPtr handler; zeek::Args args; + detail::EventMetadataVectorPtr meta; util::detail::SourceID src; analyzer::ID aid; - double ts; - Obj* obj; + zeek::IntrusivePtr obj; Event* next_event; }; class EventMgr final : public Obj, public iosource::IOSource { + class DeprecatedTimestamp; + public: ~EventMgr() override; @@ -66,10 +118,10 @@ public: * @param obj an arbitrary object to use as a "cookie" or just hold a * reference to until dispatching the event. * @param ts timestamp at which the event is intended to be executed - * (defaults to current network time). + * (defaults to current network time - deprecated). */ void Enqueue(const EventHandlerPtr& h, zeek::Args vl, util::detail::SourceID src = util::detail::SOURCE_LOCAL, - analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time); + analyzer::ID aid = 0, Obj* obj = nullptr, DeprecatedTimestamp ts = {}); /** * A version of Enqueue() taking a variable number of arguments. @@ -80,6 +132,19 @@ public: return Enqueue(h, zeek::Args{std::forward(args)...}); } + /** + * Enqueue() with metadata vector support. + * @param meta Metadata to attach to the event, can be nullptr. + * @param h reference to the event handler to later call. + * @param vl the argument list to the event handler call. + * @param src indicates the origin of the event (local versus remote). + * @param aid identifies the protocol analyzer generating the event. + * @param obj an arbitrary object to use as a "cookie" or just hold a + * reference to until dispatching the event. + */ + void Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, zeek::Args vl, + util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr); + [[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]] void Dispatch(Event* event, bool no_remote = false); @@ -116,6 +181,11 @@ public: // by returning a zero-timeout. double GetNextTimeout() override { return head ? 0.0 : -1.0; } + /** + * @return A pointer to the currently dispatched event or nullptr. + */ + const Event* CurrentEvent() const { return current; } + void Process() override; const char* Tag() override { return "EventManager"; } void InitPostScript(); @@ -124,6 +194,24 @@ public: uint64_t num_events_dispatched = 0; private: + /** + * Helper class to produce a compile time warning if Enqueue() is called with an explicit timestamp. + * + * Remove in v8.1. + */ + class DeprecatedTimestamp { + public: + DeprecatedTimestamp() : d(-1.0) {} + [[deprecated("Use overload EventMgr::Enqueue(EventMetadataVectorPtr meta, ...) to pass timestamp metadata")]] + /*implicit*/ DeprecatedTimestamp(double d) + : d(d) {} + + explicit operator double() const { return d; } + + private: + double d; + }; + void QueueEvent(Event* event); Event* current = nullptr; diff --git a/src/EventRegistry.cc b/src/EventRegistry.cc index 0585752af1..ee52e5e62b 100644 --- a/src/EventRegistry.cc +++ b/src/EventRegistry.cc @@ -3,11 +3,16 @@ #include "zeek/EventRegistry.h" #include +#include +#include "zeek/Desc.h" #include "zeek/EventHandler.h" #include "zeek/Func.h" #include "zeek/RE.h" #include "zeek/Reporter.h" +#include "zeek/Traverse.h" +#include "zeek/TraverseTypes.h" +#include "zeek/Type.h" namespace zeek { @@ -164,4 +169,67 @@ void EventGroup::Disable() { void EventGroup::AddFunc(detail::ScriptFuncPtr f) { funcs.insert(f); } +namespace { + +class EventMetadataTypeRejector : public detail::TraversalCallback { +public: + detail::TraversalCode PreType(const Type* t) override { + if ( visited.count(t) > 0 ) + return detail::TC_ABORTSTMT; + + visited.insert(t); + + if ( reject.count(t->Tag()) ) + rejected.push_back(t); + + return detail::TC_CONTINUE; + }; + + std::set visited; + std::vector rejected; + + std::set reject = {TYPE_ANY, TYPE_FUNC, TYPE_FILE, TYPE_OPAQUE}; +}; + +} // namespace + +bool EventRegistry::RegisterMetadata(EnumValPtr id, TypePtr type) { + static const auto& metadata_id_type = id::find_type("EventMetadata::ID"); + + if ( metadata_id_type != id->GetType() ) + return false; + + auto id_int = id->Get(); + if ( id_int < 0 ) { + zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int); + } + + zeek_uint_t id_uint = static_cast(id_int); + + if ( auto it = event_metadata_types.find(id_uint); it != event_metadata_types.end() ) + return same_type(it->second.Type(), type); + + EventMetadataTypeRejector cb; + type->Traverse(&cb); + + if ( cb.rejected.size() > 0 ) + return false; + + event_metadata_types.insert({id_uint, EventMetadataDescriptor{id_uint, std::move(id), std::move(type)}}); + + return true; +} + +const EventMetadataDescriptor* EventRegistry::LookupMetadata(zeek_uint_t id) const { + const auto it = event_metadata_types.find(id); + if ( it == event_metadata_types.end() ) + return nullptr; + + if ( it->second.Id() != id ) { + zeek::reporter->InternalError("inconsistent metadata descriptor: %" PRIu64 " vs %" PRId64, it->second.Id(), id); + } + + return &(it->second); +} + } // namespace zeek diff --git a/src/EventRegistry.h b/src/EventRegistry.h index 3db64960ea..d4bfb5aa30 100644 --- a/src/EventRegistry.h +++ b/src/EventRegistry.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -22,18 +23,55 @@ enum class EventGroupKind { Module, }; +class EnumVal; class EventGroup; class EventHandler; class EventHandlerPtr; class RE_Matcher; +class RecordVal; +class Type; +using EnumValPtr = IntrusivePtr; using EventGroupPtr = std::shared_ptr; +using RecordValPtr = IntrusivePtr; +using TypePtr = IntrusivePtr; namespace detail { class ScriptFunc; using ScriptFuncPtr = zeek::IntrusivePtr; + +/** + * Well-known event metadata identifiers. + */ +enum class MetadataType : uint8_t { + NetworkTimestamp = 1, +}; + } // namespace detail +/** + * Descriptor for event metadata. + * + * Event metadata is registered via @ref EventRegistry::RegisterMetadata. The descriptor + * holds the metadata identifier and registered type. For the identifier, + * *id* is the unsigned int representation, while *id_val* holds the + * script-layer zeek::EnumVal. + */ +class EventMetadataDescriptor { +public: + EventMetadataDescriptor(zeek_uint_t id, EnumValPtr id_val, TypePtr type) + : id(id), id_val(std::move(id_val)), type(std::move(type)) {} + + zeek_uint_t Id() const { return id; } + const EnumValPtr& IdVal() const { return id_val; } + const TypePtr& Type() const { return type; } + +private: + zeek_uint_t id; + EnumValPtr id_val; + TypePtr type; +}; + // The registry keeps track of all events that we provide or handle. class EventRegistry final { public: @@ -98,6 +136,23 @@ public: */ EventGroupPtr LookupGroup(EventGroupKind kind, std::string_view name); + /** + * Register a script-layer metadata identifier *id* with type *type*. + * + * @param id The script-level ``EventMetadata::ID`` enum value. + * @param type The type to expect for the given metadata identifier. + */ + bool RegisterMetadata(EnumValPtr id, TypePtr type); + + /** + * Lookup the MetadataDescriptor for metadata identifier *id* + * + * @param id The metadata identifier as unsigned int. + * @return A pointer to a MetadataDescriptor or nullptr. + */ + const EventMetadataDescriptor* LookupMetadata(zeek_uint_t id) const; + + private: std::map, std::less<>> handlers; // Tracks whether a given event handler was registered in a @@ -106,6 +161,9 @@ private: // Map event groups identified by kind and name to their instances. std::map, std::shared_ptr, std::less<>> event_groups; + + // Map for event metadata identifier to their descriptors types. + std::unordered_map event_metadata_types; }; /** diff --git a/src/Expr.cc b/src/Expr.cc index 53405ba198..cddc9d800a 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -24,6 +24,8 @@ #include "zeek/script_opt/Expr.h" #include "zeek/script_opt/ScriptOpt.h" +#include "const.bif.netvar_h" + namespace zeek::detail { const char* expr_name(ExprTag t) { @@ -3827,11 +3829,17 @@ ScheduleTimer::ScheduleTimer(const EventHandlerPtr& arg_event, Args arg_args, do void ScheduleTimer::Dispatch(double /* t */, bool /* is_expire */) { if ( event ) { - // An event's intended timestamp might be in the past as timer expiration is driven by - // network time. Guarantee that the intended timestamp is never in the future (e.g., - // when all timers are expired on shutdown). - auto ts = std::min(this->Time(), run_state::network_time); - event_mgr.Enqueue(event, std::move(args), util::detail::SOURCE_LOCAL, 0, nullptr, ts); + detail::EventMetadataVectorPtr meta; + + if ( BifConst::EventMetadata::add_network_timestamp ) { + // An event's intended timestamp might be in the past as timer expiration is driven by + // network time. Guarantee that the intended timestamp is never in the future (e.g., + // when all timers are expired on shutdown). + auto ts = std::min(this->Time(), run_state::network_time); + meta = detail::MakeEventMetadataVector(ts); + } + + event_mgr.Enqueue(std::move(meta), event, std::move(args)); } } diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 6885a1c3f0..e42bed29ab 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -1655,8 +1655,13 @@ void Manager::ProcessMessage(std::string_view topic, broker::zeek::Event& ev) { } } - if ( vl.size() == args.size() ) - event_mgr.Enqueue(handler, std::move(vl), util::detail::SOURCE_BROKER, 0, nullptr, ts); + if ( vl.size() == args.size() ) { + zeek::detail::EventMetadataVectorPtr meta; + if ( ts > 0.0 ) + meta = zeek::detail::MakeEventMetadataVector(ts); + + event_mgr.Enqueue(std::move(meta), handler, std::move(vl), util::detail::SOURCE_BROKER); + } } bool Manager::ProcessMessage(std::string_view, broker::zeek::LogCreate& lc) { diff --git a/src/cluster/Backend.cc b/src/cluster/Backend.cc index dc0cff5af0..f7300e22d6 100644 --- a/src/cluster/Backend.cc +++ b/src/cluster/Backend.cc @@ -23,7 +23,11 @@ using namespace zeek::cluster; bool detail::LocalEventHandlingStrategy::DoProcessEvent(std::string_view topic, detail::Event e) { - zeek::event_mgr.Enqueue(e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER, 0, nullptr, e.Timestamp()); + zeek::detail::EventMetadataVectorPtr meta; + if ( auto ts = e.Timestamp(); ts > 0.0 ) + meta = zeek::detail::MakeEventMetadataVector(e.Timestamp()); + + zeek::event_mgr.Enqueue(std::move(meta), e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER); return true; } diff --git a/src/const.bif b/src/const.bif index 822b0d30a6..482a325588 100644 --- a/src/const.bif +++ b/src/const.bif @@ -34,3 +34,5 @@ const Log::flush_interval: interval; const Log::write_buffer_size: count; const Storage::expire_interval: interval; + +const EventMetadata::add_network_timestamp: bool; diff --git a/src/script_opt/FuncInfo.cc b/src/script_opt/FuncInfo.cc index 49bce2d028..a06ef289ea 100644 --- a/src/script_opt/FuncInfo.cc +++ b/src/script_opt/FuncInfo.cc @@ -79,6 +79,9 @@ static std::unordered_map func_attrs = { {"Cluster::__unsubscribe", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::make_event", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Cluster::publish", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"EventMetadata::current", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"EventMetadata::current_all", ATTR_NO_SCRIPT_SIDE_EFFECTS}, + {"EventMetadata::register", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"FileExtract::__set_limit", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Files::__add_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS}, {"Files::__analyzer_enabled", ATTR_NO_ZEEK_SIDE_EFFECTS}, diff --git a/src/zeek.bif b/src/zeek.bif index f21d9fa065..93f9569850 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -401,6 +401,77 @@ function current_event_time%(%): time return zeek::make_intrusive(zeek::event_mgr.CurrentEventTime()); %} +## Register the expected Zeek type for event metadata. +## +## id: The event metadata identifier. +## +## t: A type expression or type alias. The type cannot be ``any``, ``func``, +## ``file``, ``opaque`` or a composite type containing one of these types. +## +## Returns: true if the registration was successful, false if *id* is +## registered with a different type already, or type is invalid. +## +## .. zeek:see:: EventMetadata::current EventMetadata::current_all +function EventMetadata::register%(id: EventMetadata::ID, t: any%): bool + %{ + const auto& tty = t->GetType(); + + if ( tty->Tag() != zeek::TYPE_TYPE ) + { + zeek::emit_builtin_error("register_event_metadata() expects type"); + return zeek::val_mgr->False(); + } + + EnumValPtr idvp = {zeek::NewRef{}, id->AsEnumVal()}; + + bool r = zeek::event_registry->RegisterMetadata(idvp, tty->AsTypeType()->GetType()); + return zeek::val_mgr->Bool(r); + %} + +## Query the current event's metadata with identifier *id*. +## +## id: The metadata identifier, e.g. ``EventMetadata::NETWORK_TIMESTAMP``. +## +## Returns: A vector of values. The vector is empty if no metadata with +## the given identifier is attached to this event, otherwise a +## vector whose elements are of the type used during registration. +## +## .. zeek:see:: EventMetadata::register EventMetadata::current_all +function EventMetadata::current%(id: EventMetadata::ID%): any_vec + %{ + static const auto& vt = zeek::id::find_type("any_vec"); + + const auto* event = zeek::event_mgr.CurrentEvent(); + if ( ! event ) + return zeek::make_intrusive(vt); + + return event->MetadataValues({zeek::NewRef{}, id->AsEnumVal()}); + %} + +## Query all of the current event's metadata. +## +## Returns: A vector :zeek:see:`EventMetadata::Entry` elements holding all +## the metadata attached to this event. +## +## .. zeek:see:: EventMetadata::register EventMetadata::current +function EventMetadata::current_all%(%): event_metadata_vec + %{ + static const auto& vt = zeek::id::find_type("event_metadata_vec"); + auto result = zeek::make_intrusive(vt); + + const auto* event = zeek::event_mgr.CurrentEvent(); + if ( ! event ) + return result; + + if ( const auto* mdv = event->Metadata() ) { + for ( const auto& entry : *mdv ) + result->Append(entry.BuildVal()); + + } + + return result; + %} + ## Returns a system environment variable. ## ## var: The name of the variable whose value to request. diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out b/testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out new file mode 100644 index 0000000000..01ede881c4 --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +new_connection CHhAvVGS1DHFjwGM9 all=[[id=EventMetadata::NETWORK_TIMESTAMP, val=1362692526.869344]] network_timestamp=[1362692526.869344] diff --git a/testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out b/testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out new file mode 100644 index 0000000000..9c9414d530 --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +new_connection CHhAvVGS1DHFjwGM9 all=[] network_timestamp=[] diff --git a/testing/btest/Baseline/core.event-metadata.non-event/.stderr b/testing/btest/Baseline/core.event-metadata.non-event/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.non-event/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.non-event/.stdout b/testing/btest/Baseline/core.event-metadata.non-event/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.non-event/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register/.stderr b/testing/btest/Baseline/core.event-metadata.register/.stderr new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register/.stderr @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/core.event-metadata.register/.stdout b/testing/btest/Baseline/core.event-metadata.register/.stdout new file mode 100644 index 0000000000..49d861c74c --- /dev/null +++ b/testing/btest/Baseline/core.event-metadata.register/.stdout @@ -0,0 +1 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. diff --git a/testing/btest/Baseline/opt.ZAM-bif-tracking/output b/testing/btest/Baseline/opt.ZAM-bif-tracking/output index 546f4ca5f9..3294385ed4 100644 --- a/testing/btest/Baseline/opt.ZAM-bif-tracking/output +++ b/testing/btest/Baseline/opt.ZAM-bif-tracking/output @@ -1,2 +1,2 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -560 seen BiFs, 0 unseen BiFs (), 0 new BiFs () +563 seen BiFs, 0 unseen BiFs (), 0 new BiFs () diff --git a/testing/btest/broker/remote_event_auto_ts.zeek b/testing/btest/broker/remote_event_auto_ts.zeek index 49b26d846b..42050ae458 100644 --- a/testing/btest/broker/remote_event_auto_ts.zeek +++ b/testing/btest/broker/remote_event_auto_ts.zeek @@ -17,6 +17,7 @@ # @TEST-START-FILE send.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global runs = 0; global ping: event(msg: string, intended_ts: time); @@ -58,6 +59,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) # @TEST-START-FILE recv.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global msg_count = 0; diff --git a/testing/btest/broker/remote_event_schedule_ts.zeek b/testing/btest/broker/remote_event_schedule_ts.zeek index fe90f2b0cb..4b631ed95b 100644 --- a/testing/btest/broker/remote_event_schedule_ts.zeek +++ b/testing/btest/broker/remote_event_schedule_ts.zeek @@ -17,6 +17,7 @@ # @TEST-START-FILE send.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global runs = 0; global ping: event(msg: string, intended_ts: time, publish_ts: time); @@ -64,6 +65,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) # @TEST-START-FILE recv.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global msg_count = 0; diff --git a/testing/btest/broker/remote_event_ts.zeek b/testing/btest/broker/remote_event_ts.zeek index f822fde332..d9b8c2fadd 100644 --- a/testing/btest/broker/remote_event_ts.zeek +++ b/testing/btest/broker/remote_event_ts.zeek @@ -15,6 +15,7 @@ # @TEST-START-FILE send.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global runs = 0; global ping: event(msg: string, intended_ts: time); @@ -54,6 +55,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) # @TEST-START-FILE recv.zeek redef exit_only_after_terminate = T; +redef EventMetadata::add_network_timestamp = T; global msg_count = 0; diff --git a/testing/btest/broker/remote_event_ts_compat.zeek b/testing/btest/broker/remote_event_ts_compat.zeek index 7c457f9911..991e425759 100644 --- a/testing/btest/broker/remote_event_ts_compat.zeek +++ b/testing/btest/broker/remote_event_ts_compat.zeek @@ -17,6 +17,7 @@ redef exit_only_after_terminate = T; redef allow_network_time_forward = F; +redef EventMetadata::add_network_timestamp = T; event zeek_init() { diff --git a/testing/btest/broker/web-socket-events-metadata.zeek b/testing/btest/broker/web-socket-events-metadata.zeek index 34ab366061..5581e12a0d 100644 --- a/testing/btest/broker/web-socket-events-metadata.zeek +++ b/testing/btest/broker/web-socket-events-metadata.zeek @@ -16,6 +16,7 @@ redef allow_network_time_forward = F; redef exit_only_after_terminate = T; redef Broker::disable_ssl = T; +redef EventMetadata::add_network_timestamp = T; global event_count = 0; diff --git a/testing/btest/core/event-metadata/network-timestamp.zeek b/testing/btest/core/event-metadata/network-timestamp.zeek new file mode 100644 index 0000000000..2d90f3c71f --- /dev/null +++ b/testing/btest/core/event-metadata/network-timestamp.zeek @@ -0,0 +1,17 @@ +# @TEST-DOC: Check network timestamp available if opt-in. +# +# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=T > with_ts.out +# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=F > without_ts.out +# +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff with_ts.out +# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff without_ts.out +# @TEST-EXEC: btest-diff .stderr + + +event new_connection(c: connection) + { + print fmt("new_connection %s all=%s network_timestamp=%s", + c$uid, + EventMetadata::current_all(), + EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP)); + } diff --git a/testing/btest/core/event-metadata/non-event.zeek b/testing/btest/core/event-metadata/non-event.zeek new file mode 100644 index 0000000000..acf80e941c --- /dev/null +++ b/testing/btest/core/event-metadata/non-event.zeek @@ -0,0 +1,8 @@ +# @TEST-DOC: Ensure EventMetadata::current() and EventMetadata::current_all() in non-event context returns empty vectors. +# +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +assert |EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP)| == 0; +assert |EventMetadata::current_all()| == 0; diff --git a/testing/btest/core/event-metadata/register-type-errors.zeek b/testing/btest/core/event-metadata/register-type-errors.zeek new file mode 100644 index 0000000000..440d7161c8 --- /dev/null +++ b/testing/btest/core/event-metadata/register-type-errors.zeek @@ -0,0 +1,30 @@ +# @TEST-DOC: Verify rejection of certain metadata types. +# +# @TEST-EXEC: unset ZEEK_ALLOW_INIT_ERRORS; zeek -b %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr + +module App; + +export { + redef enum EventMetadata::ID += { + MY_METADATA = 1000, + MY_TABLE = 1002, + MY_VECTOR = 1003, + }; +} + +type R: record { + f: file; + a: any; + l: function(x: count): bool; +}; + +event zeek_init() + { + assert ! EventMetadata::register(MY_METADATA, any); + assert ! EventMetadata::register(MY_METADATA, table[count] of any); + assert ! EventMetadata::register(MY_METADATA, table[count] of function(x: count): bool); + assert ! EventMetadata::register(MY_METADATA, R); + assert ! EventMetadata::register(MY_METADATA, vector of R); + } diff --git a/testing/btest/core/event-metadata/register.zeek b/testing/btest/core/event-metadata/register.zeek new file mode 100644 index 0000000000..ae90712b6f --- /dev/null +++ b/testing/btest/core/event-metadata/register.zeek @@ -0,0 +1,30 @@ +# @TEST-DOC: Very basic registration of event metadata identifiers. +# +# @TEST-EXEC: zeek -b %INPUT +# @TEST-EXEC: btest-diff .stdout +# @TEST-EXEC: btest-diff .stderr + +module App; + +export { + redef enum EventMetadata::ID += { + MY_STRING = 1000, + MY_COUNT = 1001, + MY_TABLE = 1002, + }; +} + +event zeek_init() + { + assert EventMetadata::register(MY_STRING, string); + assert EventMetadata::register(MY_STRING, string); # double register is okay + assert EventMetadata::register(MY_COUNT, count); + assert EventMetadata::register(MY_COUNT, count); + assert EventMetadata::register(MY_TABLE, table[string] of count); + assert EventMetadata::register(MY_TABLE, table[string] of count); + + # Type mismatch all return F, but no output on stderr. + assert ! EventMetadata::register(MY_STRING, count); + assert ! EventMetadata::register(MY_COUNT, string); + assert ! EventMetadata::register(MY_TABLE, table[count] of string); + } diff --git a/testing/btest/language/event-ts-scheduled.zeek b/testing/btest/language/event-ts-scheduled.zeek index bc0088845f..2a0ba07d55 100644 --- a/testing/btest/language/event-ts-scheduled.zeek +++ b/testing/btest/language/event-ts-scheduled.zeek @@ -1,6 +1,8 @@ # @TEST-EXEC: zeek -b -r $TRACES/ticks-dns-1hr.pcap %INPUT > out # @TEST-EXEC: btest-diff out +redef EventMetadata::add_network_timestamp = T; + global runs = 0; event test(schedule_time: time) diff --git a/testing/btest/language/event-ts.zeek b/testing/btest/language/event-ts.zeek index f35c3e5994..b4a8ff39ab 100644 --- a/testing/btest/language/event-ts.zeek +++ b/testing/btest/language/event-ts.zeek @@ -4,6 +4,8 @@ # Note: We use a PCAP with DNS queries only so that we have a single packet per # time step. Thus the run loop will be executed only once per time step. +redef EventMetadata::add_network_timestamp = T; + global runs = -1; event test(depth: count) diff --git a/testing/btest/opt/ZAM-bif-tracking.zeek b/testing/btest/opt/ZAM-bif-tracking.zeek index dc43f6d5fa..7a30869274 100644 --- a/testing/btest/opt/ZAM-bif-tracking.zeek +++ b/testing/btest/opt/ZAM-bif-tracking.zeek @@ -106,6 +106,9 @@ global known_BiFs = set( "Cluster::publish", "Cluster::publish_hrw", "Cluster::publish_rr", + "EventMetadata::current", + "EventMetadata::current_all", + "EventMetadata::register", "FileExtract::__set_limit", "Files::__add_analyzer", "Files::__analyzer_enabled",