Merge remote-tracking branch 'origin/topic/awelzel/4177-4178-custom-event-metadata-part-1'

* origin/topic/awelzel/4177-4178-custom-event-metadata-part-1:
  Event: Move meta after args
  Event: Use IntrusivePtr to manage obj refcount
  btest/zam: Update for new EventMetadata bifs
  broker and cluster: Switch to new Enqueue() API
  Event/zeek.bif: Add EventMetadata current() and current_values() accessors
  Event: Deprecate default network timestamp metadata
  Event: Store timestamp in metadata vector
  EventRegistry/zeek.bif/init-bare: Add event metadata infrastructure
  EventMgr: Add CurrentEvent() accessor

CHANGES

@@ -1,3 +1,14 @@
+8.0.0-dev.201 | 2025-05-23 20:57:54 +0200
+
+  * Add generic event metadata, part 1 (Arne Welzel, Corelight)
+
+    The ``ts`` member of ``Event`` was removed in favor of a more generic
+    approach that can hold any event metadata. Further, the Zeek script-layer
+    API for registering event metadata types was added.
+
+    Event instances do not network timestamp metadata by default anymore.
+    This feature can be re-enabled via EventMetadata::add_network_timestamp=T.
+
 8.0.0-dev.191 | 2025-05-23 08:32:00 -0700
 
   * Remove FindClangTidy.cmake, update cmake submodule (Tim Wojtulewicz, Corelight)

NEWS

@@ -15,6 +15,15 @@ Breaking Changes
   files. We tested builds of all of the existing third-party packages and only noticed one
   or two failures, but there is a possibility for breakage related to this cleanup.
 
+- Network timestamps are not added to events by default anymore. Use the following
+  redef line to enable them:
+
+	redef EventMetadata::add_network_timestamp = T;
+
+  The background is that event metadata has become more generic and may incur
+  a small overhead when enabled. There's not enough users of network timestamp
+  metadata to justify the complexity of treating it separate.
+
 New Functionality
 -----------------
 
@@ -65,6 +74,13 @@ Deprecated Functionality
 
   The replacement script doesn't populate the ``email_body_sections`` anymore either.
 
+- The ``zeek::Event()`` constructor was deprecated. Use ``event_mgr::Enqueue()``
+  or ``event_mgr::Dispatch()`` instead.
+
+- Passing ``ts`` as the last argument to ``EventMgr::Enqueue()`` has been deprecated
+  and will lead to compile time warnings. Use ``EventMgr::Enqueue(detail::MetadataVectorPtr meta, ...)``
+  for populating ``meta`` accordingly.
+
 Zeek 7.2.0
 ==========
 

VERSION

@@ -1 +1 @@
-8.0.0-dev.191
+8.0.0-dev.201

scripts/base/init-bare.zeek

@@ -579,6 +579,30 @@ const io_poll_interval_live = 10 &redef;
 ## while testing, but should be used sparingly.
 const running_under_test: bool = F &redef;
 
+module EventMetadata;
+
+export {
+	## Enum type for metadata identifiers.
+	type ID: enum {
+		NETWORK_TIMESTAMP = 1,
+	};
+
+	## A event metadata entry.
+	type Entry: record {
+		id: EventMetadata::ID; ##< The registered :zeek:see:`EventMetadata::ID` value.
+		val: any;              ##< The value. Its type matches what was passed to :zeek:see:`EventMetadata::register`.
+	};
+
+	## Add network timestamp metadata to all events.
+	##
+	## Adding network timestamp metadata affects local and
+	## remote events. Events scheduled have a network timestamp
+	## of when the scheduled timer was supposed to expire, which
+	## might be a value before the network_time() when the event
+	## was actually dispatched.
+	const add_network_timestamp: bool = F &redef;
+}
+
 module FTP;
 
 export {
@@ -751,6 +775,9 @@ module GLOBAL;
 ##    directly and then remove this alias.
 type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn;
 
+## A type alias for event metadata.
+type event_metadata_vec: vector of EventMetadata::Entry;
+
 ## Statistics about a :zeek:type:`connection` endpoint.
 ##
 ## .. zeek:see:: connection

src/Event.cc

@@ -2,29 +2,114 @@
 
 #include "zeek/Event.h"
 
+#include <cinttypes>
+
 #include "zeek/Desc.h"
+#include "zeek/EventRegistry.h"
 #include "zeek/Trigger.h"
+#include "zeek/Type.h"
 #include "zeek/Val.h"
 #include "zeek/iosource/Manager.h"
 #include "zeek/plugin/Manager.h"
 
+#include "const.bif.netvar_h"
 #include "event.bif.netvar_h"
 
 zeek::EventMgr zeek::event_mgr;
 
 namespace zeek {
 
+detail::EventMetadataVectorPtr detail::MakeEventMetadataVector(double t) {
+    auto tv = make_intrusive<TimeVal>(t);
+    auto entry = detail::MetadataEntry{static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp), std::move(tv)};
+    return std::make_unique<detail::EventMetadataVector>(std::vector{std::move(entry)});
+}
+
+RecordValPtr detail::MetadataEntry::BuildVal() const {
+    static const auto rt = id::find_type<RecordType>("EventMetadata::Entry");
+    auto rv = make_intrusive<RecordVal>(rt);
+    const auto* desc = event_registry->LookupMetadata(id);
+    if ( ! desc ) {
+        zeek::reporter->InternalWarning("unable to find metadata descriptor for id %" PRIu64, id);
+        return rv;
+    }
+
+    rv->Assign(0, desc->IdVal());
+    rv->Assign(1, val);
+
+    return rv;
+}
+
 Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src,
              analyzer::ID arg_aid, Obj* arg_obj, double arg_ts)
     : handler(arg_handler),
       args(std::move(arg_args)),
+      meta(detail::MakeEventMetadataVector(arg_ts)),
       src(arg_src),
       aid(arg_aid),
-      ts(arg_ts),
+      obj(zeek::NewRef{}, arg_obj),
-      obj(arg_obj),
+      next_event(nullptr) {}
-      next_event(nullptr) {
+
-    if ( obj )
+Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args,
-        Ref(obj);
+             util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj)
+    : handler(arg_handler),
+      args(std::move(arg_args)),
+      meta(std::move(arg_meta)),
+      src(arg_src),
+      aid(arg_aid),
+      obj(zeek::NewRef{}, arg_obj),
+      next_event(nullptr) {}
+
+zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const {
+    static const auto& any_vec_t = zeek::id::find_type<zeek::VectorType>("any_vec");
+    auto result = zeek::make_intrusive<zeek::VectorVal>(any_vec_t);
+
+    if ( ! meta )
+        return result;
+
+    auto id_int = id->Get();
+    if ( id_int < 0 )
+        zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int);
+
+    zeek_uint_t uintid = static_cast<zeek_uint_t>(id_int);
+    const auto* desc = event_registry->LookupMetadata(uintid);
+    if ( ! desc )
+        return result;
+
+    for ( const auto& entry : *meta ) {
+        if ( entry.Id() != uintid )
+            continue;
+
+        // Sanity check the type.
+        if ( ! same_type(desc->Type(), entry.Val()->GetType()) ) {
+            zeek::reporter->InternalWarning("metadata has unexpected type %s, wanted %s",
+                                            obj_desc_short(entry.Val()->GetType().get()).c_str(),
+                                            obj_desc_short(desc->Type().get()).c_str());
+            continue;
+        }
+
+        result->Append(entry.Val());
+    }
+
+    return result;
+}
+
+double Event::Time() const {
+    if ( ! meta )
+        return 0.0;
+
+    for ( const auto& m : *meta )
+        if ( m.Id() == static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp) ) {
+            if ( m.Val()->GetType()->Tag() != TYPE_TIME ) {
+                // This should've been caught during parsing.
+                zeek::reporter->InternalError("event metadata timestamp has wrong type: %s",
+                                              obj_desc_short(m.Val()->GetType().get()).c_str());
+            }
+
+            return m.Val()->AsTime();
+        }
+
+    return 0.0;
 }
 
 void Event::Describe(ODesc* d) const {
@@ -52,7 +137,7 @@ void Event::Dispatch(bool no_remote) {
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wdeprecated-declarations"
         // Replace in v8.1 with handler->Call(&args).
-        handler->Call(&args, no_remote, ts);
+        handler->Call(&args, no_remote, Time());
 #pragma GCC diagnostic pop
     }
 
@@ -60,9 +145,8 @@ void Event::Dispatch(bool no_remote) {
         // Already reported.
     }
 
-    if ( obj )
+    // Unref obj
-        // obj->EventDone();
+    obj.reset();
-        Unref(obj);
 
     if ( handler->ErrorHandler() )
         reporter->EndErrorHandler();
@@ -77,8 +161,60 @@ EventMgr::~EventMgr() {
 }
 
 void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID src, analyzer::ID aid, Obj* obj,
-                       double ts) {
+                       DeprecatedTimestamp deprecated_ts) {
-    QueueEvent(new Event(h, std::move(vl), src, aid, obj, ts));
+    detail::EventMetadataVectorPtr meta;
+
+    double ts = double(deprecated_ts);
+    if ( BifConst::EventMetadata::add_network_timestamp ) {
+        if ( ts < 0.0 ) // default -1.0, modify to current network_time
+            ts = run_state::network_time;
+
+        // In v8.1 when the deprecated_ts parameters is gone: Just use run_state::network_time directly here.
+        meta = detail::MakeEventMetadataVector(ts);
+    }
+    else if ( ts >= 0.0 ) {
+        // EventMetadata::add_network_timestamp is false, but EventMgr::Enqueue()
+        // with an explicit (non-negative) timestamp is used. That's a deprecated
+        // API, but we continue to support it until v8.1.
+        meta = detail::MakeEventMetadataVector(ts);
+    }
+
+    QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj));
+}
+
+void EventMgr::Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, Args vl,
+                       util::detail::SourceID src, analyzer::ID aid, Obj* obj) {
+    if ( BifConst::EventMetadata::add_network_timestamp ) {
+        // If all events are supposed to have a network time attached, ensure
+        // that the meta vector was passed *and* contains a network timestamp.
+        bool has_time = false;
+
+        if ( ! meta ) {
+            // No metadata vector at all, make one with a timestamp.
+            meta = detail::MakeEventMetadataVector(run_state::network_time);
+        }
+        else {
+            // Check all entries for a network timestamp
+            for ( const auto& m : *meta ) {
+                if ( m.Id() == static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp) ) {
+                    has_time = true;
+
+                    if ( m.Val()->GetType()->Tag() != TYPE_TIME ) {
+                        // This should've been caught during parsing.
+                        zeek::reporter->InternalError("event metadata timestamp has wrong type: %s",
+                                                      obj_desc_short(m.Val()->GetType().get()).c_str());
+                    }
+                }
+            }
+
+            if ( ! has_time ) {
+                auto tv = zeek::make_intrusive<zeek::TimeVal>(run_state::network_time);
+                meta->push_back({static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp), std::move(tv)});
+            }
+        }
+    }
+
+    QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj));
 }
 
 void EventMgr::QueueEvent(Event* event) {
@@ -107,7 +243,13 @@ void EventMgr::Dispatch(Event* event, bool no_remote) {
 }
 
 void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) {
-    auto* ev = new Event(h, std::move(vl));
+    detail::EventMetadataVectorPtr meta;
+
+    // If all events should have network timestamps, create the vector holding one.
+    if ( BifConst::EventMetadata::add_network_timestamp )
+        meta = detail::MakeEventMetadataVector(run_state::network_time);
+
+    auto* ev = new Event(std::move(meta), h, std::move(vl), util::detail::SOURCE_LOCAL, 0, nullptr);
 
     // Technically this isn't queued, but still give plugins a chance to
     // intercept the event and cancel or modify it if really wanted.
@@ -182,6 +324,19 @@ void EventMgr::Process() {
     // and had the opportunity to spawn new events.
 }
 
-void EventMgr::InitPostScript() { iosource_mgr->Register(this, true, false); }
+void EventMgr::InitPostScript() {
+    // Check if expected types and identifiers are available.
+    const auto& et = zeek::id::find_type<zeek::EnumType>("EventMetadata::ID");
+    if ( ! et )
+        zeek::reporter->FatalError("Failed to find EventMetadata::ID");
 
+    const auto& net_ts_val = et->GetEnumVal(et->Lookup("EventMetadata::NETWORK_TIMESTAMP"));
+    if ( ! net_ts_val )
+        zeek::reporter->FatalError("Failed to lookup EventMetadata::NETWORK_TIMESTAMP");
+
+    if ( ! zeek::event_registry->RegisterMetadata(net_ts_val, zeek::base_type(zeek::TYPE_TIME)) )
+        zeek::reporter->FatalError("Failed to register NETWORK_TIMESTAMP metadata");
+
+    iosource_mgr->Register(this, true, false);
+}
 } // namespace zeek

src/Event.h

@@ -4,6 +4,7 @@
 
 #include <tuple>
 #include <type_traits>
+#include <vector>
 
 #include "zeek/ZeekArgs.h"
 #include "zeek/analyzer/Analyzer.h"
@@ -18,8 +19,41 @@ extern double network_time;
 
 class EventMgr;
 
+namespace detail {
+
+/**
+ * An event metadata entry as stored in Event or @ref zeek::cluster::detail::Event.
+ */
+class MetadataEntry {
+public:
+    MetadataEntry(zeek_uint_t id, zeek::ValPtr val) : id(id), val(std::move(val)) {}
+
+    zeek_uint_t Id() const { return id; }
+    const zeek::ValPtr& Val() const { return val; }
+
+    /**
+     * @return Pointer to a script-layer ``EventMetadata::Entry`` zeek::RecordVal representing this metadata entry.
+     */
+    RecordValPtr BuildVal() const;
+
+private:
+    zeek_uint_t id;
+    zeek::ValPtr val;
+};
+
+using EventMetadataVector = std::vector<MetadataEntry>;
+using EventMetadataVectorPtr = std::unique_ptr<EventMetadataVector>;
+
+/**
+ * @return A new event metadata vector containing network timestamp value set to \a t;
+ */
+EventMetadataVectorPtr MakeEventMetadataVector(double t);
+
+} // namespace detail
+
 class Event final : public Obj {
 public:
+    [[deprecated("Remove in v8.1: Do not instantiate raw events. Use EventMgr::Dispatch() or EventMgr::Enqueue().")]]
     Event(const EventHandlerPtr& handler, zeek::Args args, util::detail::SourceID src = util::detail::SOURCE_LOCAL,
           analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time);
 
@@ -30,27 +64,45 @@ public:
     analyzer::ID Analyzer() const { return aid; }
     EventHandlerPtr Handler() const { return handler; }
     const zeek::Args& Args() const { return args; }
-    double Time() const { return ts; }
+    double Time() const;
+
+    /**
+     * @return a pointer to the MetadataVector of this event or a nullptr.
+     */
+    const detail::EventMetadataVector* Metadata() const { return meta.get(); }
+
+    /**
+     * @return a vector of values for metadata matching identifier \a id.
+     *
+     * @param id The metadata identifier as an enum value.
+     */
+    VectorValPtr MetadataValues(const EnumValPtr& id) const;
 
     void Describe(ODesc* d) const override;
 
 private:
     friend class EventMgr;
 
+    // Construct an event with a metadata vector. Passing arg_meta as nullptr is explicitly allowed.
+    Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args,
+          util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj);
+
     // This method is protected to make sure that everybody goes through
     // EventMgr::Dispatch().
     void Dispatch(bool no_remote = false);
 
     EventHandlerPtr handler;
     zeek::Args args;
+    detail::EventMetadataVectorPtr meta;
     util::detail::SourceID src;
     analyzer::ID aid;
-    double ts;
+    zeek::IntrusivePtr<Obj> obj;
-    Obj* obj;
     Event* next_event;
 };
 
 class EventMgr final : public Obj, public iosource::IOSource {
+    class DeprecatedTimestamp;
+
 public:
     ~EventMgr() override;
 
@@ -66,10 +118,10 @@ public:
      * @param obj  an arbitrary object to use as a "cookie" or just hold a
      * reference to until dispatching the event.
      * @param ts  timestamp at which the event is intended to be executed
-     * (defaults to current network time).
+     * (defaults to current network time - deprecated).
      */
     void Enqueue(const EventHandlerPtr& h, zeek::Args vl, util::detail::SourceID src = util::detail::SOURCE_LOCAL,
-                 analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time);
+                 analyzer::ID aid = 0, Obj* obj = nullptr, DeprecatedTimestamp ts = {});
 
     /**
      * A version of Enqueue() taking a variable number of arguments.
@@ -80,6 +132,19 @@ public:
         return Enqueue(h, zeek::Args{std::forward<Args>(args)...});
     }
 
+    /**
+     * Enqueue() with metadata vector support.
+     * @param meta  Metadata to attach to the event, can be nullptr.
+     * @param h  reference to the event handler to later call.
+     * @param vl  the argument list to the event handler call.
+     * @param src  indicates the origin of the event (local versus remote).
+     * @param aid  identifies the protocol analyzer generating the event.
+     * @param obj  an arbitrary object to use as a "cookie" or just hold a
+     * reference to until dispatching the event.
+     */
+    void Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, zeek::Args vl,
+                 util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr);
+
     [[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]]
     void Dispatch(Event* event, bool no_remote = false);
 
@@ -116,6 +181,11 @@ public:
     // by returning a zero-timeout.
     double GetNextTimeout() override { return head ? 0.0 : -1.0; }
 
+    /**
+     * @return A pointer to the currently dispatched event or nullptr.
+     */
+    const Event* CurrentEvent() const { return current; }
+
     void Process() override;
     const char* Tag() override { return "EventManager"; }
     void InitPostScript();
@@ -124,6 +194,24 @@ public:
     uint64_t num_events_dispatched = 0;
 
 private:
+    /**
+     * Helper class to produce a compile time warning if Enqueue() is called with an explicit timestamp.
+     *
+     * Remove in v8.1.
+     */
+    class DeprecatedTimestamp {
+    public:
+        DeprecatedTimestamp() : d(-1.0) {}
+        [[deprecated("Use overload EventMgr::Enqueue(EventMetadataVectorPtr meta, ...) to pass timestamp metadata")]]
+        /*implicit*/ DeprecatedTimestamp(double d)
+            : d(d) {}
+
+        explicit operator double() const { return d; }
+
+    private:
+        double d;
+    };
+
     void QueueEvent(Event* event);
 
     Event* current = nullptr;

src/EventRegistry.cc

@@ -3,11 +3,16 @@
 #include "zeek/EventRegistry.h"
 
 #include <algorithm>
+#include <cinttypes>
 
+#include "zeek/Desc.h"
 #include "zeek/EventHandler.h"
 #include "zeek/Func.h"
 #include "zeek/RE.h"
 #include "zeek/Reporter.h"
+#include "zeek/Traverse.h"
+#include "zeek/TraverseTypes.h"
+#include "zeek/Type.h"
 
 namespace zeek {
 
@@ -164,4 +169,67 @@ void EventGroup::Disable() {
 
 void EventGroup::AddFunc(detail::ScriptFuncPtr f) { funcs.insert(f); }
 
+namespace {
+
+class EventMetadataTypeRejector : public detail::TraversalCallback {
+public:
+    detail::TraversalCode PreType(const Type* t) override {
+        if ( visited.count(t) > 0 )
+            return detail::TC_ABORTSTMT;
+
+        visited.insert(t);
+
+        if ( reject.count(t->Tag()) )
+            rejected.push_back(t);
+
+        return detail::TC_CONTINUE;
+    };
+
+    std::set<const zeek::Type*> visited;
+    std::vector<const zeek::Type*> rejected;
+
+    std::set<zeek::TypeTag> reject = {TYPE_ANY, TYPE_FUNC, TYPE_FILE, TYPE_OPAQUE};
+};
+
+} // namespace
+
+bool EventRegistry::RegisterMetadata(EnumValPtr id, TypePtr type) {
+    static const auto& metadata_id_type = id::find_type<EnumType>("EventMetadata::ID");
+
+    if ( metadata_id_type != id->GetType() )
+        return false;
+
+    auto id_int = id->Get();
+    if ( id_int < 0 ) {
+        zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int);
+    }
+
+    zeek_uint_t id_uint = static_cast<zeek_uint_t>(id_int);
+
+    if ( auto it = event_metadata_types.find(id_uint); it != event_metadata_types.end() )
+        return same_type(it->second.Type(), type);
+
+    EventMetadataTypeRejector cb;
+    type->Traverse(&cb);
+
+    if ( cb.rejected.size() > 0 )
+        return false;
+
+    event_metadata_types.insert({id_uint, EventMetadataDescriptor{id_uint, std::move(id), std::move(type)}});
+
+    return true;
+}
+
+const EventMetadataDescriptor* EventRegistry::LookupMetadata(zeek_uint_t id) const {
+    const auto it = event_metadata_types.find(id);
+    if ( it == event_metadata_types.end() )
+        return nullptr;
+
+    if ( it->second.Id() != id ) {
+        zeek::reporter->InternalError("inconsistent metadata descriptor: %" PRIu64 " vs %" PRId64, it->second.Id(), id);
+    }
+
+    return &(it->second);
+}
+
 } // namespace zeek

src/EventRegistry.h

@@ -8,6 +8,7 @@
 #include <memory>
 #include <string>
 #include <string_view>
+#include <unordered_map>
 #include <unordered_set>
 #include <vector>
 
@@ -22,18 +23,55 @@ enum class EventGroupKind {
     Module,
 };
 
+class EnumVal;
 class EventGroup;
 class EventHandler;
 class EventHandlerPtr;
 class RE_Matcher;
+class RecordVal;
+class Type;
 
+using EnumValPtr = IntrusivePtr<EnumVal>;
 using EventGroupPtr = std::shared_ptr<EventGroup>;
+using RecordValPtr = IntrusivePtr<RecordVal>;
+using TypePtr = IntrusivePtr<Type>;
 
 namespace detail {
 class ScriptFunc;
 using ScriptFuncPtr = zeek::IntrusivePtr<ScriptFunc>;
+
+/**
+ * Well-known event metadata identifiers.
+ */
+enum class MetadataType : uint8_t {
+    NetworkTimestamp = 1,
+};
+
 } // namespace detail
 
+/**
+ * Descriptor for event metadata.
+ *
+ * Event metadata is registered via @ref EventRegistry::RegisterMetadata. The descriptor
+ * holds the metadata identifier and registered type. For the identifier,
+ * *id* is the unsigned int representation, while *id_val* holds the
+ * script-layer zeek::EnumVal.
+ */
+class EventMetadataDescriptor {
+public:
+    EventMetadataDescriptor(zeek_uint_t id, EnumValPtr id_val, TypePtr type)
+        : id(id), id_val(std::move(id_val)), type(std::move(type)) {}
+
+    zeek_uint_t Id() const { return id; }
+    const EnumValPtr& IdVal() const { return id_val; }
+    const TypePtr& Type() const { return type; }
+
+private:
+    zeek_uint_t id;
+    EnumValPtr id_val;
+    TypePtr type;
+};
+
 // The registry keeps track of all events that we provide or handle.
 class EventRegistry final {
 public:
@@ -98,6 +136,23 @@ public:
      */
     EventGroupPtr LookupGroup(EventGroupKind kind, std::string_view name);
 
+    /**
+     * Register a script-layer metadata identifier *id* with type *type*.
+     *
+     * @param id The script-level ``EventMetadata::ID`` enum value.
+     * @param type The type to expect for the given metadata identifier.
+     */
+    bool RegisterMetadata(EnumValPtr id, TypePtr type);
+
+    /**
+     * Lookup the MetadataDescriptor for metadata identifier *id*
+     *
+     * @param id The metadata identifier as unsigned int.
+     * @return A pointer to a MetadataDescriptor or nullptr.
+     */
+    const EventMetadataDescriptor* LookupMetadata(zeek_uint_t id) const;
+
+
 private:
     std::map<std::string, std::unique_ptr<EventHandler>, std::less<>> handlers;
     // Tracks whether a given event handler was registered in a
@@ -106,6 +161,9 @@ private:
 
     // Map event groups identified by kind and name to their instances.
     std::map<std::pair<EventGroupKind, std::string>, std::shared_ptr<EventGroup>, std::less<>> event_groups;
+
+    // Map for event metadata identifier to their descriptors types.
+    std::unordered_map<zeek_uint_t, EventMetadataDescriptor> event_metadata_types;
 };
 
 /**

src/Expr.cc

@@ -24,6 +24,8 @@
 #include "zeek/script_opt/Expr.h"
 #include "zeek/script_opt/ScriptOpt.h"
 
+#include "const.bif.netvar_h"
+
 namespace zeek::detail {
 
 const char* expr_name(ExprTag t) {
@@ -3827,11 +3829,17 @@ ScheduleTimer::ScheduleTimer(const EventHandlerPtr& arg_event, Args arg_args, do
 
 void ScheduleTimer::Dispatch(double /* t */, bool /* is_expire */) {
     if ( event ) {
-        // An event's intended timestamp might be in the past as timer expiration is driven by
+        detail::EventMetadataVectorPtr meta;
-        // network time. Guarantee that the intended timestamp is never in the future (e.g.,
+
-        // when all timers are expired on shutdown).
+        if ( BifConst::EventMetadata::add_network_timestamp ) {
-        auto ts = std::min(this->Time(), run_state::network_time);
+            // An event's intended timestamp might be in the past as timer expiration is driven by
-        event_mgr.Enqueue(event, std::move(args), util::detail::SOURCE_LOCAL, 0, nullptr, ts);
+            // network time. Guarantee that the intended timestamp is never in the future (e.g.,
+            // when all timers are expired on shutdown).
+            auto ts = std::min(this->Time(), run_state::network_time);
+            meta = detail::MakeEventMetadataVector(ts);
+        }
+
+        event_mgr.Enqueue(std::move(meta), event, std::move(args));
     }
 }
 

src/broker/Manager.cc

@@ -1655,8 +1655,13 @@ void Manager::ProcessMessage(std::string_view topic, broker::zeek::Event& ev) {
         }
     }
 
-    if ( vl.size() == args.size() )
+    if ( vl.size() == args.size() ) {
-        event_mgr.Enqueue(handler, std::move(vl), util::detail::SOURCE_BROKER, 0, nullptr, ts);
+        zeek::detail::EventMetadataVectorPtr meta;
+        if ( ts > 0.0 )
+            meta = zeek::detail::MakeEventMetadataVector(ts);
+
+        event_mgr.Enqueue(std::move(meta), handler, std::move(vl), util::detail::SOURCE_BROKER);
+    }
 }
 
 bool Manager::ProcessMessage(std::string_view, broker::zeek::LogCreate& lc) {

src/cluster/Backend.cc

@@ -23,7 +23,11 @@ using namespace zeek::cluster;
 
 
 bool detail::LocalEventHandlingStrategy::DoProcessEvent(std::string_view topic, detail::Event e) {
-    zeek::event_mgr.Enqueue(e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER, 0, nullptr, e.Timestamp());
+    zeek::detail::EventMetadataVectorPtr meta;
+    if ( auto ts = e.Timestamp(); ts > 0.0 )
+        meta = zeek::detail::MakeEventMetadataVector(e.Timestamp());
+
+    zeek::event_mgr.Enqueue(std::move(meta), e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER);
     return true;
 }
 

src/const.bif

@@ -34,3 +34,5 @@ const Log::flush_interval: interval;
 const Log::write_buffer_size: count;
 
 const Storage::expire_interval: interval;
+
+const EventMetadata::add_network_timestamp: bool;

src/script_opt/FuncInfo.cc

@@ -79,6 +79,9 @@ static std::unordered_map<std::string, unsigned int> func_attrs = {
     {"Cluster::__unsubscribe", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Cluster::make_event", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Cluster::publish", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"EventMetadata::current", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"EventMetadata::current_all", ATTR_NO_SCRIPT_SIDE_EFFECTS},
+    {"EventMetadata::register", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"FileExtract::__set_limit", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Files::__add_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS},
     {"Files::__analyzer_enabled", ATTR_NO_ZEEK_SIDE_EFFECTS},

src/zeek.bif

@@ -401,6 +401,77 @@ function current_event_time%(%): time
 	return zeek::make_intrusive<zeek::TimeVal>(zeek::event_mgr.CurrentEventTime());
 	%}
 
+## Register the expected Zeek type for event metadata.
+##
+## id: The event metadata identifier.
+##
+## t: A type expression or type alias. The type cannot be ``any``, ``func``,
+##    ``file``, ``opaque`` or a composite type containing one of these types.
+##
+## Returns: true if the registration was successful, false if *id* is
+##          registered with a different type already, or type is invalid.
+##
+## .. zeek:see:: EventMetadata::current EventMetadata::current_all
+function EventMetadata::register%(id: EventMetadata::ID, t: any%): bool
+       %{
+       const auto& tty = t->GetType();
+
+       if ( tty->Tag() != zeek::TYPE_TYPE )
+               {
+               zeek::emit_builtin_error("register_event_metadata() expects type");
+               return zeek::val_mgr->False();
+               }
+
+       EnumValPtr idvp = {zeek::NewRef{}, id->AsEnumVal()};
+
+       bool r = zeek::event_registry->RegisterMetadata(idvp, tty->AsTypeType()->GetType());
+       return zeek::val_mgr->Bool(r);
+       %}
+
+## Query the current event's metadata with identifier *id*.
+##
+## id: The metadata identifier, e.g. ``EventMetadata::NETWORK_TIMESTAMP``.
+##
+## Returns: A vector of values. The vector is empty if no metadata with
+##          the given identifier is attached to this event, otherwise a
+##          vector whose elements are of the type used during registration.
+##
+## .. zeek:see:: EventMetadata::register EventMetadata::current_all
+function EventMetadata::current%(id: EventMetadata::ID%): any_vec
+	%{
+	static const auto& vt = zeek::id::find_type<zeek::VectorType>("any_vec");
+
+	const auto* event = zeek::event_mgr.CurrentEvent();
+	if ( ! event )
+		return zeek::make_intrusive<zeek::VectorVal>(vt);
+
+	return event->MetadataValues({zeek::NewRef{}, id->AsEnumVal()});
+	%}
+
+## Query all of the current event's metadata.
+##
+## Returns: A vector :zeek:see:`EventMetadata::Entry` elements holding all
+##          the metadata attached to this event.
+##
+## .. zeek:see:: EventMetadata::register EventMetadata::current
+function EventMetadata::current_all%(%): event_metadata_vec
+	%{
+	static const auto& vt = zeek::id::find_type<zeek::VectorType>("event_metadata_vec");
+	auto result = zeek::make_intrusive<zeek::VectorVal>(vt);
+
+	const auto* event = zeek::event_mgr.CurrentEvent();
+	if ( ! event )
+		return result;
+
+	if ( const auto* mdv = event->Metadata() ) {
+		for ( const auto& entry : *mdv )
+			result->Append(entry.BuildVal());
+
+	}
+
+	return result;
+	%}
+
 ## Returns a system environment variable.
 ##
 ## var: The name of the variable whose value to request.

testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+new_connection CHhAvVGS1DHFjwGM9 all=[[id=EventMetadata::NETWORK_TIMESTAMP, val=1362692526.869344]] network_timestamp=[1362692526.869344]

testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+new_connection CHhAvVGS1DHFjwGM9 all=[] network_timestamp=[]

testing/btest/Baseline/core.event-metadata.non-event/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.non-event/.stdout

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.register/.stderr

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/core.event-metadata.register/.stdout

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/opt.ZAM-bif-tracking/output

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-560 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()
+563 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()

testing/btest/broker/remote_event_auto_ts.zeek

@@ -17,6 +17,7 @@
 # @TEST-START-FILE send.zeek
 
 redef exit_only_after_terminate = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global runs = 0;
 global ping: event(msg: string, intended_ts: time);
@@ -58,6 +59,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 # @TEST-START-FILE recv.zeek
 
 redef exit_only_after_terminate = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global msg_count = 0;
 

testing/btest/broker/remote_event_schedule_ts.zeek

@@ -17,6 +17,7 @@
 # @TEST-START-FILE send.zeek
 
 redef exit_only_after_terminate = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global runs = 0;
 global ping: event(msg: string, intended_ts: time, publish_ts: time);
@@ -64,6 +65,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 # @TEST-START-FILE recv.zeek
 
 redef exit_only_after_terminate = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global msg_count = 0;
 

testing/btest/broker/remote_event_ts.zeek

@@ -15,6 +15,7 @@
 # @TEST-START-FILE send.zeek
 
 redef exit_only_after_terminate = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global runs = 0;
 global ping: event(msg: string, intended_ts: time);
@@ -54,6 +55,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 # @TEST-START-FILE recv.zeek
 
 redef exit_only_after_terminate = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global msg_count = 0;
 

testing/btest/broker/remote_event_ts_compat.zeek

@@ -17,6 +17,7 @@
 
 redef exit_only_after_terminate = T;
 redef allow_network_time_forward = F;
+redef EventMetadata::add_network_timestamp = T;
 
 event zeek_init()
 	{

testing/btest/broker/web-socket-events-metadata.zeek

@@ -16,6 +16,7 @@
 redef allow_network_time_forward = F;
 redef exit_only_after_terminate = T;
 redef Broker::disable_ssl = T;
+redef EventMetadata::add_network_timestamp = T;
 
 global event_count = 0;
 

testing/btest/core/event-metadata/network-timestamp.zeek

@@ -0,0 +1,17 @@
+# @TEST-DOC: Check network timestamp available if opt-in.
+#
+# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=T > with_ts.out
+# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=F > without_ts.out
+#
+# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff with_ts.out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff without_ts.out
+# @TEST-EXEC: btest-diff .stderr
+
+
+event new_connection(c: connection)
+	{
+	print fmt("new_connection %s all=%s network_timestamp=%s",
+	          c$uid,
+	          EventMetadata::current_all(),
+	          EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP));
+	}

testing/btest/core/event-metadata/non-event.zeek

@@ -0,0 +1,8 @@
+# @TEST-DOC: Ensure EventMetadata::current() and EventMetadata::current_all() in non-event context returns empty vectors.
+#
+# @TEST-EXEC: zeek -b %INPUT
+# @TEST-EXEC: btest-diff .stdout
+# @TEST-EXEC: btest-diff .stderr
+
+assert |EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP)| == 0;
+assert |EventMetadata::current_all()| == 0;

testing/btest/core/event-metadata/register-type-errors.zeek

@@ -0,0 +1,30 @@
+# @TEST-DOC: Verify rejection of certain metadata types.
+#
+# @TEST-EXEC: unset ZEEK_ALLOW_INIT_ERRORS; zeek -b %INPUT
+# @TEST-EXEC: btest-diff .stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+
+module App;
+
+export {
+	redef enum EventMetadata::ID += {
+		MY_METADATA = 1000,
+		MY_TABLE = 1002,
+		MY_VECTOR = 1003,
+	};
+}
+
+type R: record {
+	f: file;
+	a: any;
+	l: function(x: count): bool;
+};
+
+event zeek_init()
+	{
+	assert ! EventMetadata::register(MY_METADATA, any);
+	assert ! EventMetadata::register(MY_METADATA, table[count] of any);
+	assert ! EventMetadata::register(MY_METADATA, table[count] of function(x: count): bool);
+	assert ! EventMetadata::register(MY_METADATA, R);
+	assert ! EventMetadata::register(MY_METADATA, vector of R);
+	}

testing/btest/core/event-metadata/register.zeek

@@ -0,0 +1,30 @@
+# @TEST-DOC: Very basic registration of event metadata identifiers.
+#
+# @TEST-EXEC: zeek -b %INPUT
+# @TEST-EXEC: btest-diff .stdout
+# @TEST-EXEC: btest-diff .stderr
+
+module App;
+
+export {
+	redef enum EventMetadata::ID += {
+		MY_STRING = 1000,
+		MY_COUNT = 1001,
+		MY_TABLE = 1002,
+	};
+}
+
+event zeek_init()
+	{
+	assert EventMetadata::register(MY_STRING, string);
+	assert EventMetadata::register(MY_STRING, string); # double register is okay
+	assert EventMetadata::register(MY_COUNT, count);
+	assert EventMetadata::register(MY_COUNT, count);
+	assert EventMetadata::register(MY_TABLE, table[string] of count);
+	assert EventMetadata::register(MY_TABLE, table[string] of count);
+
+	# Type mismatch all return F, but no output on stderr.
+	assert ! EventMetadata::register(MY_STRING, count);
+	assert ! EventMetadata::register(MY_COUNT, string);
+	assert ! EventMetadata::register(MY_TABLE, table[count] of string);
+	}

testing/btest/language/event-ts-scheduled.zeek

@@ -1,6 +1,8 @@
 # @TEST-EXEC: zeek -b -r $TRACES/ticks-dns-1hr.pcap %INPUT > out
 # @TEST-EXEC: btest-diff out
 
+redef EventMetadata::add_network_timestamp = T;
+
 global runs = 0;
 
 event test(schedule_time: time)

testing/btest/language/event-ts.zeek

@@ -4,6 +4,8 @@
 # Note: We use a PCAP with DNS queries only so that we have a single packet per
 # time step. Thus the run loop will be executed only once per time step.
 
+redef EventMetadata::add_network_timestamp = T;
+
 global runs = -1;
 
 event test(depth: count)

testing/btest/opt/ZAM-bif-tracking.zeek

@@ -106,6 +106,9 @@ global known_BiFs = set(
 	"Cluster::publish",
 	"Cluster::publish_hrw",
 	"Cluster::publish_rr",
+	"EventMetadata::current",
+	"EventMetadata::current_all",
+	"EventMetadata::register",
 	"FileExtract::__set_limit",
 	"Files::__add_analyzer",
 	"Files::__analyzer_enabled",