Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'origin/topic/awelzel/4177-4178-custom-event-metadata-part-1'

Browse source 
* origin/topic/awelzel/4177-4178-custom-event-metadata-part-1:
  Event: Move meta after args
  Event: Use IntrusivePtr to manage obj refcount
  btest/zam: Update for new EventMetadata bifs
  broker and cluster: Switch to new Enqueue() API
  Event/zeek.bif: Add EventMetadata current() and current_values() accessors
  Event: Deprecate default network timestamp metadata
  Event: Store timestamp in metadata vector
  EventRegistry/zeek.bif/init-bare: Add event metadata infrastructure
  EventMgr: Add CurrentEvent() accessor
This commit is contained in:
Arne Welzel 2025-05-23 20:57:54 +02:00
commit d929392a76
37 changed files with 656 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

11
CHANGES
View file

@ -1,3 +1,14 @@
8.0.0-dev.201 | 2025-05-23 20:57:54 +0200
* Add generic event metadata, part 1 (Arne Welzel, Corelight)
The ``ts`` member of ``Event`` was removed in favor of a more generic
approach that can hold any event metadata. Further, the Zeek script-layer
API for registering event metadata types was added.
Event instances do not network timestamp metadata by default anymore.
This feature can be re-enabled via EventMetadata::add_network_timestamp=T.
8.0.0-dev.191 | 2025-05-23 08:32:00 -0700
* Remove FindClangTidy.cmake, update cmake submodule (Tim Wojtulewicz, Corelight)

16
NEWS
View file

@ -15,6 +15,15 @@ Breaking Changes
files. We tested builds of all of the existing third-party packages and only noticed one
or two failures, but there is a possibility for breakage related to this cleanup.
- Network timestamps are not added to events by default anymore. Use the following
redef line to enable them:
redef EventMetadata::add_network_timestamp = T;
The background is that event metadata has become more generic and may incur
a small overhead when enabled. There's not enough users of network timestamp
metadata to justify the complexity of treating it separate.
New Functionality
-----------------
@ -65,6 +74,13 @@ Deprecated Functionality
The replacement script doesn't populate the ``email_body_sections`` anymore either.
- The ``zeek::Event()`` constructor was deprecated. Use ``event_mgr::Enqueue()``
or ``event_mgr::Dispatch()`` instead.
- Passing ``ts`` as the last argument to ``EventMgr::Enqueue()`` has been deprecated
and will lead to compile time warnings. Use ``EventMgr::Enqueue(detail::MetadataVectorPtr meta, ...)``
for populating ``meta`` accordingly.
Zeek 7.2.0
==========

2
VERSION
View file

@ -1 +1 @@
8.0.0-dev.191
8.0.0-dev.201

27
scripts/base/init-bare.zeek
View file

@ -579,6 +579,30 @@ const io_poll_interval_live = 10 &redef;
## while testing, but should be used sparingly.
const running_under_test: bool = F &redef;
module EventMetadata;
export {
## Enum type for metadata identifiers.
type ID: enum {
NETWORK_TIMESTAMP = 1,
};
## A event metadata entry.
type Entry: record {
id: EventMetadata::ID; ##< The registered :zeek:see:`EventMetadata::ID` value.
val: any; ##< The value. Its type matches what was passed to :zeek:see:`EventMetadata::register`.
};
## Add network timestamp metadata to all events.
##
## Adding network timestamp metadata affects local and
## remote events. Events scheduled have a network timestamp
## of when the scheduled timer was supposed to expire, which
## might be a value before the network_time() when the event
## was actually dispatched.
const add_network_timestamp: bool = F &redef;
}
module FTP;
export {
@ -751,6 +775,9 @@ module GLOBAL;
## directly and then remove this alias.
type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn;
## A type alias for event metadata.
type event_metadata_vec: vector of EventMetadata::Entry;
## Statistics about a :zeek:type:`connection` endpoint.
##
## .. zeek:see:: connection

181
src/Event.cc
View file

@ -2,29 +2,114 @@
#include "zeek/Event.h"
#include <cinttypes>
#include "zeek/Desc.h"
#include "zeek/EventRegistry.h"
#include "zeek/Trigger.h"
#include "zeek/Type.h"
#include "zeek/Val.h"
#include "zeek/iosource/Manager.h"
#include "zeek/plugin/Manager.h"
#include "const.bif.netvar_h"
#include "event.bif.netvar_h"
zeek::EventMgr zeek::event_mgr;
namespace zeek {
detail::EventMetadataVectorPtr detail::MakeEventMetadataVector(double t) {
auto tv = make_intrusive<TimeVal>(t);
auto entry = detail::MetadataEntry{static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp), std::move(tv)};
return std::make_unique<detail::EventMetadataVector>(std::vector{std::move(entry)});
}
RecordValPtr detail::MetadataEntry::BuildVal() const {
static const auto rt = id::find_type<RecordType>("EventMetadata::Entry");
auto rv = make_intrusive<RecordVal>(rt);
const auto* desc = event_registry->LookupMetadata(id);
if ( ! desc ) {
zeek::reporter->InternalWarning("unable to find metadata descriptor for id %" PRIu64, id);
return rv;
}
rv->Assign(0, desc->IdVal());
rv->Assign(1, val);
return rv;
}
Event::Event(const EventHandlerPtr& arg_handler, zeek::Args arg_args, util::detail::SourceID arg_src,
analyzer::ID arg_aid, Obj* arg_obj, double arg_ts)
: handler(arg_handler),
args(std::move(arg_args)),
meta(detail::MakeEventMetadataVector(arg_ts)),
src(arg_src),
aid(arg_aid),
ts(arg_ts),
obj(arg_obj),
next_event(nullptr) {
if ( obj )
Ref(obj);
obj(zeek::NewRef{}, arg_obj),
next_event(nullptr) {}
Event::Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args,
util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj)
: handler(arg_handler),
args(std::move(arg_args)),
meta(std::move(arg_meta)),
src(arg_src),
aid(arg_aid),
obj(zeek::NewRef{}, arg_obj),
next_event(nullptr) {}
zeek::VectorValPtr Event::MetadataValues(const EnumValPtr& id) const {
static const auto& any_vec_t = zeek::id::find_type<zeek::VectorType>("any_vec");
auto result = zeek::make_intrusive<zeek::VectorVal>(any_vec_t);
if ( ! meta )
return result;
auto id_int = id->Get();
if ( id_int < 0 )
zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int);
zeek_uint_t uintid = static_cast<zeek_uint_t>(id_int);
const auto* desc = event_registry->LookupMetadata(uintid);
if ( ! desc )
return result;
for ( const auto& entry : *meta ) {
if ( entry.Id() != uintid )
continue;
// Sanity check the type.
if ( ! same_type(desc->Type(), entry.Val()->GetType()) ) {
zeek::reporter->InternalWarning("metadata has unexpected type %s, wanted %s",
obj_desc_short(entry.Val()->GetType().get()).c_str(),
obj_desc_short(desc->Type().get()).c_str());
continue;
}
result->Append(entry.Val());
}
return result;
}
double Event::Time() const {
if ( ! meta )
return 0.0;
for ( const auto& m : *meta )
if ( m.Id() == static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp) ) {
if ( m.Val()->GetType()->Tag() != TYPE_TIME ) {
// This should've been caught during parsing.
zeek::reporter->InternalError("event metadata timestamp has wrong type: %s",
obj_desc_short(m.Val()->GetType().get()).c_str());
}
return m.Val()->AsTime();
}
return 0.0;
}
void Event::Describe(ODesc* d) const {
@ -52,7 +137,7 @@ void Event::Dispatch(bool no_remote) {
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
// Replace in v8.1 with handler->Call(&args).
handler->Call(&args, no_remote, ts);
handler->Call(&args, no_remote, Time());
#pragma GCC diagnostic pop
}
@ -60,9 +145,8 @@ void Event::Dispatch(bool no_remote) {
// Already reported.
}
if ( obj )
// obj->EventDone();
Unref(obj);
// Unref obj
obj.reset();
if ( handler->ErrorHandler() )
reporter->EndErrorHandler();
@ -77,8 +161,60 @@ EventMgr::~EventMgr() {
}
void EventMgr::Enqueue(const EventHandlerPtr& h, Args vl, util::detail::SourceID src, analyzer::ID aid, Obj* obj,
double ts) {
QueueEvent(new Event(h, std::move(vl), src, aid, obj, ts));
DeprecatedTimestamp deprecated_ts) {
detail::EventMetadataVectorPtr meta;
double ts = double(deprecated_ts);
if ( BifConst::EventMetadata::add_network_timestamp ) {
if ( ts < 0.0 ) // default -1.0, modify to current network_time
ts = run_state::network_time;
// In v8.1 when the deprecated_ts parameters is gone: Just use run_state::network_time directly here.
meta = detail::MakeEventMetadataVector(ts);
}
else if ( ts >= 0.0 ) {
// EventMetadata::add_network_timestamp is false, but EventMgr::Enqueue()
// with an explicit (non-negative) timestamp is used. That's a deprecated
// API, but we continue to support it until v8.1.
meta = detail::MakeEventMetadataVector(ts);
}
QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj));
}
void EventMgr::Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, Args vl,
util::detail::SourceID src, analyzer::ID aid, Obj* obj) {
if ( BifConst::EventMetadata::add_network_timestamp ) {
// If all events are supposed to have a network time attached, ensure
// that the meta vector was passed *and* contains a network timestamp.
bool has_time = false;
if ( ! meta ) {
// No metadata vector at all, make one with a timestamp.
meta = detail::MakeEventMetadataVector(run_state::network_time);
}
else {
// Check all entries for a network timestamp
for ( const auto& m : *meta ) {
if ( m.Id() == static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp) ) {
has_time = true;
if ( m.Val()->GetType()->Tag() != TYPE_TIME ) {
// This should've been caught during parsing.
zeek::reporter->InternalError("event metadata timestamp has wrong type: %s",
obj_desc_short(m.Val()->GetType().get()).c_str());
}
}
}
if ( ! has_time ) {
auto tv = zeek::make_intrusive<zeek::TimeVal>(run_state::network_time);
meta->push_back({static_cast<zeek_uint_t>(detail::MetadataType::NetworkTimestamp), std::move(tv)});
}
}
}
QueueEvent(new Event(std::move(meta), h, std::move(vl), src, aid, obj));
}
void EventMgr::QueueEvent(Event* event) {
@ -107,7 +243,13 @@ void EventMgr::Dispatch(Event* event, bool no_remote) {
}
void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) {
auto* ev = new Event(h, std::move(vl));
detail::EventMetadataVectorPtr meta;
// If all events should have network timestamps, create the vector holding one.
if ( BifConst::EventMetadata::add_network_timestamp )
meta = detail::MakeEventMetadataVector(run_state::network_time);
auto* ev = new Event(std::move(meta), h, std::move(vl), util::detail::SOURCE_LOCAL, 0, nullptr);
// Technically this isn't queued, but still give plugins a chance to
// intercept the event and cancel or modify it if really wanted.
@ -182,6 +324,19 @@ void EventMgr::Process() {
// and had the opportunity to spawn new events.
}
void EventMgr::InitPostScript() { iosource_mgr->Register(this, true, false); }
void EventMgr::InitPostScript() {
// Check if expected types and identifiers are available.
const auto& et = zeek::id::find_type<zeek::EnumType>("EventMetadata::ID");
if ( ! et )
zeek::reporter->FatalError("Failed to find EventMetadata::ID");
const auto& net_ts_val = et->GetEnumVal(et->Lookup("EventMetadata::NETWORK_TIMESTAMP"));
if ( ! net_ts_val )
zeek::reporter->FatalError("Failed to lookup EventMetadata::NETWORK_TIMESTAMP");
if ( ! zeek::event_registry->RegisterMetadata(net_ts_val, zeek::base_type(zeek::TYPE_TIME)) )
zeek::reporter->FatalError("Failed to register NETWORK_TIMESTAMP metadata");
iosource_mgr->Register(this, true, false);
}
} // namespace zeek

98
src/Event.h
View file

@ -4,6 +4,7 @@
#include <tuple>
#include <type_traits>
#include <vector>
#include "zeek/ZeekArgs.h"
#include "zeek/analyzer/Analyzer.h"
@ -18,8 +19,41 @@ extern double network_time;
class EventMgr;
namespace detail {
/**
* An event metadata entry as stored in Event or @ref zeek::cluster::detail::Event.
*/
class MetadataEntry {
public:
MetadataEntry(zeek_uint_t id, zeek::ValPtr val) : id(id), val(std::move(val)) {}
zeek_uint_t Id() const { return id; }
const zeek::ValPtr& Val() const { return val; }
/**
* @return Pointer to a script-layer ``EventMetadata::Entry`` zeek::RecordVal representing this metadata entry.
*/
RecordValPtr BuildVal() const;
private:
zeek_uint_t id;
zeek::ValPtr val;
};
using EventMetadataVector = std::vector<MetadataEntry>;
using EventMetadataVectorPtr = std::unique_ptr<EventMetadataVector>;
/**
* @return A new event metadata vector containing network timestamp value set to \a t;
*/
EventMetadataVectorPtr MakeEventMetadataVector(double t);
} // namespace detail
class Event final : public Obj {
public:
[[deprecated("Remove in v8.1: Do not instantiate raw events. Use EventMgr::Dispatch() or EventMgr::Enqueue().")]]
Event(const EventHandlerPtr& handler, zeek::Args args, util::detail::SourceID src = util::detail::SOURCE_LOCAL,
analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time);
@ -30,27 +64,45 @@ public:
analyzer::ID Analyzer() const { return aid; }
EventHandlerPtr Handler() const { return handler; }
const zeek::Args& Args() const { return args; }
double Time() const { return ts; }
double Time() const;
/**
* @return a pointer to the MetadataVector of this event or a nullptr.
*/
const detail::EventMetadataVector* Metadata() const { return meta.get(); }
/**
* @return a vector of values for metadata matching identifier \a id.
*
* @param id The metadata identifier as an enum value.
*/
VectorValPtr MetadataValues(const EnumValPtr& id) const;
void Describe(ODesc* d) const override;
private:
friend class EventMgr;
// Construct an event with a metadata vector. Passing arg_meta as nullptr is explicitly allowed.
Event(detail::EventMetadataVectorPtr arg_meta, const EventHandlerPtr& arg_handler, zeek::Args arg_args,
util::detail::SourceID arg_src, analyzer::ID arg_aid, Obj* arg_obj);
// This method is protected to make sure that everybody goes through
// EventMgr::Dispatch().
void Dispatch(bool no_remote = false);
EventHandlerPtr handler;
zeek::Args args;
detail::EventMetadataVectorPtr meta;
util::detail::SourceID src;
analyzer::ID aid;
double ts;
Obj* obj;
zeek::IntrusivePtr<Obj> obj;
Event* next_event;
};
class EventMgr final : public Obj, public iosource::IOSource {
class DeprecatedTimestamp;
public:
~EventMgr() override;
@ -66,10 +118,10 @@ public:
* @param obj an arbitrary object to use as a "cookie" or just hold a
* reference to until dispatching the event.
* @param ts timestamp at which the event is intended to be executed
* (defaults to current network time).
* (defaults to current network time - deprecated).
*/
void Enqueue(const EventHandlerPtr& h, zeek::Args vl, util::detail::SourceID src = util::detail::SOURCE_LOCAL,
analyzer::ID aid = 0, Obj* obj = nullptr, double ts = run_state::network_time);
analyzer::ID aid = 0, Obj* obj = nullptr, DeprecatedTimestamp ts = {});
/**
* A version of Enqueue() taking a variable number of arguments.
@ -80,6 +132,19 @@ public:
return Enqueue(h, zeek::Args{std::forward<Args>(args)...});
}
/**
* Enqueue() with metadata vector support.
* @param meta Metadata to attach to the event, can be nullptr.
* @param h reference to the event handler to later call.
* @param vl the argument list to the event handler call.
* @param src indicates the origin of the event (local versus remote).
* @param aid identifies the protocol analyzer generating the event.
* @param obj an arbitrary object to use as a "cookie" or just hold a
* reference to until dispatching the event.
*/
void Enqueue(detail::EventMetadataVectorPtr meta, const EventHandlerPtr& h, zeek::Args vl,
util::detail::SourceID src = util::detail::SOURCE_LOCAL, analyzer::ID aid = 0, Obj* obj = nullptr);
[[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]]
void Dispatch(Event* event, bool no_remote = false);
@ -116,6 +181,11 @@ public:
// by returning a zero-timeout.
double GetNextTimeout() override { return head ? 0.0 : -1.0; }
/**
* @return A pointer to the currently dispatched event or nullptr.
*/
const Event* CurrentEvent() const { return current; }
void Process() override;
const char* Tag() override { return "EventManager"; }
void InitPostScript();
@ -124,6 +194,24 @@ public:
uint64_t num_events_dispatched = 0;
private:
/**
* Helper class to produce a compile time warning if Enqueue() is called with an explicit timestamp.
*
* Remove in v8.1.
*/
class DeprecatedTimestamp {
public:
DeprecatedTimestamp() : d(-1.0) {}
[[deprecated("Use overload EventMgr::Enqueue(EventMetadataVectorPtr meta, ...) to pass timestamp metadata")]]
/*implicit*/ DeprecatedTimestamp(double d)
: d(d) {}
explicit operator double() const { return d; }
private:
double d;
};
void QueueEvent(Event* event);
Event* current = nullptr;

68
src/EventRegistry.cc
View file

@ -3,11 +3,16 @@
#include "zeek/EventRegistry.h"
#include <algorithm>
#include <cinttypes>
#include "zeek/Desc.h"
#include "zeek/EventHandler.h"
#include "zeek/Func.h"
#include "zeek/RE.h"
#include "zeek/Reporter.h"
#include "zeek/Traverse.h"
#include "zeek/TraverseTypes.h"
#include "zeek/Type.h"
namespace zeek {
@ -164,4 +169,67 @@ void EventGroup::Disable() {
void EventGroup::AddFunc(detail::ScriptFuncPtr f) { funcs.insert(f); }
namespace {
class EventMetadataTypeRejector : public detail::TraversalCallback {
public:
detail::TraversalCode PreType(const Type* t) override {
if ( visited.count(t) > 0 )
return detail::TC_ABORTSTMT;
visited.insert(t);
if ( reject.count(t->Tag()) )
rejected.push_back(t);
return detail::TC_CONTINUE;
};
std::set<const zeek::Type*> visited;
std::vector<const zeek::Type*> rejected;
std::set<zeek::TypeTag> reject = {TYPE_ANY, TYPE_FUNC, TYPE_FILE, TYPE_OPAQUE};
};
} // namespace
bool EventRegistry::RegisterMetadata(EnumValPtr id, TypePtr type) {
static const auto& metadata_id_type = id::find_type<EnumType>("EventMetadata::ID");
if ( metadata_id_type != id->GetType() )
return false;
auto id_int = id->Get();
if ( id_int < 0 ) {
zeek::reporter->InternalError("Negative enum value %s: %" PRId64, obj_desc_short(id.get()).c_str(), id_int);
}
zeek_uint_t id_uint = static_cast<zeek_uint_t>(id_int);
if ( auto it = event_metadata_types.find(id_uint); it != event_metadata_types.end() )
return same_type(it->second.Type(), type);
EventMetadataTypeRejector cb;
type->Traverse(&cb);
if ( cb.rejected.size() > 0 )
return false;
event_metadata_types.insert({id_uint, EventMetadataDescriptor{id_uint, std::move(id), std::move(type)}});
return true;
}
const EventMetadataDescriptor* EventRegistry::LookupMetadata(zeek_uint_t id) const {
const auto it = event_metadata_types.find(id);
if ( it == event_metadata_types.end() )
return nullptr;
if ( it->second.Id() != id ) {
zeek::reporter->InternalError("inconsistent metadata descriptor: %" PRIu64 " vs %" PRId64, it->second.Id(), id);
}
return &(it->second);
}
} // namespace zeek

58
src/EventRegistry.h
View file

@ -8,6 +8,7 @@
#include <memory>
#include <string>
#include <string_view>
#include <unordered_map>
#include <unordered_set>
#include <vector>
@ -22,18 +23,55 @@ enum class EventGroupKind {
Module,
};
class EnumVal;
class EventGroup;
class EventHandler;
class EventHandlerPtr;
class RE_Matcher;
class RecordVal;
class Type;
using EnumValPtr = IntrusivePtr<EnumVal>;
using EventGroupPtr = std::shared_ptr<EventGroup>;
using RecordValPtr = IntrusivePtr<RecordVal>;
using TypePtr = IntrusivePtr<Type>;
namespace detail {
class ScriptFunc;
using ScriptFuncPtr = zeek::IntrusivePtr<ScriptFunc>;
/**
* Well-known event metadata identifiers.
*/
enum class MetadataType : uint8_t {
NetworkTimestamp = 1,
};
} // namespace detail
/**
* Descriptor for event metadata.
*
* Event metadata is registered via @ref EventRegistry::RegisterMetadata. The descriptor
* holds the metadata identifier and registered type. For the identifier,
* *id* is the unsigned int representation, while *id_val* holds the
* script-layer zeek::EnumVal.
*/
class EventMetadataDescriptor {
public:
EventMetadataDescriptor(zeek_uint_t id, EnumValPtr id_val, TypePtr type)
: id(id), id_val(std::move(id_val)), type(std::move(type)) {}
zeek_uint_t Id() const { return id; }
const EnumValPtr& IdVal() const { return id_val; }
const TypePtr& Type() const { return type; }
private:
zeek_uint_t id;
EnumValPtr id_val;
TypePtr type;
};
// The registry keeps track of all events that we provide or handle.
class EventRegistry final {
public:
@ -98,6 +136,23 @@ public:
*/
EventGroupPtr LookupGroup(EventGroupKind kind, std::string_view name);
/**
* Register a script-layer metadata identifier *id* with type *type*.
*
* @param id The script-level ``EventMetadata::ID`` enum value.
* @param type The type to expect for the given metadata identifier.
*/
bool RegisterMetadata(EnumValPtr id, TypePtr type);
/**
* Lookup the MetadataDescriptor for metadata identifier *id*
*
* @param id The metadata identifier as unsigned int.
* @return A pointer to a MetadataDescriptor or nullptr.
*/
const EventMetadataDescriptor* LookupMetadata(zeek_uint_t id) const;
private:
std::map<std::string, std::unique_ptr<EventHandler>, std::less<>> handlers;
// Tracks whether a given event handler was registered in a
@ -106,6 +161,9 @@ private:
// Map event groups identified by kind and name to their instances.
std::map<std::pair<EventGroupKind, std::string>, std::shared_ptr<EventGroup>, std::less<>> event_groups;
// Map for event metadata identifier to their descriptors types.
std::unordered_map<zeek_uint_t, EventMetadataDescriptor> event_metadata_types;
};
/**

18
src/Expr.cc
View file

@ -24,6 +24,8 @@
#include "zeek/script_opt/Expr.h"
#include "zeek/script_opt/ScriptOpt.h"
#include "const.bif.netvar_h"
namespace zeek::detail {
const char* expr_name(ExprTag t) {
@ -3827,11 +3829,17 @@ ScheduleTimer::ScheduleTimer(const EventHandlerPtr& arg_event, Args arg_args, do
void ScheduleTimer::Dispatch(double /* t */, bool /* is_expire */) {
if ( event ) {
// An event's intended timestamp might be in the past as timer expiration is driven by
// network time. Guarantee that the intended timestamp is never in the future (e.g.,
// when all timers are expired on shutdown).
auto ts = std::min(this->Time(), run_state::network_time);
event_mgr.Enqueue(event, std::move(args), util::detail::SOURCE_LOCAL, 0, nullptr, ts);
detail::EventMetadataVectorPtr meta;
if ( BifConst::EventMetadata::add_network_timestamp ) {
// An event's intended timestamp might be in the past as timer expiration is driven by
// network time. Guarantee that the intended timestamp is never in the future (e.g.,
// when all timers are expired on shutdown).
auto ts = std::min(this->Time(), run_state::network_time);
meta = detail::MakeEventMetadataVector(ts);
}
event_mgr.Enqueue(std::move(meta), event, std::move(args));
}
}

9
src/broker/Manager.cc
View file

@ -1655,8 +1655,13 @@ void Manager::ProcessMessage(std::string_view topic, broker::zeek::Event& ev) {
}
}
if ( vl.size() == args.size() )
event_mgr.Enqueue(handler, std::move(vl), util::detail::SOURCE_BROKER, 0, nullptr, ts);
if ( vl.size() == args.size() ) {
zeek::detail::EventMetadataVectorPtr meta;
if ( ts > 0.0 )
meta = zeek::detail::MakeEventMetadataVector(ts);
event_mgr.Enqueue(std::move(meta), handler, std::move(vl), util::detail::SOURCE_BROKER);
}
}
bool Manager::ProcessMessage(std::string_view, broker::zeek::LogCreate& lc) {

6
src/cluster/Backend.cc
View file

@ -23,7 +23,11 @@ using namespace zeek::cluster;
bool detail::LocalEventHandlingStrategy::DoProcessEvent(std::string_view topic, detail::Event e) {
zeek::event_mgr.Enqueue(e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER, 0, nullptr, e.Timestamp());
zeek::detail::EventMetadataVectorPtr meta;
if ( auto ts = e.Timestamp(); ts > 0.0 )
meta = zeek::detail::MakeEventMetadataVector(e.Timestamp());
zeek::event_mgr.Enqueue(std::move(meta), e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER);
return true;
}

2
src/const.bif
View file

@ -34,3 +34,5 @@ const Log::flush_interval: interval;
const Log::write_buffer_size: count;
const Storage::expire_interval: interval;
const EventMetadata::add_network_timestamp: bool;

3
src/script_opt/FuncInfo.cc
View file

@ -79,6 +79,9 @@ static std::unordered_map<std::string, unsigned int> func_attrs = {
{"Cluster::__unsubscribe", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"Cluster::make_event", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"Cluster::publish", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"EventMetadata::current", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"EventMetadata::current_all", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"EventMetadata::register", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"FileExtract::__set_limit", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"Files::__add_analyzer", ATTR_NO_SCRIPT_SIDE_EFFECTS},
{"Files::__analyzer_enabled", ATTR_NO_ZEEK_SIDE_EFFECTS},

71
src/zeek.bif
View file

@ -401,6 +401,77 @@ function current_event_time%(%): time
return zeek::make_intrusive<zeek::TimeVal>(zeek::event_mgr.CurrentEventTime());
%}
## Register the expected Zeek type for event metadata.
##
## id: The event metadata identifier.
##
## t: A type expression or type alias. The type cannot be ``any``, ``func``,
## ``file``, ``opaque`` or a composite type containing one of these types.
##
## Returns: true if the registration was successful, false if *id* is
## registered with a different type already, or type is invalid.
##
## .. zeek:see:: EventMetadata::current EventMetadata::current_all
function EventMetadata::register%(id: EventMetadata::ID, t: any%): bool
%{
const auto& tty = t->GetType();
if ( tty->Tag() != zeek::TYPE_TYPE )
{
zeek::emit_builtin_error("register_event_metadata() expects type");
return zeek::val_mgr->False();
}
EnumValPtr idvp = {zeek::NewRef{}, id->AsEnumVal()};
bool r = zeek::event_registry->RegisterMetadata(idvp, tty->AsTypeType()->GetType());
return zeek::val_mgr->Bool(r);
%}
## Query the current event's metadata with identifier *id*.
##
## id: The metadata identifier, e.g. ``EventMetadata::NETWORK_TIMESTAMP``.
##
## Returns: A vector of values. The vector is empty if no metadata with
## the given identifier is attached to this event, otherwise a
## vector whose elements are of the type used during registration.
##
## .. zeek:see:: EventMetadata::register EventMetadata::current_all
function EventMetadata::current%(id: EventMetadata::ID%): any_vec
%{
static const auto& vt = zeek::id::find_type<zeek::VectorType>("any_vec");
const auto* event = zeek::event_mgr.CurrentEvent();
if ( ! event )
return zeek::make_intrusive<zeek::VectorVal>(vt);
return event->MetadataValues({zeek::NewRef{}, id->AsEnumVal()});
%}
## Query all of the current event's metadata.
##
## Returns: A vector :zeek:see:`EventMetadata::Entry` elements holding all
## the metadata attached to this event.
##
## .. zeek:see:: EventMetadata::register EventMetadata::current
function EventMetadata::current_all%(%): event_metadata_vec
%{
static const auto& vt = zeek::id::find_type<zeek::VectorType>("event_metadata_vec");
auto result = zeek::make_intrusive<zeek::VectorVal>(vt);
const auto* event = zeek::event_mgr.CurrentEvent();
if ( ! event )
return result;
if ( const auto* mdv = event->Metadata() ) {
for ( const auto& entry : *mdv )
result->Append(entry.BuildVal());
}
return result;
%}
## Returns a system environment variable.
##
## var: The name of the variable whose value to request.

1
testing/btest/Baseline/core.event-metadata.network-timestamp/.stderr Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

1
testing/btest/Baseline/core.event-metadata.network-timestamp/.stdout Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

2
testing/btest/Baseline/core.event-metadata.network-timestamp/with_ts.out Normal file
View file

@ -0,0 +1,2 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
new_connection CHhAvVGS1DHFjwGM9 all=[[id=EventMetadata::NETWORK_TIMESTAMP, val=1362692526.869344]] network_timestamp=[1362692526.869344]

2
testing/btest/Baseline/core.event-metadata.network-timestamp/without_ts.out Normal file
View file

@ -0,0 +1,2 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
new_connection CHhAvVGS1DHFjwGM9 all=[] network_timestamp=[]

1
testing/btest/Baseline/core.event-metadata.non-event/.stderr Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

1
testing/btest/Baseline/core.event-metadata.non-event/.stdout Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

1
testing/btest/Baseline/core.event-metadata.register-type-errors/.stderr Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

1
testing/btest/Baseline/core.event-metadata.register-type-errors/.stdout Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

1
testing/btest/Baseline/core.event-metadata.register/.stderr Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

1
testing/btest/Baseline/core.event-metadata.register/.stdout Normal file
View file

@ -0,0 +1 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

2
testing/btest/Baseline/opt.ZAM-bif-tracking/output
View file

@ -1,2 +1,2 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
560 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()
563 seen BiFs, 0 unseen BiFs (), 0 new BiFs ()

2
testing/btest/broker/remote_event_auto_ts.zeek
View file

@ -17,6 +17,7 @@
# @TEST-START-FILE send.zeek
redef exit_only_after_terminate = T;
redef EventMetadata::add_network_timestamp = T;
global runs = 0;
global ping: event(msg: string, intended_ts: time);
@ -58,6 +59,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
# @TEST-START-FILE recv.zeek
redef exit_only_after_terminate = T;
redef EventMetadata::add_network_timestamp = T;
global msg_count = 0;

2
testing/btest/broker/remote_event_schedule_ts.zeek
View file

@ -17,6 +17,7 @@
# @TEST-START-FILE send.zeek
redef exit_only_after_terminate = T;
redef EventMetadata::add_network_timestamp = T;
global runs = 0;
global ping: event(msg: string, intended_ts: time, publish_ts: time);
@ -64,6 +65,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
# @TEST-START-FILE recv.zeek
redef exit_only_after_terminate = T;
redef EventMetadata::add_network_timestamp = T;
global msg_count = 0;

2
testing/btest/broker/remote_event_ts.zeek
View file

@ -15,6 +15,7 @@
# @TEST-START-FILE send.zeek
redef exit_only_after_terminate = T;
redef EventMetadata::add_network_timestamp = T;
global runs = 0;
global ping: event(msg: string, intended_ts: time);
@ -54,6 +55,7 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
# @TEST-START-FILE recv.zeek
redef exit_only_after_terminate = T;
redef EventMetadata::add_network_timestamp = T;
global msg_count = 0;

1
testing/btest/broker/remote_event_ts_compat.zeek
View file

@ -17,6 +17,7 @@
redef exit_only_after_terminate = T;
redef allow_network_time_forward = F;
redef EventMetadata::add_network_timestamp = T;
event zeek_init()
{

1
testing/btest/broker/web-socket-events-metadata.zeek
View file

@ -16,6 +16,7 @@
redef allow_network_time_forward = F;
redef exit_only_after_terminate = T;
redef Broker::disable_ssl = T;
redef EventMetadata::add_network_timestamp = T;
global event_count = 0;

17
testing/btest/core/event-metadata/network-timestamp.zeek Normal file
View file

@ -0,0 +1,17 @@
# @TEST-DOC: Check network timestamp available if opt-in.
#
# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=T > with_ts.out
# @TEST-EXEC: zeek -r $TRACES/http/get.trace %INPUT EventMetadata::add_network_timestamp=F > without_ts.out
#
# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff with_ts.out
# @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff without_ts.out
# @TEST-EXEC: btest-diff .stderr
event new_connection(c: connection)
{
print fmt("new_connection %s all=%s network_timestamp=%s",
c$uid,
EventMetadata::current_all(),
EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP));
}

8
testing/btest/core/event-metadata/non-event.zeek Normal file
View file

@ -0,0 +1,8 @@
# @TEST-DOC: Ensure EventMetadata::current() and EventMetadata::current_all() in non-event context returns empty vectors.
#
# @TEST-EXEC: zeek -b %INPUT
# @TEST-EXEC: btest-diff .stdout
# @TEST-EXEC: btest-diff .stderr
assert |EventMetadata::current(EventMetadata::NETWORK_TIMESTAMP)| == 0;
assert |EventMetadata::current_all()| == 0;

30
testing/btest/core/event-metadata/register-type-errors.zeek Normal file
View file

@ -0,0 +1,30 @@
# @TEST-DOC: Verify rejection of certain metadata types.
#
# @TEST-EXEC: unset ZEEK_ALLOW_INIT_ERRORS; zeek -b %INPUT
# @TEST-EXEC: btest-diff .stdout
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
module App;
export {
redef enum EventMetadata::ID += {
MY_METADATA = 1000,
MY_TABLE = 1002,
MY_VECTOR = 1003,
};
}
type R: record {
f: file;
a: any;
l: function(x: count): bool;
};
event zeek_init()
{
assert ! EventMetadata::register(MY_METADATA, any);
assert ! EventMetadata::register(MY_METADATA, table[count] of any);
assert ! EventMetadata::register(MY_METADATA, table[count] of function(x: count): bool);
assert ! EventMetadata::register(MY_METADATA, R);
assert ! EventMetadata::register(MY_METADATA, vector of R);
}

30
testing/btest/core/event-metadata/register.zeek Normal file
View file

@ -0,0 +1,30 @@
# @TEST-DOC: Very basic registration of event metadata identifiers.
#
# @TEST-EXEC: zeek -b %INPUT
# @TEST-EXEC: btest-diff .stdout
# @TEST-EXEC: btest-diff .stderr
module App;
export {
redef enum EventMetadata::ID += {
MY_STRING = 1000,
MY_COUNT = 1001,
MY_TABLE = 1002,
};
}
event zeek_init()
{
assert EventMetadata::register(MY_STRING, string);
assert EventMetadata::register(MY_STRING, string); # double register is okay
assert EventMetadata::register(MY_COUNT, count);
assert EventMetadata::register(MY_COUNT, count);
assert EventMetadata::register(MY_TABLE, table[string] of count);
assert EventMetadata::register(MY_TABLE, table[string] of count);
# Type mismatch all return F, but no output on stderr.
assert ! EventMetadata::register(MY_STRING, count);
assert ! EventMetadata::register(MY_COUNT, string);
assert ! EventMetadata::register(MY_TABLE, table[count] of string);
}

2
testing/btest/language/event-ts-scheduled.zeek
View file

@ -1,6 +1,8 @@
# @TEST-EXEC: zeek -b -r $TRACES/ticks-dns-1hr.pcap %INPUT > out
# @TEST-EXEC: btest-diff out
redef EventMetadata::add_network_timestamp = T;
global runs = 0;
event test(schedule_time: time)

2
testing/btest/language/event-ts.zeek
View file

@ -4,6 +4,8 @@
# Note: We use a PCAP with DNS queries only so that we have a single packet per
# time step. Thus the run loop will be executed only once per time step.
redef EventMetadata::add_network_timestamp = T;
global runs = -1;
event test(depth: count)

3
testing/btest/opt/ZAM-bif-tracking.zeek
View file

@ -106,6 +106,9 @@ global known_BiFs = set(
"Cluster::publish",
"Cluster::publish_hrw",
"Cluster::publish_rr",
"EventMetadata::current",
"EventMetadata::current_all",
"EventMetadata::register",
"FileExtract::__set_limit",
"Files::__add_analyzer",
"Files::__analyzer_enabled",