Merge remote-tracking branch 'origin/topic/awelzel/generic-metadata-pre-work'

* origin/topic/awelzel/generic-metadata-pre-work:
  cluster/Backend: Add name and lookup component tag
  cluster/Event: Hide members behind accessors
  cluster/PublishEvent:: Make event non-const
  broker/Manager: Re-use broker serializer for conversion
  EventMgr: Add Dispatch() with handler and args
  plugin/Manager: Fix MetaHookPre and MetaHookPost using HOOK_CALL_FUNCTION
This commit is contained in:
Arne Welzel 2025-04-13 17:11:40 +02:00
commit c2e039f14d
26 changed files with 318 additions and 66 deletions

29
CHANGES
View file

@ -1,3 +1,32 @@
7.2.0-dev.517 | 2025-04-13 17:11:40 +0200
* cluster/Backend: Add name and lookup component tag (Arne Welzel, Corelight)
This adds two new accessors on Backend, Name() and Tag() that can
be used for introspection of a Backend instance.
* cluster/Event: Hide members behind accessors (Arne Welzel, Corelight)
* cluster/PublishEvent:: Make event non-const (Arne Welzel, Corelight)
We want to introduce a hook that can modify the cluster event instances, so
need to pass around a non-const version of it.
* broker/Manager: Re-use broker serializer for conversion (Arne Welzel, Corelight)
* EventMgr: Add Dispatch() with handler and args (Arne Welzel, Corelight)
Allow users to call event_mgr.Dispatch(handler, args) instead of
constructing the Event instance themselves. Deprecate the old API
and replace users.
There's a subtle change that net_done() may be propagated via
auto_publish() now, but that still needs opt-in from script land
and likely no one did that, or else they'd expected to have it
work anyhow.
* plugin/Manager: Fix MetaHookPre and MetaHookPost using HOOK_CALL_FUNCTION (Arne Welzel, Corelight)
7.2.0-dev.510 | 2025-04-11 15:16:53 +0200
* Prevent event timestamps set to future (Jan Grashoefer, Corelight)

View file

@ -1 +1 @@
7.2.0-dev.510
7.2.0-dev.517

View file

@ -106,6 +106,22 @@ void EventMgr::Dispatch(Event* event, bool no_remote) {
Unref(event);
}
void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) {
auto* ev = new Event(h, std::move(vl));
// Technically this isn't queued, but still give plugins a chance to
// intercept the event and cancel or modify it if really wanted.
bool done = PLUGIN_HOOK_WITH_RESULT(HOOK_QUEUE_EVENT, HookQueueEvent(ev), false);
if ( done )
return;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
// TODO: Open-code the old Dispatch() implementation here in v8.1.
Dispatch(ev);
#pragma GCC diagnostic pop
}
void EventMgr::Drain() {
if ( event_queue_flush_point )
Enqueue(event_queue_flush_point, Args{});

View file

@ -80,8 +80,15 @@ public:
return Enqueue(h, zeek::Args{std::forward<Args>(args)...});
}
[[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]]
void Dispatch(Event* event, bool no_remote = false);
// Dispatch an event with the given handler and arguments immediately.
//
// While the event is technically not queued, HookQueueEvent() is
// invoked on the Event instance regardless.
void Dispatch(const EventHandlerPtr& h, zeek::Args vl);
void Drain();
bool IsDraining() const { return current != nullptr; }

View file

@ -106,12 +106,8 @@ void EventHandler::NewEvent(Args* vl) {
return;
auto vargs = MakeCallArgumentVector(*vl, GetType()->Params());
auto ev = new Event(new_event, {
make_intrusive<StringVal>(name),
std::move(vargs),
});
event_mgr.Dispatch(ev);
auto args = zeek::Args{make_intrusive<StringVal>(name), std::move(vargs)};
event_mgr.Dispatch(new_event, std::move(args));
}
uint64_t EventHandler::CallCount() const { return call_count ? call_count->Value() : 0; }

View file

@ -300,8 +300,7 @@ void File::RaiseOpenEvent() {
return;
FilePtr bf{NewRef{}, this};
auto* event = new Event(::file_opened, {make_intrusive<FileVal>(std::move(bf))});
event_mgr.Dispatch(event, true);
event_mgr.Dispatch(::file_opened, {make_intrusive<FileVal>(std::move(bf))});
}
double File::Size() {

View file

@ -259,12 +259,11 @@ void ProfileLogger::Log() {
}
// Create an event so that scripts can log their information, too.
// (and for consistency we dispatch it *now*)
// (and for consistency we dispatch it *now*). Don't propagate this
// event to remote clients.
if ( profiling_update ) {
event_mgr.Dispatch(new Event(profiling_update, {
make_intrusive<FileVal>(IntrusivePtr{NewRef{}, file}),
val_mgr->Bool(expensive),
}));
zeek::Args args{make_intrusive<FileVal>(IntrusivePtr{NewRef{}, file}), val_mgr->Bool(expensive)};
event_mgr.Dispatch(profiling_update, std::move(args));
}
}

View file

@ -34,6 +34,7 @@
#include "zeek/broker/data.bif.h"
#include "zeek/broker/messaging.bif.h"
#include "zeek/broker/store.bif.h"
#include "zeek/cluster/serializer/broker/Serializer.h"
#include "zeek/iosource/Manager.h"
#include "zeek/logging/Manager.h"
#include "zeek/logging/Types.h"
@ -276,7 +277,7 @@ std::string RenderEvent(const std::string& topic, const std::string& name, const
} // namespace
#endif
Manager::Manager(bool arg_use_real_time) : Backend(nullptr, nullptr, nullptr), iosource::IOSource(true) {
Manager::Manager(bool arg_use_real_time) : Backend("Broker", nullptr, nullptr, nullptr), iosource::IOSource(true) {
bound_port = 0;
use_real_time = arg_use_real_time;
peer_count = 0;
@ -623,29 +624,17 @@ std::vector<broker::peer_info> Manager::Peers() const {
std::string Manager::NodeID() const { return to_string(bstate->endpoint.node_id()); }
bool Manager::DoPublishEvent(const std::string& topic, const cluster::detail::Event& event) {
broker::vector xs;
xs.reserve(event.args.size());
for ( const auto& a : event.args ) {
if ( a->GetType() == zeek::BifType::Record::Broker::Data ) {
// When encountering a Broker::Data instance within args, pick out
// the broker::data directly to avoid double encoding of the record.
const auto& val = a->AsRecordVal()->GetField(0);
auto* data_val = static_cast<zeek::Broker::detail::DataVal*>(val.get());
xs.emplace_back(data_val->data);
}
else if ( auto r = detail::val_to_data(a.get()) ) {
xs.emplace_back(std::move(r.value()));
}
else {
Error("Failed to convert %s to broker::data", zeek::obj_desc(a.get()).c_str());
bool Manager::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) {
auto maybe_ev = zeek::cluster::detail::to_broker_event(event);
if ( ! maybe_ev )
return false;
}
}
std::string name(event.HandlerName());
return PublishEvent(topic, std::move(name), std::move(xs), event.timestamp);
auto& ev = maybe_ev.value();
DBG_LOG(DBG_BROKER, "Publishing event: %s", RenderEvent(topic, std::string(ev.name()), ev.args()).c_str());
bstate->endpoint.publish(topic, ev.move_data());
num_events_outgoing_metric->Inc();
return true;
}
bool Manager::PublishEvent(string topic, std::string name, broker::vector args, double ts) {

View file

@ -396,7 +396,7 @@ private:
void DoTerminate() override;
// Broker overrides this to do its own serialization.
bool DoPublishEvent(const std::string& topic, const cluster::detail::Event& event) override;
bool DoPublishEvent(const std::string& topic, cluster::detail::Event& event) override;
// This should never be reached, broker itself doesn't call this and overrides
// the generic DoPublishEvent() method that would call this.

View file

@ -10,6 +10,7 @@
#include "zeek/Func.h"
#include "zeek/Reporter.h"
#include "zeek/Type.h"
#include "zeek/cluster/Manager.h"
#include "zeek/cluster/OnLoop.h"
#include "zeek/cluster/Serializer.h"
#include "zeek/logging/Manager.h"
@ -19,7 +20,7 @@ using namespace zeek::cluster;
bool detail::LocalEventHandlingStrategy::DoHandleRemoteEvent(std::string_view topic, detail::Event e) {
zeek::event_mgr.Enqueue(e.Handler(), std::move(e.args), util::detail::SOURCE_BROKER, 0, nullptr, e.timestamp);
zeek::event_mgr.Enqueue(e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER, 0, nullptr, e.Timestamp());
return true;
}
@ -69,9 +70,16 @@ std::optional<zeek::Args> detail::check_args(const zeek::FuncValPtr& handler, ze
return result;
}
Backend::Backend(std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
Backend::Backend(std::string_view arg_name, std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
std::unique_ptr<detail::EventHandlingStrategy> ehs)
: event_serializer(std::move(es)), log_serializer(std::move(ls)), event_handling_strategy(std::move(ehs)) {}
: name(arg_name),
event_serializer(std::move(es)),
log_serializer(std::move(ls)),
event_handling_strategy(std::move(ehs)) {
tag = zeek::cluster::manager->Backends().GetComponentTag(name);
if ( ! tag )
reporter->InternalError("unknown cluster backend name '%s'; mismatch with tag component?", name.c_str());
}
std::optional<detail::Event> Backend::MakeClusterEvent(FuncValPtr handler, ArgsSpan args, double timestamp) const {
auto checked_args = detail::check_args(handler, args);
@ -91,7 +99,7 @@ std::optional<detail::Event> Backend::MakeClusterEvent(FuncValPtr handler, ArgsS
}
// Default implementation doing the serialization.
bool Backend::DoPublishEvent(const std::string& topic, const cluster::detail::Event& event) {
bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) {
cluster::detail::byte_buffer buf;
if ( ! event_serializer->SerializeEvent(buf, event) )
@ -158,10 +166,10 @@ bool ThreadedBackend::ProcessBackendMessage(int tag, detail::byte_buffer_span pa
return DoProcessBackendMessage(tag, payload);
}
ThreadedBackend::ThreadedBackend(std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
std::unique_ptr<detail::EventHandlingStrategy> ehs)
: Backend(std::move(es), std::move(ls), std::move(ehs)) {
onloop = new zeek::detail::OnLoopProcess<ThreadedBackend, QueueMessage>(this, "ThreadedBackend");
ThreadedBackend::ThreadedBackend(std::string_view name, std::unique_ptr<EventSerializer> es,
std::unique_ptr<LogSerializer> ls, std::unique_ptr<detail::EventHandlingStrategy> ehs)
: Backend(name, std::move(es), std::move(ls), std::move(ehs)) {
onloop = new zeek::detail::OnLoopProcess<ThreadedBackend, QueueMessage>(this, Name());
onloop->Register(true); // Register as don't count first
}

View file

@ -12,6 +12,7 @@
#include "zeek/EventHandler.h"
#include "zeek/IntrusivePtr.h"
#include "zeek/Span.h"
#include "zeek/Tag.h"
#include "zeek/cluster/Serializer.h"
#include "zeek/logging/Types.h"
@ -45,14 +46,36 @@ public:
Event(const EventHandlerPtr& handler, zeek::Args args, double timestamp = 0.0)
: handler(handler), args(std::move(args)), timestamp(timestamp) {}
/**
* @return The name of the event.
*/
std::string_view HandlerName() const { return handler->Name(); }
/**
* @return The event's handler.
*/
const EventHandlerPtr& Handler() const { return handler; }
/**
* @return The event's arguments.
*/
const zeek::Args& Args() const { return args; }
/**
* @return The event's arguments.
*/
zeek::Args& Args() { return args; }
/**
* @return The network timestamp metadata of this event or 0.0.
*/
double Timestamp() const { return timestamp; }
private:
EventHandlerPtr handler;
zeek::Args args;
double timestamp; // TODO: This should be more generic, possibly holding a
// vector of key/value metadata, rather than just
// the timestamp.
std::string_view HandlerName() const { return handler->Name(); }
const EventHandlerPtr& Handler() const { return handler; }
};
/**
@ -178,14 +201,17 @@ public:
/**
* Publish a cluster::detail::Event instance to a given topic.
*
* The event is allowed to be modified by plugins, e.g. to add additional
* metadata, modify the arguments, or rewrite it in other ways, too. The
* caller will observe these changes on the event as it is passed by
* reference.
*
* @param topic The topic string to publish the event to.
* @param event The event to publish.
*
* @return true if the event was successfully published.
*/
bool PublishEvent(const std::string& topic, const cluster::detail::Event& event) {
return DoPublishEvent(topic, event);
}
bool PublishEvent(const std::string& topic, cluster::detail::Event& event) { return DoPublishEvent(topic, event); }
/**
* Status codes for callbacks.
@ -244,6 +270,16 @@ public:
return DoPublishLogWrites(header, records);
}
/**
* @return This backend's implementation name.
*/
const std::string& Name() const { return name; }
/**
* @return This backend's implementation component tag.
*/
const zeek::Tag& Tag() const { return tag; }
/**
* @return This backend's node identifier.
*/
@ -253,11 +289,12 @@ protected:
/**
* Constructor.
*
* @param name The name corresponding to the component tag.
* @param es The event serializer to use.
* @param ls The log batch serializer to use.
* @param ehs The event handling strategy to use for this backend.
*/
Backend(std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
Backend(std::string_view name, std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
std::unique_ptr<detail::EventHandlingStrategy> ehs);
/**
@ -319,7 +356,7 @@ private:
* This hook method only exists for the existing Broker implementation that
* short-circuits serialization. Other backends should not override this.
*/
virtual bool DoPublishEvent(const std::string& topic, const cluster::detail::Event& event);
virtual bool DoPublishEvent(const std::string& topic, cluster::detail::Event& event);
/**
* Send a serialized cluster::detail::Event to the given topic.
@ -405,6 +442,8 @@ private:
virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format,
detail::byte_buffer& buf) = 0;
std::string name;
zeek::Tag tag;
std::unique_ptr<EventSerializer> event_serializer;
std::unique_ptr<LogSerializer> log_serializer;
std::unique_ptr<detail::EventHandlingStrategy> event_handling_strategy;
@ -471,7 +510,7 @@ protected:
/**
* Constructor.
*/
ThreadedBackend(std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
ThreadedBackend(std::string_view name, std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
std::unique_ptr<detail::EventHandlingStrategy> ehs);
/**

View file

@ -64,7 +64,7 @@ constexpr DebugFlag operator&(zeek_uint_t x, DebugFlag y) {
ZeroMQBackend::ZeroMQBackend(std::unique_ptr<EventSerializer> es, std::unique_ptr<LogSerializer> ls,
std::unique_ptr<detail::EventHandlingStrategy> ehs)
: ThreadedBackend(std::move(es), std::move(ls), std::move(ehs)) {
: ThreadedBackend("ZeroMQ", std::move(es), std::move(ls), std::move(ehs)) {
log_push = zmq::socket_t(ctx, zmq::socket_type::push);
main_inproc = zmq::socket_t(ctx, zmq::socket_type::pair);
}

View file

@ -22,9 +22,9 @@ using namespace zeek::cluster;
std::optional<broker::zeek::Event> detail::to_broker_event(const detail::Event& ev) {
broker::vector xs;
xs.reserve(ev.args.size());
xs.reserve(ev.Args().size());
for ( const auto& a : ev.args ) {
for ( const auto& a : ev.Args() ) {
if ( a->GetType() == zeek::BifType::Record::Broker::Data ) {
// When encountering a Broker::Data instance within args, pick out
// the broker::data directly to avoid double encoding, Broker::Data.
@ -40,7 +40,7 @@ std::optional<broker::zeek::Event> detail::to_broker_event(const detail::Event&
}
}
return broker::zeek::Event(ev.HandlerName(), xs, broker::to_timestamp(ev.timestamp));
return broker::zeek::Event(ev.HandlerName(), xs, broker::to_timestamp(ev.Timestamp()));
}
std::optional<detail::Event> detail::to_zeek_event(const broker::zeek::Event& ev) {
@ -188,7 +188,7 @@ TEST_CASE("roundtrip") {
REQUIRE(result);
CHECK_EQ(result->Handler(), handler);
CHECK_EQ(result->HandlerName(), "Supervisor::node_status");
CHECK_EQ(result->args.size(), 2);
CHECK_EQ(result->Args().size(), 2);
}
SUBCASE("binary") {
@ -211,7 +211,7 @@ TEST_CASE("roundtrip") {
REQUIRE(result);
CHECK_EQ(result->Handler(), handler);
CHECK_EQ(result->HandlerName(), "Supervisor::node_status");
CHECK_EQ(result->args.size(), 2);
CHECK_EQ(result->Args().size(), 2);
}
}
TEST_SUITE_END();

View file

@ -454,7 +454,7 @@ void WebSocketEventDispatcher::HandleEvent(WebSocketClientEntry& entry, std::str
//
// Switching to a JSON v2 format that ensures all Zeek types are represented
// explicitly would help.
const auto& zeek_ev = cluster::detail::to_zeek_event(broker_ev);
auto zeek_ev = cluster::detail::to_zeek_event(broker_ev);
if ( ! zeek_ev ) {
entry.wsc->SendError(broker::enum_str(broker::ec::deserialization_failed), "failed to create Zeek event");
return;

View file

@ -962,13 +962,13 @@ void Manager::HookUnprocessedPacket(const Packet* packet) const {
}
void Manager::MetaHookPre(HookType hook, const HookArgumentList& args) const {
if ( hook_list* l = hooks[HOOK_CALL_FUNCTION] )
if ( hook_list* l = hooks[META_HOOK_PRE] )
for ( const auto& [hook_type, plugin] : *l )
plugin->MetaHookPre(hook, args);
}
void Manager::MetaHookPost(HookType hook, const HookArgumentList& args, const HookArgument& result) const {
if ( hook_list* l = hooks[HOOK_CALL_FUNCTION] )
if ( hook_list* l = hooks[META_HOOK_POST] )
for ( const auto& [hook_type, plugin] : *l )
plugin->MetaHookPost(hook, args, result);
}

View file

@ -307,7 +307,7 @@ static void done_with_network() {
if ( net_done ) {
event_mgr.Drain();
// Don't propagate this event to remote clients.
event_mgr.Dispatch(new Event(net_done, {make_intrusive<TimeVal>(timer_mgr->Time())}), true);
event_mgr.Dispatch(net_done, {make_intrusive<TimeVal>(timer_mgr->Time())});
}
if ( profiling_logger )
@ -683,8 +683,6 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
input_mgr = new input::Manager();
file_mgr = new file_analysis::Manager();
cluster::manager = new cluster::Manager();
auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode;
broker_mgr = new Broker::Manager(broker_real_time);
trigger_mgr = new trigger::Manager();
#ifdef HAVE_SPICY
spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager
@ -695,6 +693,11 @@ SetupResult setup(int argc, char** argv, Options* zopts) {
file_mgr->InitPreScript();
zeekygen_mgr->InitPreScript();
// Needs the "broker plugin" loaded during plugin_mgr->InitPreScript()
// before Broker::Manager can be instantiated.
auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode;
broker_mgr = new Broker::Manager(broker_real_time);
// This has to happen before ActivateDynamicPlugin() below or the list of plugins in the
// manager will be missing the plugins we want to try to add to the path.
plugin_mgr->ExtendZeekPathForPlugins();

View file

@ -3352,6 +3352,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost LogWrite(Log::WRITER_ASCII, default, conn(XXXX
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(Broker::log_flush()) -> false
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]])) -> false
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T)) -> false
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(net_done(XXXXXXXXXX.XXXXXX)) -> false
XXXXXXXXXX.XXXXXX MetaHookPost UpdateNetworkTime(XXXXXXXXXX.XXXXXX) -> <void>
XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::__flush_logs, <frame>, ())
XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::flush_logs, <frame>, ())
@ -3382,6 +3383,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre LogWrite(Log::WRITER_ASCII, default, conn(XXXX
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(Broker::log_flush())
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]]))
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T))
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(net_done(XXXXXXXXXX.XXXXXX))
XXXXXXXXXX.XXXXXX MetaHookPre UpdateNetworkTime(XXXXXXXXXX.XXXXXX)
XXXXXXXXXX.XXXXXX | HookUpdateNetworkTime XXXXXXXXXX.XXXXXX
XXXXXXXXXX.XXXXXX | HookCallFunction Broker::__flush_logs()
@ -3413,3 +3415,4 @@ XXXXXXXXXX.XXXXXX | HookLogWrite conn [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjw
XXXXXXXXXX.XXXXXX | HookQueueEvent Broker::log_flush()
XXXXXXXXXX.XXXXXX | HookQueueEvent connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]])
XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T)
XXXXXXXXXX.XXXXXX | HookQueueEvent net_done(XXXXXXXXXX.XXXXXX)

View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 MetaHookPre QueueEvent(zeek_init())
0.000000 HookQueueEvent zeek_init()
0.000000 MetaHookPost QueueEvent(zeek_init()) -> false
zeek_init()
0.000000 MetaHookPre QueueEvent(net_done(1.0))
0.000000 HookQueueEvent net_done()
0.000000 MetaHookPost QueueEvent(net_done(1.0)) -> false
0.000000 MetaHookPre QueueEvent(Broker::log_flush())
0.000000 HookQueueEvent Broker::log_flush()
0.000000 MetaHookPost QueueEvent(Broker::log_flush()) -> false

View file

@ -0,0 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 HookQueueEvent zeek_init()
zeek_init()
0.000000 HookQueueEvent net_done()
0.000000 HookQueueEvent Broker::log_flush()

View file

@ -0,0 +1,8 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 HookQueueEvent zeek_init()
0.000000 MetaHookPost QueueEvent() -> false
zeek_init()
0.000000 HookQueueEvent net_done()
0.000000 MetaHookPost QueueEvent() -> false
0.000000 HookQueueEvent Broker::log_flush()
0.000000 MetaHookPost QueueEvent() -> false

View file

@ -0,0 +1,8 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 MetaHookPre QueueEvent(zeek_init())
0.000000 HookQueueEvent zeek_init()
zeek_init()
0.000000 MetaHookPre QueueEvent(net_done(1.0))
0.000000 HookQueueEvent net_done()
0.000000 MetaHookPre QueueEvent(Broker::log_flush())
0.000000 HookQueueEvent Broker::log_flush()

View file

@ -0,0 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 HookQueueEvent zeek_init()
zeek_init()
0.000000 HookQueueEvent net_done()
0.000000 HookQueueEvent Broker::log_flush()

View file

@ -0,0 +1,83 @@
#include "Plugin.h"
#include <zeek/Desc.h>
#include <zeek/Event.h>
#include <zeek/Func.h>
#include <zeek/threading/Formatter.h>
#include <cstdlib>
namespace btest::plugin::Demo_Meta_Hooks {
Plugin plugin;
}
using namespace btest::plugin::Demo_Meta_Hooks;
zeek::plugin::Configuration Plugin::Configure() {
zeek::plugin::Configuration config;
config.name = "Demo::Meta_Hooks";
config.description = "Test if the meta hooks are working";
config.version.major = 1;
config.version.minor = 0;
config.version.patch = 0;
// This plugin enables HookQueueEvent() and optionally the pre and post
// meta hooks controlled by environment variables for easier testing.
EnableHook(zeek::plugin::HOOK_QUEUE_EVENT);
if ( getenv("TEST_META_HOOK_PRE") )
EnableHook(zeek::plugin::META_HOOK_PRE);
if ( getenv("TEST_META_HOOK_POST") )
EnableHook(zeek::plugin::META_HOOK_POST);
return config;
}
static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek::ODesc* d) {
bool first = true;
for ( const auto& arg : args ) {
if ( ! first )
d->Add(", ");
arg.Describe(d);
first = false;
}
}
bool Plugin::HookQueueEvent(zeek::Event* e) {
fprintf(stdout, "%.6f %-15s %s()\n", zeek::run_state::network_time, " HookQueueEvent", e->Handler()->Name());
return false;
}
void Plugin::MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) {
// The spicy integration enables HOOK_LOAD_FILE and this plugin receives
// meta hooks also for that :-/
if ( hook != zeek::plugin::HOOK_QUEUE_EVENT )
return;
zeek::ODesc d;
d.SetShort();
describe_hook_args(args, &d);
fprintf(stdout, "%.6f %-15s %s(%s)\n", zeek::run_state::network_time, " MetaHookPre", hook_name(hook),
d.Description());
}
void Plugin::MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args,
zeek::plugin::HookArgument result) {
// The spicy integration enables HOOK_LOAD_FILE and this plugin receives
// meta hooks also for that :-/
if ( hook != zeek::plugin::HOOK_QUEUE_EVENT )
return;
zeek::ODesc d1;
zeek::ODesc d2;
describe_hook_args(args, &d1);
result.Describe(&d2);
fprintf(stdout, "%.6f %-15s %s(%s) -> %s\n", zeek::run_state::network_time, " MetaHookPost", hook_name(hook),
d1.Description(), d2.Description());
}

View file

@ -0,0 +1,21 @@
#pragma once
#include <zeek/plugin/Plugin.h>
namespace btest::plugin::Demo_Meta_Hooks {
class Plugin : public zeek::plugin::Plugin {
protected:
bool HookQueueEvent(zeek::Event* e) override;
void MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) override;
void MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args,
zeek::plugin::HookArgument result) override;
// Overridden from plugin::Plugin.
zeek::plugin::Configuration Configure() override;
};
extern Plugin plugin;
} // namespace btest::plugin::Demo_Meta_Hooks

View file

@ -0,0 +1,23 @@
# @TEST-DOC: Plugin testing the meta hooks specifically. This is a regression test for these being enabled with HookCallFunction() instead.
#
# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Meta_Hooks
# @TEST-EXEC: cp -r %DIR/meta-hook-plugin/* .
# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
# @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-none
# @TEST-EXEC: TEST_META_HOOK_PRE=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-pre-only
# @TEST-EXEC: TEST_META_HOOK_POST=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-post-only
# @TEST-EXEC: TEST_META_HOOK_PRE=1 TEST_META_HOOK_POST=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-both
#
# @TEST-EXEC: btest-diff out-none
# @TEST-EXEC: btest-diff out-pre-only
# @TEST-EXEC: btest-diff out-post-only
# @TEST-EXEC: btest-diff out-both
@load-plugin Demo::Meta_Hooks
redef allow_network_time_forward = F;
event zeek_init()
{
print "zeek_init()";
}