diff --git a/CHANGES b/CHANGES index c7a634bfad..b9ded8e2cc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,32 @@ +7.2.0-dev.517 | 2025-04-13 17:11:40 +0200 + + * cluster/Backend: Add name and lookup component tag (Arne Welzel, Corelight) + + This adds two new accessors on Backend, Name() and Tag() that can + be used for introspection of a Backend instance. + + * cluster/Event: Hide members behind accessors (Arne Welzel, Corelight) + + * cluster/PublishEvent:: Make event non-const (Arne Welzel, Corelight) + + We want to introduce a hook that can modify the cluster event instances, so + need to pass around a non-const version of it. + + * broker/Manager: Re-use broker serializer for conversion (Arne Welzel, Corelight) + + * EventMgr: Add Dispatch() with handler and args (Arne Welzel, Corelight) + + Allow users to call event_mgr.Dispatch(handler, args) instead of + constructing the Event instance themselves. Deprecate the old API + and replace users. + + There's a subtle change that net_done() may be propagated via + auto_publish() now, but that still needs opt-in from script land + and likely no one did that, or else they'd expected to have it + work anyhow. + + * plugin/Manager: Fix MetaHookPre and MetaHookPost using HOOK_CALL_FUNCTION (Arne Welzel, Corelight) + 7.2.0-dev.510 | 2025-04-11 15:16:53 +0200 * Prevent event timestamps set to future (Jan Grashoefer, Corelight) diff --git a/VERSION b/VERSION index ecc9ca0edc..78d59ffdf9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.2.0-dev.510 +7.2.0-dev.517 diff --git a/src/Event.cc b/src/Event.cc index 6d449684e5..d113e475cf 100644 --- a/src/Event.cc +++ b/src/Event.cc @@ -106,6 +106,22 @@ void EventMgr::Dispatch(Event* event, bool no_remote) { Unref(event); } +void EventMgr::Dispatch(const EventHandlerPtr& h, zeek::Args vl) { + auto* ev = new Event(h, std::move(vl)); + + // Technically this isn't queued, but still give plugins a chance to + // intercept the event and cancel or modify it if really wanted. + bool done = PLUGIN_HOOK_WITH_RESULT(HOOK_QUEUE_EVENT, HookQueueEvent(ev), false); + if ( done ) + return; + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" + // TODO: Open-code the old Dispatch() implementation here in v8.1. + Dispatch(ev); +#pragma GCC diagnostic pop +} + void EventMgr::Drain() { if ( event_queue_flush_point ) Enqueue(event_queue_flush_point, Args{}); diff --git a/src/Event.h b/src/Event.h index 078595a7fe..5639fea96f 100644 --- a/src/Event.h +++ b/src/Event.h @@ -80,8 +80,15 @@ public: return Enqueue(h, zeek::Args{std::forward(args)...}); } + [[deprecated("Remove in v8.1: Use Dispatch(handler, args) instead.")]] void Dispatch(Event* event, bool no_remote = false); + // Dispatch an event with the given handler and arguments immediately. + // + // While the event is technically not queued, HookQueueEvent() is + // invoked on the Event instance regardless. + void Dispatch(const EventHandlerPtr& h, zeek::Args vl); + void Drain(); bool IsDraining() const { return current != nullptr; } diff --git a/src/EventHandler.cc b/src/EventHandler.cc index b6f11a0b53..2d60aed663 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -106,12 +106,8 @@ void EventHandler::NewEvent(Args* vl) { return; auto vargs = MakeCallArgumentVector(*vl, GetType()->Params()); - - auto ev = new Event(new_event, { - make_intrusive(name), - std::move(vargs), - }); - event_mgr.Dispatch(ev); + auto args = zeek::Args{make_intrusive(name), std::move(vargs)}; + event_mgr.Dispatch(new_event, std::move(args)); } uint64_t EventHandler::CallCount() const { return call_count ? call_count->Value() : 0; } diff --git a/src/File.cc b/src/File.cc index cc77df035d..28826bb516 100644 --- a/src/File.cc +++ b/src/File.cc @@ -300,8 +300,7 @@ void File::RaiseOpenEvent() { return; FilePtr bf{NewRef{}, this}; - auto* event = new Event(::file_opened, {make_intrusive(std::move(bf))}); - event_mgr.Dispatch(event, true); + event_mgr.Dispatch(::file_opened, {make_intrusive(std::move(bf))}); } double File::Size() { diff --git a/src/Stats.cc b/src/Stats.cc index c4e14d9561..83b258124c 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -259,12 +259,11 @@ void ProfileLogger::Log() { } // Create an event so that scripts can log their information, too. - // (and for consistency we dispatch it *now*) + // (and for consistency we dispatch it *now*). Don't propagate this + // event to remote clients. if ( profiling_update ) { - event_mgr.Dispatch(new Event(profiling_update, { - make_intrusive(IntrusivePtr{NewRef{}, file}), - val_mgr->Bool(expensive), - })); + zeek::Args args{make_intrusive(IntrusivePtr{NewRef{}, file}), val_mgr->Bool(expensive)}; + event_mgr.Dispatch(profiling_update, std::move(args)); } } diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index e036f82c27..3ea29b8b58 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -34,6 +34,7 @@ #include "zeek/broker/data.bif.h" #include "zeek/broker/messaging.bif.h" #include "zeek/broker/store.bif.h" +#include "zeek/cluster/serializer/broker/Serializer.h" #include "zeek/iosource/Manager.h" #include "zeek/logging/Manager.h" #include "zeek/logging/Types.h" @@ -276,7 +277,7 @@ std::string RenderEvent(const std::string& topic, const std::string& name, const } // namespace #endif -Manager::Manager(bool arg_use_real_time) : Backend(nullptr, nullptr, nullptr), iosource::IOSource(true) { +Manager::Manager(bool arg_use_real_time) : Backend("Broker", nullptr, nullptr, nullptr), iosource::IOSource(true) { bound_port = 0; use_real_time = arg_use_real_time; peer_count = 0; @@ -623,29 +624,17 @@ std::vector Manager::Peers() const { std::string Manager::NodeID() const { return to_string(bstate->endpoint.node_id()); } -bool Manager::DoPublishEvent(const std::string& topic, const cluster::detail::Event& event) { - broker::vector xs; - xs.reserve(event.args.size()); +bool Manager::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) { + auto maybe_ev = zeek::cluster::detail::to_broker_event(event); + if ( ! maybe_ev ) + return false; - for ( const auto& a : event.args ) { - if ( a->GetType() == zeek::BifType::Record::Broker::Data ) { - // When encountering a Broker::Data instance within args, pick out - // the broker::data directly to avoid double encoding of the record. - const auto& val = a->AsRecordVal()->GetField(0); - auto* data_val = static_cast(val.get()); - xs.emplace_back(data_val->data); - } - else if ( auto r = detail::val_to_data(a.get()) ) { - xs.emplace_back(std::move(r.value())); - } - else { - Error("Failed to convert %s to broker::data", zeek::obj_desc(a.get()).c_str()); - return false; - } - } + auto& ev = maybe_ev.value(); - std::string name(event.HandlerName()); - return PublishEvent(topic, std::move(name), std::move(xs), event.timestamp); + DBG_LOG(DBG_BROKER, "Publishing event: %s", RenderEvent(topic, std::string(ev.name()), ev.args()).c_str()); + bstate->endpoint.publish(topic, ev.move_data()); + num_events_outgoing_metric->Inc(); + return true; } bool Manager::PublishEvent(string topic, std::string name, broker::vector args, double ts) { diff --git a/src/broker/Manager.h b/src/broker/Manager.h index e72e5d0df5..c9e7f3c8e9 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -396,7 +396,7 @@ private: void DoTerminate() override; // Broker overrides this to do its own serialization. - bool DoPublishEvent(const std::string& topic, const cluster::detail::Event& event) override; + bool DoPublishEvent(const std::string& topic, cluster::detail::Event& event) override; // This should never be reached, broker itself doesn't call this and overrides // the generic DoPublishEvent() method that would call this. diff --git a/src/cluster/Backend.cc b/src/cluster/Backend.cc index 92d9dd4624..8fba80b48b 100644 --- a/src/cluster/Backend.cc +++ b/src/cluster/Backend.cc @@ -10,6 +10,7 @@ #include "zeek/Func.h" #include "zeek/Reporter.h" #include "zeek/Type.h" +#include "zeek/cluster/Manager.h" #include "zeek/cluster/OnLoop.h" #include "zeek/cluster/Serializer.h" #include "zeek/logging/Manager.h" @@ -19,7 +20,7 @@ using namespace zeek::cluster; bool detail::LocalEventHandlingStrategy::DoHandleRemoteEvent(std::string_view topic, detail::Event e) { - zeek::event_mgr.Enqueue(e.Handler(), std::move(e.args), util::detail::SOURCE_BROKER, 0, nullptr, e.timestamp); + zeek::event_mgr.Enqueue(e.Handler(), std::move(e.Args()), util::detail::SOURCE_BROKER, 0, nullptr, e.Timestamp()); return true; } @@ -69,9 +70,16 @@ std::optional detail::check_args(const zeek::FuncValPtr& handler, ze return result; } -Backend::Backend(std::unique_ptr es, std::unique_ptr ls, +Backend::Backend(std::string_view arg_name, std::unique_ptr es, std::unique_ptr ls, std::unique_ptr ehs) - : event_serializer(std::move(es)), log_serializer(std::move(ls)), event_handling_strategy(std::move(ehs)) {} + : name(arg_name), + event_serializer(std::move(es)), + log_serializer(std::move(ls)), + event_handling_strategy(std::move(ehs)) { + tag = zeek::cluster::manager->Backends().GetComponentTag(name); + if ( ! tag ) + reporter->InternalError("unknown cluster backend name '%s'; mismatch with tag component?", name.c_str()); +} std::optional Backend::MakeClusterEvent(FuncValPtr handler, ArgsSpan args, double timestamp) const { auto checked_args = detail::check_args(handler, args); @@ -91,7 +99,7 @@ std::optional Backend::MakeClusterEvent(FuncValPtr handler, ArgsS } // Default implementation doing the serialization. -bool Backend::DoPublishEvent(const std::string& topic, const cluster::detail::Event& event) { +bool Backend::DoPublishEvent(const std::string& topic, cluster::detail::Event& event) { cluster::detail::byte_buffer buf; if ( ! event_serializer->SerializeEvent(buf, event) ) @@ -158,10 +166,10 @@ bool ThreadedBackend::ProcessBackendMessage(int tag, detail::byte_buffer_span pa return DoProcessBackendMessage(tag, payload); } -ThreadedBackend::ThreadedBackend(std::unique_ptr es, std::unique_ptr ls, - std::unique_ptr ehs) - : Backend(std::move(es), std::move(ls), std::move(ehs)) { - onloop = new zeek::detail::OnLoopProcess(this, "ThreadedBackend"); +ThreadedBackend::ThreadedBackend(std::string_view name, std::unique_ptr es, + std::unique_ptr ls, std::unique_ptr ehs) + : Backend(name, std::move(es), std::move(ls), std::move(ehs)) { + onloop = new zeek::detail::OnLoopProcess(this, Name()); onloop->Register(true); // Register as don't count first } diff --git a/src/cluster/Backend.h b/src/cluster/Backend.h index 794b270eff..5f0495ac18 100644 --- a/src/cluster/Backend.h +++ b/src/cluster/Backend.h @@ -12,6 +12,7 @@ #include "zeek/EventHandler.h" #include "zeek/IntrusivePtr.h" #include "zeek/Span.h" +#include "zeek/Tag.h" #include "zeek/cluster/Serializer.h" #include "zeek/logging/Types.h" @@ -45,14 +46,36 @@ public: Event(const EventHandlerPtr& handler, zeek::Args args, double timestamp = 0.0) : handler(handler), args(std::move(args)), timestamp(timestamp) {} + /** + * @return The name of the event. + */ + std::string_view HandlerName() const { return handler->Name(); } + + /** + * @return The event's handler. + */ + const EventHandlerPtr& Handler() const { return handler; } + + /** + * @return The event's arguments. + */ + const zeek::Args& Args() const { return args; } + /** + * @return The event's arguments. + */ + zeek::Args& Args() { return args; } + + /** + * @return The network timestamp metadata of this event or 0.0. + */ + double Timestamp() const { return timestamp; } + +private: EventHandlerPtr handler; zeek::Args args; double timestamp; // TODO: This should be more generic, possibly holding a // vector of key/value metadata, rather than just // the timestamp. - - std::string_view HandlerName() const { return handler->Name(); } - const EventHandlerPtr& Handler() const { return handler; } }; /** @@ -178,14 +201,17 @@ public: /** * Publish a cluster::detail::Event instance to a given topic. * + * The event is allowed to be modified by plugins, e.g. to add additional + * metadata, modify the arguments, or rewrite it in other ways, too. The + * caller will observe these changes on the event as it is passed by + * reference. + * * @param topic The topic string to publish the event to. * @param event The event to publish. * * @return true if the event was successfully published. */ - bool PublishEvent(const std::string& topic, const cluster::detail::Event& event) { - return DoPublishEvent(topic, event); - } + bool PublishEvent(const std::string& topic, cluster::detail::Event& event) { return DoPublishEvent(topic, event); } /** * Status codes for callbacks. @@ -244,6 +270,16 @@ public: return DoPublishLogWrites(header, records); } + /** + * @return This backend's implementation name. + */ + const std::string& Name() const { return name; } + + /** + * @return This backend's implementation component tag. + */ + const zeek::Tag& Tag() const { return tag; } + /** * @return This backend's node identifier. */ @@ -253,11 +289,12 @@ protected: /** * Constructor. * + * @param name The name corresponding to the component tag. * @param es The event serializer to use. * @param ls The log batch serializer to use. * @param ehs The event handling strategy to use for this backend. */ - Backend(std::unique_ptr es, std::unique_ptr ls, + Backend(std::string_view name, std::unique_ptr es, std::unique_ptr ls, std::unique_ptr ehs); /** @@ -319,7 +356,7 @@ private: * This hook method only exists for the existing Broker implementation that * short-circuits serialization. Other backends should not override this. */ - virtual bool DoPublishEvent(const std::string& topic, const cluster::detail::Event& event); + virtual bool DoPublishEvent(const std::string& topic, cluster::detail::Event& event); /** * Send a serialized cluster::detail::Event to the given topic. @@ -405,6 +442,8 @@ private: virtual bool DoPublishLogWrites(const zeek::logging::detail::LogWriteHeader& header, const std::string& format, detail::byte_buffer& buf) = 0; + std::string name; + zeek::Tag tag; std::unique_ptr event_serializer; std::unique_ptr log_serializer; std::unique_ptr event_handling_strategy; @@ -471,7 +510,7 @@ protected: /** * Constructor. */ - ThreadedBackend(std::unique_ptr es, std::unique_ptr ls, + ThreadedBackend(std::string_view name, std::unique_ptr es, std::unique_ptr ls, std::unique_ptr ehs); /** diff --git a/src/cluster/backend/zeromq/ZeroMQ.cc b/src/cluster/backend/zeromq/ZeroMQ.cc index f05ab88d48..43a48aec89 100644 --- a/src/cluster/backend/zeromq/ZeroMQ.cc +++ b/src/cluster/backend/zeromq/ZeroMQ.cc @@ -64,7 +64,7 @@ constexpr DebugFlag operator&(zeek_uint_t x, DebugFlag y) { ZeroMQBackend::ZeroMQBackend(std::unique_ptr es, std::unique_ptr ls, std::unique_ptr ehs) - : ThreadedBackend(std::move(es), std::move(ls), std::move(ehs)) { + : ThreadedBackend("ZeroMQ", std::move(es), std::move(ls), std::move(ehs)) { log_push = zmq::socket_t(ctx, zmq::socket_type::push); main_inproc = zmq::socket_t(ctx, zmq::socket_type::pair); } diff --git a/src/cluster/serializer/broker/Serializer.cc b/src/cluster/serializer/broker/Serializer.cc index f2308cb627..5bd32f8738 100644 --- a/src/cluster/serializer/broker/Serializer.cc +++ b/src/cluster/serializer/broker/Serializer.cc @@ -22,9 +22,9 @@ using namespace zeek::cluster; std::optional detail::to_broker_event(const detail::Event& ev) { broker::vector xs; - xs.reserve(ev.args.size()); + xs.reserve(ev.Args().size()); - for ( const auto& a : ev.args ) { + for ( const auto& a : ev.Args() ) { if ( a->GetType() == zeek::BifType::Record::Broker::Data ) { // When encountering a Broker::Data instance within args, pick out // the broker::data directly to avoid double encoding, Broker::Data. @@ -40,7 +40,7 @@ std::optional detail::to_broker_event(const detail::Event& } } - return broker::zeek::Event(ev.HandlerName(), xs, broker::to_timestamp(ev.timestamp)); + return broker::zeek::Event(ev.HandlerName(), xs, broker::to_timestamp(ev.Timestamp())); } std::optional detail::to_zeek_event(const broker::zeek::Event& ev) { @@ -188,7 +188,7 @@ TEST_CASE("roundtrip") { REQUIRE(result); CHECK_EQ(result->Handler(), handler); CHECK_EQ(result->HandlerName(), "Supervisor::node_status"); - CHECK_EQ(result->args.size(), 2); + CHECK_EQ(result->Args().size(), 2); } SUBCASE("binary") { @@ -211,7 +211,7 @@ TEST_CASE("roundtrip") { REQUIRE(result); CHECK_EQ(result->Handler(), handler); CHECK_EQ(result->HandlerName(), "Supervisor::node_status"); - CHECK_EQ(result->args.size(), 2); + CHECK_EQ(result->Args().size(), 2); } } TEST_SUITE_END(); diff --git a/src/cluster/websocket/WebSocket.cc b/src/cluster/websocket/WebSocket.cc index fa2efb51ae..4d9f4da30a 100644 --- a/src/cluster/websocket/WebSocket.cc +++ b/src/cluster/websocket/WebSocket.cc @@ -454,7 +454,7 @@ void WebSocketEventDispatcher::HandleEvent(WebSocketClientEntry& entry, std::str // // Switching to a JSON v2 format that ensures all Zeek types are represented // explicitly would help. - const auto& zeek_ev = cluster::detail::to_zeek_event(broker_ev); + auto zeek_ev = cluster::detail::to_zeek_event(broker_ev); if ( ! zeek_ev ) { entry.wsc->SendError(broker::enum_str(broker::ec::deserialization_failed), "failed to create Zeek event"); return; diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index fd15c638fe..88f301fdba 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -962,13 +962,13 @@ void Manager::HookUnprocessedPacket(const Packet* packet) const { } void Manager::MetaHookPre(HookType hook, const HookArgumentList& args) const { - if ( hook_list* l = hooks[HOOK_CALL_FUNCTION] ) + if ( hook_list* l = hooks[META_HOOK_PRE] ) for ( const auto& [hook_type, plugin] : *l ) plugin->MetaHookPre(hook, args); } void Manager::MetaHookPost(HookType hook, const HookArgumentList& args, const HookArgument& result) const { - if ( hook_list* l = hooks[HOOK_CALL_FUNCTION] ) + if ( hook_list* l = hooks[META_HOOK_POST] ) for ( const auto& [hook_type, plugin] : *l ) plugin->MetaHookPost(hook, args, result); } diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index ba5dc89e1f..d08d21ef88 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -307,7 +307,7 @@ static void done_with_network() { if ( net_done ) { event_mgr.Drain(); // Don't propagate this event to remote clients. - event_mgr.Dispatch(new Event(net_done, {make_intrusive(timer_mgr->Time())}), true); + event_mgr.Dispatch(net_done, {make_intrusive(timer_mgr->Time())}); } if ( profiling_logger ) @@ -683,8 +683,6 @@ SetupResult setup(int argc, char** argv, Options* zopts) { input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); cluster::manager = new cluster::Manager(); - auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; - broker_mgr = new Broker::Manager(broker_real_time); trigger_mgr = new trigger::Manager(); #ifdef HAVE_SPICY spicy_mgr = new spicy::Manager(); // registers as plugin with the plugin manager @@ -695,6 +693,11 @@ SetupResult setup(int argc, char** argv, Options* zopts) { file_mgr->InitPreScript(); zeekygen_mgr->InitPreScript(); + // Needs the "broker plugin" loaded during plugin_mgr->InitPreScript() + // before Broker::Manager can be instantiated. + auto broker_real_time = ! options.pcap_file && ! options.deterministic_mode; + broker_mgr = new Broker::Manager(broker_real_time); + // This has to happen before ActivateDynamicPlugin() below or the list of plugins in the // manager will be missing the plugins we want to try to add to the path. plugin_mgr->ExtendZeekPathForPlugins(); diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 85b9a18e43..1e97599ebf 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -3352,6 +3352,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost LogWrite(Log::WRITER_ASCII, default, conn(XXXX XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(Broker::log_flush()) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=, service_violation={}, conn=, extract_orig=F, extract_resp=F, thresholds=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]])) -> false XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=, service_violation={}, conn=, extract_orig=F, extract_resp=F, thresholds=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T)) -> false +XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(net_done(XXXXXXXXXX.XXXXXX)) -> false XXXXXXXXXX.XXXXXX MetaHookPost UpdateNetworkTime(XXXXXXXXXX.XXXXXX) -> XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::__flush_logs, , ()) XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::flush_logs, , ()) @@ -3382,6 +3383,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre LogWrite(Log::WRITER_ASCII, default, conn(XXXX XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(Broker::log_flush()) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=, service_violation={}, conn=, extract_orig=F, extract_resp=F, thresholds=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]])) XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=, service_violation={}, conn=, extract_orig=F, extract_resp=F, thresholds=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T)) +XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(net_done(XXXXXXXXXX.XXXXXX)) XXXXXXXXXX.XXXXXX MetaHookPre UpdateNetworkTime(XXXXXXXXXX.XXXXXX) XXXXXXXXXX.XXXXXX | HookUpdateNetworkTime XXXXXXXXXX.XXXXXX XXXXXXXXXX.XXXXXX | HookCallFunction Broker::__flush_logs() @@ -3413,3 +3415,4 @@ XXXXXXXXXX.XXXXXX | HookLogWrite conn [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjw XXXXXXXXXX.XXXXXX | HookQueueEvent Broker::log_flush() XXXXXXXXXX.XXXXXX | HookQueueEvent connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=, service_violation={}, conn=, extract_orig=F, extract_resp=F, thresholds=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]]) XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=, vlan=, inner_vlan=, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=, service_violation={}, conn=, extract_orig=F, extract_resp=F, thresholds=, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=, info_msg=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_filenames=, orig_mime_types=, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=, resp_mime_types=[text/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T) +XXXXXXXXXX.XXXXXX | HookQueueEvent net_done(XXXXXXXXXX.XXXXXX) diff --git a/testing/btest/Baseline/plugins.meta-hook/out-both b/testing/btest/Baseline/plugins.meta-hook/out-both new file mode 100644 index 0000000000..120a2827f1 --- /dev/null +++ b/testing/btest/Baseline/plugins.meta-hook/out-both @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0.000000 MetaHookPre QueueEvent(zeek_init()) +0.000000 HookQueueEvent zeek_init() +0.000000 MetaHookPost QueueEvent(zeek_init()) -> false +zeek_init() +0.000000 MetaHookPre QueueEvent(net_done(1.0)) +0.000000 HookQueueEvent net_done() +0.000000 MetaHookPost QueueEvent(net_done(1.0)) -> false +0.000000 MetaHookPre QueueEvent(Broker::log_flush()) +0.000000 HookQueueEvent Broker::log_flush() +0.000000 MetaHookPost QueueEvent(Broker::log_flush()) -> false diff --git a/testing/btest/Baseline/plugins.meta-hook/out-none b/testing/btest/Baseline/plugins.meta-hook/out-none new file mode 100644 index 0000000000..33cf95de62 --- /dev/null +++ b/testing/btest/Baseline/plugins.meta-hook/out-none @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0.000000 HookQueueEvent zeek_init() +zeek_init() +0.000000 HookQueueEvent net_done() +0.000000 HookQueueEvent Broker::log_flush() diff --git a/testing/btest/Baseline/plugins.meta-hook/out-post-only b/testing/btest/Baseline/plugins.meta-hook/out-post-only new file mode 100644 index 0000000000..a7950ffca0 --- /dev/null +++ b/testing/btest/Baseline/plugins.meta-hook/out-post-only @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0.000000 HookQueueEvent zeek_init() +0.000000 MetaHookPost QueueEvent() -> false +zeek_init() +0.000000 HookQueueEvent net_done() +0.000000 MetaHookPost QueueEvent() -> false +0.000000 HookQueueEvent Broker::log_flush() +0.000000 MetaHookPost QueueEvent() -> false diff --git a/testing/btest/Baseline/plugins.meta-hook/out-pre-only b/testing/btest/Baseline/plugins.meta-hook/out-pre-only new file mode 100644 index 0000000000..eac1fa039c --- /dev/null +++ b/testing/btest/Baseline/plugins.meta-hook/out-pre-only @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0.000000 MetaHookPre QueueEvent(zeek_init()) +0.000000 HookQueueEvent zeek_init() +zeek_init() +0.000000 MetaHookPre QueueEvent(net_done(1.0)) +0.000000 HookQueueEvent net_done() +0.000000 MetaHookPre QueueEvent(Broker::log_flush()) +0.000000 HookQueueEvent Broker::log_flush() diff --git a/testing/btest/Baseline/plugins.meta-hook/out1 b/testing/btest/Baseline/plugins.meta-hook/out1 new file mode 100644 index 0000000000..33cf95de62 --- /dev/null +++ b/testing/btest/Baseline/plugins.meta-hook/out1 @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0.000000 HookQueueEvent zeek_init() +zeek_init() +0.000000 HookQueueEvent net_done() +0.000000 HookQueueEvent Broker::log_flush() diff --git a/testing/btest/plugins/meta-hook-plugin/.btest-ignore b/testing/btest/plugins/meta-hook-plugin/.btest-ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/plugins/meta-hook-plugin/src/Plugin.cc b/testing/btest/plugins/meta-hook-plugin/src/Plugin.cc new file mode 100644 index 0000000000..fc477a6eeb --- /dev/null +++ b/testing/btest/plugins/meta-hook-plugin/src/Plugin.cc @@ -0,0 +1,83 @@ + +#include "Plugin.h" + +#include +#include +#include +#include +#include + +namespace btest::plugin::Demo_Meta_Hooks { +Plugin plugin; +} + +using namespace btest::plugin::Demo_Meta_Hooks; + +zeek::plugin::Configuration Plugin::Configure() { + zeek::plugin::Configuration config; + config.name = "Demo::Meta_Hooks"; + config.description = "Test if the meta hooks are working"; + config.version.major = 1; + config.version.minor = 0; + config.version.patch = 0; + + // This plugin enables HookQueueEvent() and optionally the pre and post + // meta hooks controlled by environment variables for easier testing. + + EnableHook(zeek::plugin::HOOK_QUEUE_EVENT); + + if ( getenv("TEST_META_HOOK_PRE") ) + EnableHook(zeek::plugin::META_HOOK_PRE); + + if ( getenv("TEST_META_HOOK_POST") ) + EnableHook(zeek::plugin::META_HOOK_POST); + + return config; +} + +static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek::ODesc* d) { + bool first = true; + + for ( const auto& arg : args ) { + if ( ! first ) + d->Add(", "); + + arg.Describe(d); + first = false; + } +} + + +bool Plugin::HookQueueEvent(zeek::Event* e) { + fprintf(stdout, "%.6f %-15s %s()\n", zeek::run_state::network_time, " HookQueueEvent", e->Handler()->Name()); + return false; +} + +void Plugin::MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) { + // The spicy integration enables HOOK_LOAD_FILE and this plugin receives + // meta hooks also for that :-/ + if ( hook != zeek::plugin::HOOK_QUEUE_EVENT ) + return; + + zeek::ODesc d; + d.SetShort(); + describe_hook_args(args, &d); + fprintf(stdout, "%.6f %-15s %s(%s)\n", zeek::run_state::network_time, " MetaHookPre", hook_name(hook), + d.Description()); +} + +void Plugin::MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args, + zeek::plugin::HookArgument result) { + // The spicy integration enables HOOK_LOAD_FILE and this plugin receives + // meta hooks also for that :-/ + if ( hook != zeek::plugin::HOOK_QUEUE_EVENT ) + return; + + zeek::ODesc d1; + zeek::ODesc d2; + describe_hook_args(args, &d1); + result.Describe(&d2); + + fprintf(stdout, "%.6f %-15s %s(%s) -> %s\n", zeek::run_state::network_time, " MetaHookPost", hook_name(hook), + d1.Description(), d2.Description()); +} diff --git a/testing/btest/plugins/meta-hook-plugin/src/Plugin.h b/testing/btest/plugins/meta-hook-plugin/src/Plugin.h new file mode 100644 index 0000000000..5d9cc496b2 --- /dev/null +++ b/testing/btest/plugins/meta-hook-plugin/src/Plugin.h @@ -0,0 +1,21 @@ + +#pragma once + +#include + +namespace btest::plugin::Demo_Meta_Hooks { + +class Plugin : public zeek::plugin::Plugin { +protected: + bool HookQueueEvent(zeek::Event* e) override; + void MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) override; + void MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args, + zeek::plugin::HookArgument result) override; + + // Overridden from plugin::Plugin. + zeek::plugin::Configuration Configure() override; +}; + +extern Plugin plugin; + +} // namespace btest::plugin::Demo_Meta_Hooks diff --git a/testing/btest/plugins/meta-hook.zeek b/testing/btest/plugins/meta-hook.zeek new file mode 100644 index 0000000000..e6fc272462 --- /dev/null +++ b/testing/btest/plugins/meta-hook.zeek @@ -0,0 +1,23 @@ +# @TEST-DOC: Plugin testing the meta hooks specifically. This is a regression test for these being enabled with HookCallFunction() instead. +# +# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Meta_Hooks +# @TEST-EXEC: cp -r %DIR/meta-hook-plugin/* . +# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make +# @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-none +# @TEST-EXEC: TEST_META_HOOK_PRE=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-pre-only +# @TEST-EXEC: TEST_META_HOOK_POST=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-post-only +# @TEST-EXEC: TEST_META_HOOK_PRE=1 TEST_META_HOOK_POST=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-both +# +# @TEST-EXEC: btest-diff out-none +# @TEST-EXEC: btest-diff out-pre-only +# @TEST-EXEC: btest-diff out-post-only +# @TEST-EXEC: btest-diff out-both + +@load-plugin Demo::Meta_Hooks + +redef allow_network_time_forward = F; + +event zeek_init() + { + print "zeek_init()"; + }