Merge remote-tracking branch 'origin/topic/awelzel/generic-metadata-pre-work'

* origin/topic/awelzel/generic-metadata-pre-work:
  cluster/Backend: Add name and lookup component tag
  cluster/Event: Hide members behind accessors
  cluster/PublishEvent:: Make event non-const
  broker/Manager: Re-use broker serializer for conversion
  EventMgr: Add Dispatch() with handler and args
  plugin/Manager: Fix MetaHookPre and MetaHookPost using HOOK_CALL_FUNCTION
This commit is contained in:
Arne Welzel 2025-04-13 17:11:40 +02:00
commit c2e039f14d
26 changed files with 318 additions and 66 deletions

View file

@ -3352,6 +3352,7 @@ XXXXXXXXXX.XXXXXX MetaHookPost LogWrite(Log::WRITER_ASCII, default, conn(XXXX
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(Broker::log_flush()) -> false
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]])) -> false
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T)) -> false
XXXXXXXXXX.XXXXXX MetaHookPost QueueEvent(net_done(XXXXXXXXXX.XXXXXX)) -> false
XXXXXXXXXX.XXXXXX MetaHookPost UpdateNetworkTime(XXXXXXXXXX.XXXXXX) -> <void>
XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::__flush_logs, <frame>, ())
XXXXXXXXXX.XXXXXX MetaHookPre CallFunction(Broker::flush_logs, <frame>, ())
@ -3382,6 +3383,7 @@ XXXXXXXXXX.XXXXXX MetaHookPre LogWrite(Log::WRITER_ASCII, default, conn(XXXX
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(Broker::log_flush())
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]]))
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T))
XXXXXXXXXX.XXXXXX MetaHookPre QueueEvent(net_done(XXXXXXXXXX.XXXXXX))
XXXXXXXXXX.XXXXXX MetaHookPre UpdateNetworkTime(XXXXXXXXXX.XXXXXX)
XXXXXXXXXX.XXXXXX | HookUpdateNetworkTime XXXXXXXXXX.XXXXXX
XXXXXXXXXX.XXXXXX | HookCallFunction Broker::__flush_logs()
@ -3413,3 +3415,4 @@ XXXXXXXXXX.XXXXXX | HookLogWrite conn [ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjw
XXXXXXXXXX.XXXXXX | HookQueueEvent Broker::log_flush()
XXXXXXXXXX.XXXXXX | HookQueueEvent connection_state_remove([id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]])
XXXXXXXXXX.XXXXXX | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0, l2_addr=c8:bc:c8:96:d2:a0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0, l2_addr=00:10:db:88:d2:ef], start_time=XXXXXXXXXX.XXXXXX, duration=211.0 msecs 483.955383 usecs, service={HTTP}, history=ShADadFf, uid=CHhAvVGS1DHFjwGM9, tunnel=<uninitialized>, vlan=<uninitialized>, inner_vlan=<uninitialized>, removal_hooks={HTTP::finalize_http: Conn::RemovalHook{ if (HTTP::c?$http_state) { for ([HTTP::r], HTTP::info in HTTP::c$http_state$pending) { if (0 == HTTP::r) next Log::write(HTTP::LOG, to_any_coerce HTTP::info)}}}}, dpd=<uninitialized>, service_violation={}, conn=<uninitialized>, extract_orig=F, extract_resp=F, thresholds=<uninitialized>, http=[ts=XXXXXXXXXX.XXXXXX, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp, proto=6], trans_depth=1, method=GET, host=bro.org, uri=<...>/CHANGES.bro-aux.txt, referrer=<uninitialized>, version=1.1, user_agent=Wget/1.14 (darwin12.2.0), origin=<uninitialized>, request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, tags={}, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_filenames=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FMnxxt3xjVcWNS2141], resp_filenames=<uninitialized>, resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1]], T)
XXXXXXXXXX.XXXXXX | HookQueueEvent net_done(XXXXXXXXXX.XXXXXX)

View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 MetaHookPre QueueEvent(zeek_init())
0.000000 HookQueueEvent zeek_init()
0.000000 MetaHookPost QueueEvent(zeek_init()) -> false
zeek_init()
0.000000 MetaHookPre QueueEvent(net_done(1.0))
0.000000 HookQueueEvent net_done()
0.000000 MetaHookPost QueueEvent(net_done(1.0)) -> false
0.000000 MetaHookPre QueueEvent(Broker::log_flush())
0.000000 HookQueueEvent Broker::log_flush()
0.000000 MetaHookPost QueueEvent(Broker::log_flush()) -> false

View file

@ -0,0 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 HookQueueEvent zeek_init()
zeek_init()
0.000000 HookQueueEvent net_done()
0.000000 HookQueueEvent Broker::log_flush()

View file

@ -0,0 +1,8 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 HookQueueEvent zeek_init()
0.000000 MetaHookPost QueueEvent() -> false
zeek_init()
0.000000 HookQueueEvent net_done()
0.000000 MetaHookPost QueueEvent() -> false
0.000000 HookQueueEvent Broker::log_flush()
0.000000 MetaHookPost QueueEvent() -> false

View file

@ -0,0 +1,8 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 MetaHookPre QueueEvent(zeek_init())
0.000000 HookQueueEvent zeek_init()
zeek_init()
0.000000 MetaHookPre QueueEvent(net_done(1.0))
0.000000 HookQueueEvent net_done()
0.000000 MetaHookPre QueueEvent(Broker::log_flush())
0.000000 HookQueueEvent Broker::log_flush()

View file

@ -0,0 +1,5 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0.000000 HookQueueEvent zeek_init()
zeek_init()
0.000000 HookQueueEvent net_done()
0.000000 HookQueueEvent Broker::log_flush()

View file

@ -0,0 +1,83 @@
#include "Plugin.h"
#include <zeek/Desc.h>
#include <zeek/Event.h>
#include <zeek/Func.h>
#include <zeek/threading/Formatter.h>
#include <cstdlib>
namespace btest::plugin::Demo_Meta_Hooks {
Plugin plugin;
}
using namespace btest::plugin::Demo_Meta_Hooks;
zeek::plugin::Configuration Plugin::Configure() {
zeek::plugin::Configuration config;
config.name = "Demo::Meta_Hooks";
config.description = "Test if the meta hooks are working";
config.version.major = 1;
config.version.minor = 0;
config.version.patch = 0;
// This plugin enables HookQueueEvent() and optionally the pre and post
// meta hooks controlled by environment variables for easier testing.
EnableHook(zeek::plugin::HOOK_QUEUE_EVENT);
if ( getenv("TEST_META_HOOK_PRE") )
EnableHook(zeek::plugin::META_HOOK_PRE);
if ( getenv("TEST_META_HOOK_POST") )
EnableHook(zeek::plugin::META_HOOK_POST);
return config;
}
static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek::ODesc* d) {
bool first = true;
for ( const auto& arg : args ) {
if ( ! first )
d->Add(", ");
arg.Describe(d);
first = false;
}
}
bool Plugin::HookQueueEvent(zeek::Event* e) {
fprintf(stdout, "%.6f %-15s %s()\n", zeek::run_state::network_time, " HookQueueEvent", e->Handler()->Name());
return false;
}
void Plugin::MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) {
// The spicy integration enables HOOK_LOAD_FILE and this plugin receives
// meta hooks also for that :-/
if ( hook != zeek::plugin::HOOK_QUEUE_EVENT )
return;
zeek::ODesc d;
d.SetShort();
describe_hook_args(args, &d);
fprintf(stdout, "%.6f %-15s %s(%s)\n", zeek::run_state::network_time, " MetaHookPre", hook_name(hook),
d.Description());
}
void Plugin::MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args,
zeek::plugin::HookArgument result) {
// The spicy integration enables HOOK_LOAD_FILE and this plugin receives
// meta hooks also for that :-/
if ( hook != zeek::plugin::HOOK_QUEUE_EVENT )
return;
zeek::ODesc d1;
zeek::ODesc d2;
describe_hook_args(args, &d1);
result.Describe(&d2);
fprintf(stdout, "%.6f %-15s %s(%s) -> %s\n", zeek::run_state::network_time, " MetaHookPost", hook_name(hook),
d1.Description(), d2.Description());
}

View file

@ -0,0 +1,21 @@
#pragma once
#include <zeek/plugin/Plugin.h>
namespace btest::plugin::Demo_Meta_Hooks {
class Plugin : public zeek::plugin::Plugin {
protected:
bool HookQueueEvent(zeek::Event* e) override;
void MetaHookPre(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args) override;
void MetaHookPost(zeek::plugin::HookType hook, const zeek::plugin::HookArgumentList& args,
zeek::plugin::HookArgument result) override;
// Overridden from plugin::Plugin.
zeek::plugin::Configuration Configure() override;
};
extern Plugin plugin;
} // namespace btest::plugin::Demo_Meta_Hooks

View file

@ -0,0 +1,23 @@
# @TEST-DOC: Plugin testing the meta hooks specifically. This is a regression test for these being enabled with HookCallFunction() instead.
#
# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Meta_Hooks
# @TEST-EXEC: cp -r %DIR/meta-hook-plugin/* .
# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
# @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-none
# @TEST-EXEC: TEST_META_HOOK_PRE=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-pre-only
# @TEST-EXEC: TEST_META_HOOK_POST=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-post-only
# @TEST-EXEC: TEST_META_HOOK_PRE=1 TEST_META_HOOK_POST=1 ZEEK_PLUGIN_PATH=`pwd` zeek -b %INPUT >out-both
#
# @TEST-EXEC: btest-diff out-none
# @TEST-EXEC: btest-diff out-pre-only
# @TEST-EXEC: btest-diff out-post-only
# @TEST-EXEC: btest-diff out-both
@load-plugin Demo::Meta_Hooks
redef allow_network_time_forward = F;
event zeek_init()
{
print "zeek_init()";
}