diff --git a/auxil/bifcl b/auxil/bifcl index 89878a05ef..bfb1f62ade 160000 --- a/auxil/bifcl +++ b/auxil/bifcl @@ -1 +1 @@ -Subproject commit 89878a05efe2a99eb14c286716676efca6c18b71 +Subproject commit bfb1f62adeb9e558227a57d7c73b48f5ee2d5670 diff --git a/scripts/base/init-frameworks-and-bifs.zeek b/scripts/base/init-frameworks-and-bifs.zeek index a667110a87..e7ffcedc8b 100644 --- a/scripts/base/init-frameworks-and-bifs.zeek +++ b/scripts/base/init-frameworks-and-bifs.zeek @@ -14,3 +14,13 @@ # Load BiFs defined by plugins. @load base/bif/plugins + +# This sets up secondary/subdir BIFs such that they can be used by any +# further scripts within their global initializations and is intended to be +# the last thing done within this script. It's called within @if simply so +# that it executes at parse-time. An alternative way to do that is to call +# it during a global variable assignment/initialization. Formally adding a +# @run directive to the language whose sole purpose is parse-time code +# execution would be another idea. +@if ( __init_secondary_bifs() ) +@endif diff --git a/src/Func.cc b/src/Func.cc index 0ea206e83c..7317723f9f 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -74,6 +74,7 @@ extern RETSIGTYPE sig_handler(int signo); namespace zeek::detail { std::vector call_stack; bool did_builtin_init = false; +std::vector bif_initializers; static const std::pair empty_hook_result(false, nullptr); } // namespace zeek::detail @@ -936,9 +937,6 @@ void builtin_error(const char* msg, zeek::Obj* arg) zeek::emit_builtin_error(msg, arg); } -#include "__all__.bif.cc" // Autogenerated for compiling in the bif_target() code. -#include "__all__.bif.register.cc" // Autogenerated for compiling in the bif_target() code. - void init_builtin_funcs() { ProcStats = zeek::id::find_type("ProcStats"); @@ -966,8 +964,3 @@ void init_builtin_funcs() zeek::detail::did_builtin_init = true; } - -void init_builtin_funcs_subdirs() - { -#include "__all__.bif.init.cc" // Autogenerated for compiling in the bif_target() code. - } diff --git a/src/Func.h b/src/Func.h index 7ac7158d60..121d721b4d 100644 --- a/src/Func.h +++ b/src/Func.h @@ -276,6 +276,15 @@ extern std::vector call_stack; // This is set to true after the built-in functions have been initialized. extern bool did_builtin_init; +extern std::vector bif_initializers; + +inline void run_bif_initializers() + { + for ( const auto& bi : bif_initializers ) + bi(); + + bif_initializers = {}; + } extern void emit_builtin_exception(const char* msg); extern void emit_builtin_exception(const char* msg, const ValPtr& arg); @@ -305,7 +314,6 @@ constexpr auto render_call_stack [[deprecated("Remove in v4.1. Use zeek::render_ // renamed version inside the namespace, but the way that the code gets included complicates the matter. It // might need to be revisited after everything is namespaced everywhere else. void init_builtin_funcs(); -void init_builtin_funcs_subdirs(); // TODO: do call_stack and did_builtin_init need to be aliased? diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index c0b2073dc1..d84f011b21 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -637,7 +637,7 @@ SetupResult setup(int argc, char** argv, init_general_global_var(); init_net_var(); - init_builtin_funcs_subdirs(); + run_bif_initializers(); // Must come after plugin activation (and also after hash // initialization). diff --git a/src/zeek.bif b/src/zeek.bif index f2a1d4b2cb..8f516ccaaf 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -5046,6 +5046,24 @@ function match_signatures%(c: connection, pattern_type: int, s: string, return zeek::val_mgr->True(); %} +%%{ +// Autogenerated from CMake bif_target() +#include "__all__.bif.cc" +#include "__all__.bif.register.cc" + +static void init_secondary_bifs() + { + #include "__all__.bif.init.cc" + } +%%} + +## An internal function that helps initialize BIFs. +function __init_secondary_bifs%(%): bool + %{ + init_secondary_bifs(); + return zeek::val_mgr->True(); + %} + # =========================================================================== # # Anonymization Functions diff --git a/testing/btest/Baseline/language.global-init-calls-bif/out b/testing/btest/Baseline/language.global-init-calls-bif/out new file mode 100644 index 0000000000..fce6bb6608 --- /dev/null +++ b/testing/btest/Baseline/language.global-init-calls-bif/out @@ -0,0 +1 @@ +3.002199 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index f9fa09deab..6dadea6dfe 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -282,7 +282,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1594057891.73307, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -463,7 +463,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1594057891.73307, node=zeek, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -569,6 +569,7 @@ 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::UNIQUE, lambda_<10387912117292132662>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, , (SumStats::VARIANCE, lambda_<6557258612059469785>{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugins, , ()) -> +0.000000 MetaHookPost CallFunction(__init_secondary_bifs, , ()) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(filter_change_tracking, , ()) -> 0.000000 MetaHookPost CallFunction(getenv, , (CLUSTER_NODE)) -> @@ -1205,7 +1206,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1594057891.73307, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1386,7 +1387,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1594057891.73307, node=zeek, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1492,6 +1493,7 @@ 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::UNIQUE, lambda_<10387912117292132662>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, , (SumStats::VARIANCE, lambda_<6557258612059469785>{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugins, , ()) +0.000000 MetaHookPre CallFunction(__init_secondary_bifs, , ()) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(filter_change_tracking, , ()) 0.000000 MetaHookPre CallFunction(getenv, , (CLUSTER_NODE)) @@ -2127,7 +2129,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1594057891.73307, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2308,7 +2310,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1594057891.73307, node=zeek, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2414,6 +2416,7 @@ 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, lambda_<10387912117292132662>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals}) 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::VARIANCE, lambda_<6557258612059469785>{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average}) 0.000000 | HookCallFunction SumStats::register_observe_plugins() +0.000000 | HookCallFunction __init_secondary_bifs() 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction filter_change_tracking() 0.000000 | HookCallFunction getenv(CLUSTER_NODE) @@ -2762,7 +2765,7 @@ 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLoadFile base<...>/zeek.bif.zeek 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1594057891.733070, node=zeek, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent filter_change_tracking() 0.000000 | HookQueueEvent zeek_init() diff --git a/testing/btest/language/global-init-calls-bif.zeek b/testing/btest/language/global-init-calls-bif.zeek new file mode 100644 index 0000000000..215037aa7f --- /dev/null +++ b/testing/btest/language/global-init-calls-bif.zeek @@ -0,0 +1,14 @@ +# @TEST-EXEC: zeek -b %INPUT >out +# @TEST-EXEC: btest-diff out + +# This test isn't specifically testing the HLL cardinality functionality, +# rather that a global variable can be initialized using a BIF call. +# Also, it's particularly not a top-level BIF, but one defined in a subdir +# of the Zeek source tree (those are treated differently than top-level BIFs). + +global my_cc = hll_cardinality_init(0.1, 0.999); + +hll_cardinality_add(my_cc, 1); +hll_cardinality_add(my_cc, 2); +hll_cardinality_add(my_cc, 3); +print hll_cardinality_estimate(my_cc);