Improve how primary/top-level BIFs get initialized

scripts/base/init-bare.zeek

@@ -5331,3 +5331,13 @@ const digest_salt = "Please change this value." &redef;
 global done_with_network = F;
 event net_done(t: time)
 	{ done_with_network = T; }
+
+# This sets up primary BIFs such that they can be used by any
+# further scripts within their global initializations and is intended to be
+# the last thing done within this script.  It's called within @if simply so
+# that it executes at parse-time.  An alternative way to do that is to call
+# it during a global variable assignment/initialization.  Formally adding a
+# @run directive to the language whose sole purpose is parse-time code
+# execution would be another idea.
+@if ( __init_primary_bifs() )
+@endif

src/Func.cc

@@ -902,6 +902,37 @@ void emit_builtin_exception(const char* msg, Obj* arg)
 	emit_builtin_error_common(msg, arg, true);
 	}
 
+void init_primary_bifs()
+	{
+	if ( did_builtin_init )
+		return;
+
+	ProcStats = id::find_type<RecordType>("ProcStats");
+	NetStats = id::find_type<RecordType>("NetStats");
+	MatcherStats = id::find_type<RecordType>("MatcherStats");
+	ConnStats = id::find_type<RecordType>("ConnStats");
+	ReassemblerStats = id::find_type<RecordType>("ReassemblerStats");
+	DNSStats = id::find_type<RecordType>("DNSStats");
+	GapStats = id::find_type<RecordType>("GapStats");
+	EventStats = id::find_type<RecordType>("EventStats");
+	TimerStats = id::find_type<RecordType>("TimerStats");
+	FileAnalysisStats = id::find_type<RecordType>("FileAnalysisStats");
+	ThreadStats = id::find_type<RecordType>("ThreadStats");
+	BrokerStats = id::find_type<RecordType>("BrokerStats");
+	ReporterStats = id::find_type<RecordType>("ReporterStats");
+
+	var_sizes = id::find_type("var_sizes")->AsTableType();
+
+#include "zeek.bif.func_init"
+#include "stats.bif.func_init"
+#include "reporter.bif.func_init"
+#include "strings.bif.func_init"
+#include "option.bif.func_init"
+#include "supervisor.bif.func_init"
+
+	did_builtin_init = true;
+	}
+
 } // namespace detail
 
 
@@ -936,31 +967,3 @@ void builtin_error(const char* msg, zeek::Obj* arg)
 	{
 	zeek::emit_builtin_error(msg, arg);
 	}
-
-void init_builtin_funcs()
-	{
-	ProcStats = zeek::id::find_type<zeek::RecordType>("ProcStats");
-	NetStats = zeek::id::find_type<zeek::RecordType>("NetStats");
-	MatcherStats = zeek::id::find_type<zeek::RecordType>("MatcherStats");
-	ConnStats = zeek::id::find_type<zeek::RecordType>("ConnStats");
-	ReassemblerStats = zeek::id::find_type<zeek::RecordType>("ReassemblerStats");
-	DNSStats = zeek::id::find_type<zeek::RecordType>("DNSStats");
-	GapStats = zeek::id::find_type<zeek::RecordType>("GapStats");
-	EventStats = zeek::id::find_type<zeek::RecordType>("EventStats");
-	TimerStats = zeek::id::find_type<zeek::RecordType>("TimerStats");
-	FileAnalysisStats = zeek::id::find_type<zeek::RecordType>("FileAnalysisStats");
-	ThreadStats = zeek::id::find_type<zeek::RecordType>("ThreadStats");
-	BrokerStats = zeek::id::find_type<zeek::RecordType>("BrokerStats");
-	ReporterStats = zeek::id::find_type<zeek::RecordType>("ReporterStats");
-
-	var_sizes = zeek::id::find_type("var_sizes")->AsTableType();
-
-#include "zeek.bif.func_init"
-#include "stats.bif.func_init"
-#include "reporter.bif.func_init"
-#include "strings.bif.func_init"
-#include "option.bif.func_init"
-#include "supervisor.bif.func_init"
-
-	zeek::detail::did_builtin_init = true;
-	}

src/Func.h

@@ -277,6 +277,7 @@ extern std::vector<CallInfo> call_stack;
 // This is set to true after the built-in functions have been initialized.
 extern bool did_builtin_init;
 extern std::vector<void (*)()> bif_initializers;
+extern void init_primary_bifs();
 
 inline void run_bif_initializers()
 	{
@@ -310,11 +311,6 @@ using function_ingredients [[deprecated("Remove in v4.1. Use zeek::detail::funct
 constexpr auto check_built_in_call [[deprecated("Remove in v4.1. Use zeek::detail::check_built_in_call.")]] = zeek::detail::check_built_in_call;
 constexpr auto render_call_stack [[deprecated("Remove in v4.1. Use zeek::render_call_stack.")]] = zeek::render_call_stack;
 
-// TODO: these are still here because of how all of the bif code gets included in Func.c. There could be a
-// renamed version inside the namespace, but the way that the code gets included complicates the matter. It
-// might need to be revisited after everything is namespaced everywhere else.
-void init_builtin_funcs();
-
 // TODO: do call_stack and did_builtin_init need to be aliased?
 
 // These have to be implemented as actual methods due to function overloading breaking the use of aliases.

src/scan.l

@@ -842,16 +842,6 @@ int yywrap()
 	if ( zeek::reporter->Errors() > 0 )
 		return 1;
 
-	if ( ! did_builtin_init && file_stack.length() == 1 )
-		{
-		// ### This is a gross hack - we know that the first file
-		// we parse is init-bare.zeek, and after it it's safe to initialize
-		// the built-ins.  Furthermore, we want to initialize the
-		// built-in's *right* after parsing bro.init, so that other
-		// source files can use built-in's when initializing globals.
-		init_builtin_funcs();
-		}
-
 	yy_delete_buffer(YY_CURRENT_BUFFER);
 
 	if ( file_stack.length() > 0 )

src/zeek-setup.cc

@@ -62,6 +62,7 @@ extern "C" {
 #include "broker/Manager.h"
 
 #include "binpac_zeek.h"
+#include "module_util.h"
 
 #include "3rdparty/sqlite3.h"
 
@@ -388,8 +389,7 @@ static std::vector<std::string> get_script_signature_files()
 	return rval;
 	}
 
-SetupResult setup(int argc, char** argv,
+SetupResult setup(int argc, char** argv, Options* zopts)
-                                Options* zopts)
 	{
 	ZEEK_LSAN_DISABLE();
 	std::set_new_handler(bro_new_handler);
@@ -628,6 +628,19 @@ SetupResult setup(int argc, char** argv,
 	HeapLeakChecker::Disabler disabler;
 #endif
 
+	auto ipbid = install_ID("__init_primary_bifs", GLOBAL_MODULE_NAME,
+	                        true, true);
+	auto ipbft = make_intrusive<FuncType>(make_intrusive<RecordType>(nullptr),
+	                                      base_type(TYPE_BOOL),
+	                                      FUNC_FLAVOR_FUNCTION);
+	ipbid->SetType(std::move(ipbft));
+	auto init_bifs = [](Frame* frame, const Args* args) -> BifReturnVal
+		{
+		init_primary_bifs();
+		return val_mgr->True();
+		};
+	auto ipbb = make_intrusive<BuiltinFunc>(init_bifs, ipbid->Name(), false);
+
 	run_state::is_parsing = true;
 	yyparse();
 	run_state::is_parsing = false;

testing/btest/Baseline/plugins.hooks/output

@@ -282,7 +282,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Broker::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Config::LOG)) -> <no result>
@@ -463,7 +463,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -569,6 +569,7 @@
 0.000000   MetaHookPost  CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::UNIQUE, lambda_<10387912117292132662>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})) -> <no result>
 0.000000   MetaHookPost  CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::VARIANCE, lambda_<6557258612059469785>{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) -> <no result>
 0.000000   MetaHookPost  CallFunction(SumStats::register_observe_plugins, <frame>, ()) -> <no result>
+0.000000   MetaHookPost  CallFunction(__init_primary_bifs, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(__init_secondary_bifs, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(current_time, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(filter_change_tracking, <null>, ()) -> <no result>
@@ -1206,7 +1207,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Broker::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Config::LOG))
@@ -1387,7 +1388,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1493,6 +1494,7 @@
 0.000000   MetaHookPre   CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::UNIQUE, lambda_<10387912117292132662>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals}))
 0.000000   MetaHookPre   CallFunction(SumStats::register_observe_plugin, <frame>, (SumStats::VARIANCE, lambda_<6557258612059469785>{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average}))
 0.000000   MetaHookPre   CallFunction(SumStats::register_observe_plugins, <frame>, ())
+0.000000   MetaHookPre   CallFunction(__init_primary_bifs, <null>, ())
 0.000000   MetaHookPre   CallFunction(__init_secondary_bifs, <null>, ())
 0.000000   MetaHookPre   CallFunction(current_time, <frame>, ())
 0.000000   MetaHookPre   CallFunction(filter_change_tracking, <null>, ())
@@ -2129,7 +2131,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG)
@@ -2310,7 +2312,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()
@@ -2416,6 +2418,7 @@
 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, lambda_<10387912117292132662>{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || sizeofSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = sizeofSumStats::rv$unique_vals})
 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::VARIANCE, lambda_<6557258612059469785>{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})
 0.000000 | HookCallFunction SumStats::register_observe_plugins()
+0.000000 | HookCallFunction __init_primary_bifs()
 0.000000 | HookCallFunction __init_secondary_bifs()
 0.000000 | HookCallFunction current_time()
 0.000000 | HookCallFunction filter_change_tracking()
@@ -2765,7 +2768,7 @@
 0.000000 | HookLoadFile  base<...>/xmpp
 0.000000 | HookLoadFile  base<...>/zeek.bif.zeek
 0.000000 | HookLogInit   packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)}
-0.000000 | HookLogWrite  packet_filter [ts=1598552848.405432, node=zeek, filter=ip or not ip, init=T, success=T]
+0.000000 | HookLogWrite  packet_filter [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T]
 0.000000 | HookQueueEvent NetControl::init()
 0.000000 | HookQueueEvent filter_change_tracking()
 0.000000 | HookQueueEvent zeek_init()