diff --git a/scripts/base/frameworks/broker/main.bro b/scripts/base/frameworks/broker/main.bro index 451d4cf86b..5368d5422b 100644 --- a/scripts/base/frameworks/broker/main.bro +++ b/scripts/base/frameworks/broker/main.bro @@ -14,7 +14,7 @@ export { ## Default address on which to listen. ## ## .. bro:see:: Broker::listen - const default_listen_address = "" &redef; + const default_listen_address = getenv("BRO_DEFAULT_LISTEN_ADDRESS") &redef; ## Default interval to retry connecting to a peer if it cannot be made to work ## initially, or if it ever becomes disconnected. diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index d5bb8f2be9..c20a31279f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1,4 +1,4 @@ -@load base/bif/const.bif.bro +@load base/bif/const.bif @load base/bif/types.bif # Type declarations @@ -1797,9 +1797,11 @@ type gtp_delete_pdp_ctx_response_elements: record { }; # Prototypes of Bro built-in functions. -@load base/bif/strings.bif @load base/bif/bro.bif +@load base/bif/stats.bif @load base/bif/reporter.bif +@load base/bif/strings.bif +@load base/bif/option.bif ## Deprecated. This is superseded by the new logging framework. global log_file_name: function(tag: string): string &redef; @@ -4822,17 +4824,3 @@ const global_hash_seed: string = "" &redef; ## files. The larger the value, the more confidence in UID uniqueness. ## The maximum is currently 128 bits. const bits_per_uid: count = 96 &redef; - -# Load these frameworks here because they use fairly deep integration with -# BiFs and script-land defined types. -@load base/frameworks/logging -@load base/frameworks/broker -@load base/frameworks/input -@load base/frameworks/analyzer -@load base/frameworks/files - -@load base/bif - -# Load BiFs defined by plugins. -@load base/bif/plugins - diff --git a/scripts/base/init-frameworks-and-bifs.bro b/scripts/base/init-frameworks-and-bifs.bro new file mode 100644 index 0000000000..f772e2d223 --- /dev/null +++ b/scripts/base/init-frameworks-and-bifs.bro @@ -0,0 +1,15 @@ +# Load these frameworks here because they use fairly deep integration with +# BiFs and script-land defined types. They are also more likely to +# make use of calling BIFs for variable initializations, and that +# can't be done until init-bare.bro has been loaded completely (hence +# the separate file). +@load base/frameworks/logging +@load base/frameworks/broker +@load base/frameworks/input +@load base/frameworks/analyzer +@load base/frameworks/files + +@load base/bif + +# Load BiFs defined by plugins. +@load base/bif/plugins diff --git a/src/input.h b/src/input.h index f0f402b23b..3d0caa459a 100644 --- a/src/input.h +++ b/src/input.h @@ -14,6 +14,7 @@ extern int yydebug; extern int brolex(); extern char last_tok[128]; +extern void add_essential_input_file(const char* file); extern void add_input_file(const char* file); extern void add_input_file_at_front(const char* file); diff --git a/src/main.cc b/src/main.cc index 2277ab0cba..2a61c753b8 100644 --- a/src/main.cc +++ b/src/main.cc @@ -755,7 +755,9 @@ int main(int argc, char** argv) broxygen_mgr = new broxygen::Manager(broxygen_config, bro_argv[0]); - add_input_file("base/init-bare.bro"); + add_essential_input_file("base/init-bare.bro"); + add_essential_input_file("base/init-frameworks-and-bifs.bro"); + if ( ! bare_mode ) add_input_file("base/init-default.bro"); diff --git a/src/scan.l b/src/scan.l index 41fb758bc6..27490c13ad 100644 --- a/src/scan.l +++ b/src/scan.l @@ -821,6 +821,18 @@ void do_atendif() // are referred to (in order to save the locations of tokens and statements, // for error reporting and debugging). static name_list input_files; +static name_list essential_input_files; + +void add_essential_input_file(const char* file) + { + if ( ! file ) + reporter->InternalError("empty filename"); + + if ( ! filename ) + (void) load_files(file); + else + essential_input_files.append(copy_string(file)); + } void add_input_file(const char* file) { @@ -869,7 +881,7 @@ int yywrap() if ( ! did_builtin_init && file_stack.length() == 1 ) { // ### This is a gross hack - we know that the first file - // we parse is bro.init, and after it it's safe to initialize + // we parse is init-bare.bro, and after it it's safe to initialize // the built-ins. Furthermore, we want to initialize the // built-in's *right* after parsing bro.init, so that other // source files can use built-in's when initializing globals. @@ -885,19 +897,22 @@ int yywrap() return 0; // Stack is now empty. - while ( input_files.length() > 0 ) + while ( essential_input_files.length() > 0 || input_files.length() > 0 ) { - if ( load_files(input_files[0]) ) + name_list& files = essential_input_files.length() > 0 ? + essential_input_files : input_files; + + if ( load_files(files[0]) ) { // Don't delete the filename - it's pointed to by // every BroObj created when parsing it. - (void) input_files.remove_nth(0); + (void) files.remove_nth(0); return 0; } // We already scanned the file. Pop it and try the next, // if any. - (void) input_files.remove_nth(0); + (void) files.remove_nth(0); } // For each file scanned so far, and for each @prefix, look for a diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 768eb520ea..c56e53f28d 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,18 +3,21 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2018-03-01-16-07-03 +#open 2018-06-01-20-31-44 #fields name #types string scripts/base/init-bare.bro build/scripts/base/bif/const.bif.bro build/scripts/base/bif/types.bif.bro - build/scripts/base/bif/strings.bif.bro build/scripts/base/bif/bro.bif.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/reporter.bif.bro + build/scripts/base/bif/strings.bif.bro + build/scripts/base/bif/option.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/plugins/Bro_KRB.types.bif.bro build/scripts/base/bif/event.bif.bro +scripts/base/init-frameworks-and-bifs.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -52,8 +55,6 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro - build/scripts/base/bif/stats.bif.bro - build/scripts/base/bif/option.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/pcap.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -175,4 +176,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2018-03-01-16-07-03 +#close 2018-06-01-20-31-44 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 5ca6cdd812..11bb9e1fa2 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,18 +3,21 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2018-05-17-23-49-59 +#open 2018-06-01-20-31-46 #fields name #types string scripts/base/init-bare.bro build/scripts/base/bif/const.bif.bro build/scripts/base/bif/types.bif.bro - build/scripts/base/bif/strings.bif.bro build/scripts/base/bif/bro.bif.bro + build/scripts/base/bif/stats.bif.bro build/scripts/base/bif/reporter.bif.bro + build/scripts/base/bif/strings.bif.bro + build/scripts/base/bif/option.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/plugins/Bro_KRB.types.bif.bro build/scripts/base/bif/event.bif.bro +scripts/base/init-frameworks-and-bifs.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -52,8 +55,6 @@ scripts/base/init-bare.bro scripts/base/utils/patterns.bro scripts/base/frameworks/files/magic/__load__.bro build/scripts/base/bif/__load__.bro - build/scripts/base/bif/stats.bif.bro - build/scripts/base/bif/option.bif.bro build/scripts/base/bif/broxygen.bif.bro build/scripts/base/bif/pcap.bif.bro build/scripts/base/bif/bloom-filter.bif.bro @@ -364,4 +365,4 @@ scripts/base/init-default.bro scripts/base/misc/find-filtered-trace.bro scripts/base/misc/version.bro scripts/policy/misc/loaded-scripts.bro -#close 2018-05-17-23-49-59 +#close 2018-06-01-20-31-46 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 37a613347c..4a7b2c0854 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -264,7 +264,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -441,7 +441,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -469,6 +469,7 @@ 0.000000 MetaHookPost CallFunction(bro_init, , ()) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(filter_change_tracking, , ()) -> +0.000000 MetaHookPost CallFunction(getenv, , (BRO_DEFAULT_LISTEN_ADDRESS)) -> 0.000000 MetaHookPost CallFunction(getenv, , (CLUSTER_NODE)) -> 0.000000 MetaHookPost CallFunction(global_ids, , ()) -> 0.000000 MetaHookPost CallFunction(network_time, , ()) -> @@ -727,6 +728,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/http) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/imap) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/init-default.bro) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/init-frameworks-and-bifs.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/input.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/intel) -> -1 @@ -744,6 +746,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/ntlm) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/numbers.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/openflow) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/option.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/packet-filter) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/paths.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/patterns.bro) -> -1 @@ -766,6 +769,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/software) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssh) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/ssl) -> -1 +0.000000 MetaHookPost LoadFile(0, base<...>/stats.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/store.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/strings.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/strings.bro) -> -1 @@ -1062,7 +1066,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1239,7 +1243,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1267,6 +1271,7 @@ 0.000000 MetaHookPre CallFunction(bro_init, , ()) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(filter_change_tracking, , ()) +0.000000 MetaHookPre CallFunction(getenv, , (BRO_DEFAULT_LISTEN_ADDRESS)) 0.000000 MetaHookPre CallFunction(getenv, , (CLUSTER_NODE)) 0.000000 MetaHookPre CallFunction(global_ids, , ()) 0.000000 MetaHookPre CallFunction(network_time, , ()) @@ -1525,6 +1530,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/http) 0.000000 MetaHookPre LoadFile(0, base<...>/imap) 0.000000 MetaHookPre LoadFile(0, base<...>/init-default.bro) +0.000000 MetaHookPre LoadFile(0, base<...>/init-frameworks-and-bifs.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/input) 0.000000 MetaHookPre LoadFile(0, base<...>/input.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/intel) @@ -1542,6 +1548,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/ntlm) 0.000000 MetaHookPre LoadFile(0, base<...>/numbers.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/openflow) +0.000000 MetaHookPre LoadFile(0, base<...>/option.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/packet-filter) 0.000000 MetaHookPre LoadFile(0, base<...>/paths.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/patterns.bro) @@ -1564,6 +1571,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/software) 0.000000 MetaHookPre LoadFile(0, base<...>/ssh) 0.000000 MetaHookPre LoadFile(0, base<...>/ssl) +0.000000 MetaHookPre LoadFile(0, base<...>/stats.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/store.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/strings.bif.bro) 0.000000 MetaHookPre LoadFile(0, base<...>/strings.bro) @@ -1859,7 +1867,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2036,7 +2044,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2064,6 +2072,7 @@ 0.000000 | HookCallFunction bro_init() 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction filter_change_tracking() +0.000000 | HookCallFunction getenv(BRO_DEFAULT_LISTEN_ADDRESS) 0.000000 | HookCallFunction getenv(CLUSTER_NODE) 0.000000 | HookCallFunction global_ids() 0.000000 | HookCallFunction network_time() @@ -2331,6 +2340,7 @@ 0.000000 | HookLoadFile base<...>/http 0.000000 | HookLoadFile base<...>/imap 0.000000 | HookLoadFile base<...>/init-default.bro +0.000000 | HookLoadFile base<...>/init-frameworks-and-bifs.bro 0.000000 | HookLoadFile base<...>/input 0.000000 | HookLoadFile base<...>/input.bif.bro 0.000000 | HookLoadFile base<...>/intel @@ -2348,6 +2358,7 @@ 0.000000 | HookLoadFile base<...>/ntlm 0.000000 | HookLoadFile base<...>/numbers.bro 0.000000 | HookLoadFile base<...>/openflow +0.000000 | HookLoadFile base<...>/option.bif.bro 0.000000 | HookLoadFile base<...>/packet-filter 0.000000 | HookLoadFile base<...>/paths.bro 0.000000 | HookLoadFile base<...>/patterns.bro @@ -2370,6 +2381,7 @@ 0.000000 | HookLoadFile base<...>/software 0.000000 | HookLoadFile base<...>/ssh 0.000000 | HookLoadFile base<...>/ssl +0.000000 | HookLoadFile base<...>/stats.bif.bro 0.000000 | HookLoadFile base<...>/store.bif.bro 0.000000 | HookLoadFile base<...>/strings.bif.bro 0.000000 | HookLoadFile base<...>/strings.bro @@ -2387,7 +2399,7 @@ 0.000000 | HookLoadFile base<...>/x509 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1525287517.317589, node=bro, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1527879383.723919, node=bro, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent bro_init() 0.000000 | HookQueueEvent filter_change_tracking() diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 8a69f5eddc..81ee5bd5e5 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -25,3 +25,4 @@ TMPDIR=%(testbase)s/.tmp BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX BTEST_RST_FILTER=$SCRIPTS/rst-filter BRO_DNS_FAKE=1 +BRO_DEFAULT_LISTEN_ADDRESS=127.0.0.1