From de640d651f35787042b4f1585e3a282b430a84c7 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 2 Nov 2018 18:30:01 -0500 Subject: [PATCH 01/14] Fix a unit test relying on a bash-ism --- CHANGES | 4 ++++ VERSION | 2 +- testing/btest/language/index-assignment-invalid.bro | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 989ba30e5c..d7efb8de5b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-beta2-68 | 2018-11-02 18:30:01 -0500 + + * Fix a unit test relying on a bash-ism (Jon Siwek, Corelight) + 2.6-beta2-67 | 2018-11-02 17:41:46 -0500 * Add script-layer call stack to internal errors messages that abort (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index b73b32485f..68c60ba15c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-beta2-67 +2.6-beta2-68 diff --git a/testing/btest/language/index-assignment-invalid.bro b/testing/btest/language/index-assignment-invalid.bro index 170183fb8a..3e164b6064 100644 --- a/testing/btest/language/index-assignment-invalid.bro +++ b/testing/btest/language/index-assignment-invalid.bro @@ -1,6 +1,6 @@ # @TEST-EXEC-FAIL: bro -b %INPUT >output 2>&1 # @TEST-EXEC: grep "internal error" output >output2 -# @TEST-EXEC: for i in {1..5}; do cat output2 | cut -d'|' -f$i >>out; done +# @TEST-EXEC: for i in 1 2 3 4 5; do cat output2 | cut -d'|' -f$i >>out; done # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out @load base/utils/queue From 7ec480f05df21be533db53780320e0f2a568cade Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Sat, 3 Nov 2018 10:57:25 -0500 Subject: [PATCH 02/14] Updating submodule(s). [nomail] --- aux/broker | 2 +- src/3rdparty | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broker b/aux/broker index 54db7298eb..615d15e423 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 54db7298eba858c4ceb4df7ef05cdbc8f18c8598 +Subproject commit 615d15e4230259f946c8003f1a607f9fe3214f65 diff --git a/src/3rdparty b/src/3rdparty index 16c6bd63fc..1594233572 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 16c6bd63fca5c901a8616b657b58180e0f19588b +Subproject commit 159423357282935f72060bd3a2780501cba7dd9a From 93c094fff2b21b4b0ac8ae19089bceb101a5258a Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Thu, 1 Nov 2018 14:23:06 -0500 Subject: [PATCH 03/14] Switch GridFTP options from redef to option --- scripts/base/protocols/ftp/gridftp.bro | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/base/protocols/ftp/gridftp.bro b/scripts/base/protocols/ftp/gridftp.bro index 38f6d8186c..cdbe354a08 100644 --- a/scripts/base/protocols/ftp/gridftp.bro +++ b/scripts/base/protocols/ftp/gridftp.bro @@ -30,15 +30,15 @@ module GridFTP; export { ## Number of bytes transferred before guessing a connection is a ## GridFTP data channel. - const size_threshold = 1073741824 &redef; + option size_threshold = 1073741824; ## Time during which we check whether a connection's size exceeds the ## :bro:see:`GridFTP::size_threshold`. - const max_time = 2 min &redef; + option max_time = 2 min; ## Whether to skip further processing of the GridFTP data channel once ## detected, which may help performance. - const skip_data = T &redef; + option skip_data = T; ## Raised when a GridFTP data channel is detected. ## From 38259afa8dba94975133042150ced67293712df8 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 5 Nov 2018 13:27:49 -0600 Subject: [PATCH 04/14] Switch frag_timeout from redef to option --- scripts/base/init-bare.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 93bcc203b7..9e65f0b6c6 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1150,7 +1150,7 @@ const rpc_timeout = 24 sec &redef; ## How long to hold onto fragments for possible reassembly. A value of 0.0 ## means "forever", which resists evasion, but can lead to state accrual. -const frag_timeout = 0.0 sec &redef; +option frag_timeout = 0.0 sec; ## If positive, indicates the encapsulation header size that should ## be skipped. This applies to all packets. From 4f6db2f960892edeef33bbaf17efb1886923a27c Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Mon, 5 Nov 2018 14:00:44 -0600 Subject: [PATCH 05/14] Update baseline for plugins.hooks for redef -> option changes --- testing/btest/Baseline/plugins.hooks/output | 26 +++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index dce61908e3..503a0258f9 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -274,7 +274,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -459,7 +459,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -482,6 +482,9 @@ 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (FileExtract::default_limit, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Files::enable_reassembler, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (GridFTP::max_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (GridFTP::size_threshold, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (GridFTP::skip_data, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (HTTP::default_capture_password, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (HTTP::proxy_headers, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> @@ -532,6 +535,7 @@ 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Weird::weird_do_not_ignore_repeats, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (default_file_bof_buffer_size, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (default_file_timeout_interval, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (frag_timeout, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) -> 0.000000 MetaHookPost CallFunction(PacketFilter::install, , ()) -> @@ -1165,7 +1169,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1350,7 +1354,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1373,6 +1377,9 @@ 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (FileExtract::default_limit, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Files::enable_reassembler, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (GridFTP::max_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (GridFTP::size_threshold, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (GridFTP::skip_data, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (HTTP::default_capture_password, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (HTTP::proxy_headers, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) @@ -1423,6 +1430,7 @@ 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Weird::weird_do_not_ignore_repeats, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (default_file_bof_buffer_size, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (default_file_timeout_interval, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (frag_timeout, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(PacketFilter::build, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) 0.000000 MetaHookPre CallFunction(PacketFilter::install, , ()) @@ -2055,7 +2063,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2240,7 +2248,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2263,6 +2271,9 @@ 0.000000 | HookCallFunction Option::set_change_handler(FTP::logged_commands, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(FileExtract::default_limit, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(Files::enable_reassembler, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(GridFTP::max_time, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(GridFTP::size_threshold, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(GridFTP::skip_data, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(HTTP::default_capture_password, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(HTTP::http_methods, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(HTTP::proxy_headers, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) @@ -2313,6 +2324,7 @@ 0.000000 | HookCallFunction Option::set_change_handler(Weird::weird_do_not_ignore_repeats, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(default_file_bof_buffer_size, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(default_file_timeout_interval, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(frag_timeout, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) 0.000000 | HookCallFunction PacketFilter::install() @@ -2666,7 +2678,7 @@ 0.000000 | HookLoadFile base<...>/x509 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1539361390.052019, node=bro, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent bro_init() 0.000000 | HookQueueEvent filter_change_tracking() From c4a1299359fcb41b41a565231102d414750a989a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 5 Nov 2018 14:48:57 -0600 Subject: [PATCH 06/14] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 615d15e423..6c43fbfd45 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 615d15e4230259f946c8003f1a607f9fe3214f65 +Subproject commit 6c43fbfd45fec2f069134f900d4f1f3a5264debe From 03f42fabf35015e2379dae6f9f92913c4a4e9a1b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 5 Nov 2018 14:53:18 -0600 Subject: [PATCH 07/14] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 6c43fbfd45..f62604ea46 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 6c43fbfd45fec2f069134f900d4f1f3a5264debe +Subproject commit f62604ea464d3c99c1a175bcd70627bfec5ac35f From 2d82fe7e2e7eb249fb50783957acd27555edf775 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 5 Nov 2018 17:10:21 -0600 Subject: [PATCH 08/14] Improve error handling in x509_ocsp_verify function --- src/file_analysis/analyzer/x509/functions.bif | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif index 3622e0d13a..0b18feb8fe 100644 --- a/src/file_analysis/analyzer/x509/functions.bif +++ b/src/file_analysis/analyzer/x509/functions.bif @@ -303,7 +303,12 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c int result = -1; X509* issuer_certificate = 0; X509* signer = 0; + ASN1_GENERALIZEDTIME* thisUpdate = nullptr; + ASN1_GENERALIZEDTIME* nextUpdate = nullptr; + int type = -1; + OCSP_RESPONSE *resp = d2i_OCSP_RESPONSE(NULL, &start, ocsp_reply->Len()); + if ( ! resp ) { rval = x509_result_record(-1, "Could not parse OCSP response"); @@ -441,13 +446,35 @@ function x509_ocsp_verify%(certs: x509_opaque_vector, ocsp_reply: string, root_c return x509_result_record(-1, "OCSP reply is not for host certificate"); // next - check freshness of proof... - ASN1_GENERALIZEDTIME *thisUpdate; - ASN1_GENERALIZEDTIME *nextUpdate; - int type; type = OCSP_single_get0_status(single, NULL, NULL, &thisUpdate, &nextUpdate); - if ( ! ASN1_GENERALIZEDTIME_check(thisUpdate) || ! ASN1_GENERALIZEDTIME_check(nextUpdate) ) + + if ( type == -1 ) { - rval = x509_result_record(-1, "OCSP reply contains invalid dates"); + rval = x509_result_record(-1, "OCSP reply failed to retrieve update times"); + goto x509_ocsp_cleanup; + } + + if ( ! thisUpdate ) + { + rval = x509_result_record(-1, "OCSP reply missing thisUpdate field"); + goto x509_ocsp_cleanup; + } + + if ( ! nextUpdate ) + { + rval = x509_result_record(-1, "OCSP reply missing nextUpdate field"); + goto x509_ocsp_cleanup; + } + + if ( ! ASN1_GENERALIZEDTIME_check(thisUpdate) ) + { + rval = x509_result_record(-1, "OCSP reply contains invalid thisUpdate field"); + goto x509_ocsp_cleanup; + } + + if ( ! ASN1_GENERALIZEDTIME_check(nextUpdate) ) + { + rval = x509_result_record(-1, "OCSP reply contains invalid nextUpdate field"); goto x509_ocsp_cleanup; } From c1f02aaa618494466dbffaaabc56736528fb0bb1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 6 Nov 2018 09:20:44 -0600 Subject: [PATCH 09/14] Revert "Switch frag_timeout from redef to option" This reverts commit 38259afa8dba94975133042150ced67293712df8. The change to an option wasn't yet incomplete -- that would require dynamic lookup of the value at the C++ layer. --- CHANGES | 6 ++++++ VERSION | 2 +- scripts/base/init-bare.bro | 2 +- testing/btest/Baseline/plugins.hooks/output | 17 +++++++---------- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index d7efb8de5b..34a6e6ed81 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.6-beta2-77 | 2018-11-06 09:32:17 -0600 + + * Switch GridFTP options from redef to option (Vlad Grigorescu) + + * Improve error handling in x509_ocsp_verify function (Jon Siwek, Corelight) + 2.6-beta2-68 | 2018-11-02 18:30:01 -0500 * Fix a unit test relying on a bash-ism (Jon Siwek, Corelight) diff --git a/VERSION b/VERSION index 68c60ba15c..550c3aaebf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-beta2-68 +2.6-beta2-77 diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 9e65f0b6c6..93bcc203b7 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1150,7 +1150,7 @@ const rpc_timeout = 24 sec &redef; ## How long to hold onto fragments for possible reassembly. A value of 0.0 ## means "forever", which resists evasion, but can lead to state accrual. -option frag_timeout = 0.0 sec; +const frag_timeout = 0.0 sec &redef; ## If positive, indicates the encapsulation header size that should ## be skipped. This applies to all packets. diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 503a0258f9..9f609d3690 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -274,7 +274,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Broker::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Config::LOG)) -> @@ -459,7 +459,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -535,7 +535,6 @@ 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (Weird::weird_do_not_ignore_repeats, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (default_file_bof_buffer_size, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (default_file_timeout_interval, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> -0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (frag_timeout, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) -> 0.000000 MetaHookPost CallFunction(PacketFilter::install, , ()) -> @@ -1169,7 +1168,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Broker::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Config::LOG)) @@ -1354,7 +1353,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1430,7 +1429,6 @@ 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (Weird::weird_do_not_ignore_repeats, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (default_file_bof_buffer_size, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (default_file_timeout_interval, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (frag_timeout, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(PacketFilter::build, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) 0.000000 MetaHookPre CallFunction(PacketFilter::install, , ()) @@ -2063,7 +2061,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG) @@ -2248,7 +2246,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2324,7 +2322,6 @@ 0.000000 | HookCallFunction Option::set_change_handler(Weird::weird_do_not_ignore_repeats, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(default_file_bof_buffer_size, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(default_file_timeout_interval, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) -0.000000 | HookCallFunction Option::set_change_handler(frag_timeout, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) 0.000000 | HookCallFunction PacketFilter::install() @@ -2678,7 +2675,7 @@ 0.000000 | HookLoadFile base<...>/x509 0.000000 | HookLoadFile base<...>/xmpp 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} -0.000000 | HookLogWrite packet_filter [ts=1541448007.385835, node=bro, filter=ip or not ip, init=T, success=T] +0.000000 | HookLogWrite packet_filter [ts=1541517939.221621, node=bro, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init() 0.000000 | HookQueueEvent bro_init() 0.000000 | HookQueueEvent filter_change_tracking() From f470dfdb92f534674d0b8789697566681ef28cdc Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 6 Nov 2018 18:45:00 -0600 Subject: [PATCH 10/14] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index f62604ea46..04c08cb534 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit f62604ea464d3c99c1a175bcd70627bfec5ac35f +Subproject commit 04c08cb5343ea5cfcc00d978dc74ccebe1ca3757 From 808f1d1524a341afbaa3f69c15724e3b713adf17 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 7 Nov 2018 10:27:00 -0600 Subject: [PATCH 11/14] Fix coding conventions nits/typos --- CHANGES | 4 ++++ VERSION | 2 +- src/Conn.cc | 4 ++-- src/parse.y | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/CHANGES b/CHANGES index 34a6e6ed81..a0abba26a9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-beta2-79 | 2018-11-07 10:27:00 -0600 + + * Fix coding conventions nits/typos (Vern Paxson, Corelight) + 2.6-beta2-77 | 2018-11-06 09:32:17 -0600 * Switch GridFTP options from redef to option (Vlad Grigorescu) diff --git a/VERSION b/VERSION index 550c3aaebf..48e96e7c51 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-beta2-77 +2.6-beta2-79 diff --git a/src/Conn.cc b/src/Conn.cc index 2bcb82de26..9781fb175d 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -412,7 +412,7 @@ RecordVal* Connection::BuildConnVal() id_val->Assign(2, new AddrVal(resp_addr)); id_val->Assign(3, port_mgr->Get(ntohs(resp_port), prot_type)); - RecordVal *orig_endp = new RecordVal(endpoint); + RecordVal* orig_endp = new RecordVal(endpoint); orig_endp->Assign(0, new Val(0, TYPE_COUNT)); orig_endp->Assign(1, new Val(0, TYPE_COUNT)); orig_endp->Assign(4, new Val(orig_flow_label, TYPE_COUNT)); @@ -423,7 +423,7 @@ RecordVal* Connection::BuildConnVal() if ( memcmp(&orig_l2_addr, &null, l2_len) != 0 ) orig_endp->Assign(5, new StringVal(fmt_mac(orig_l2_addr, l2_len))); - RecordVal *resp_endp = new RecordVal(endpoint); + RecordVal* resp_endp = new RecordVal(endpoint); resp_endp->Assign(0, new Val(0, TYPE_COUNT)); resp_endp->Assign(1, new Val(0, TYPE_COUNT)); resp_endp->Assign(4, new Val(resp_flow_label, TYPE_COUNT)); diff --git a/src/parse.y b/src/parse.y index 6afab9c3bb..b5809a91d0 100644 --- a/src/parse.y +++ b/src/parse.y @@ -990,7 +990,7 @@ type: { NullStmt here; if ( $1 ) - $1->Error("not a BRO type", &here); + $1->Error("not a Bro type", &here); $$ = error_type(); } else From ca014d6e35786477c66cd2e4254d39c737a5dd99 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 7 Nov 2018 11:46:34 -0600 Subject: [PATCH 12/14] Support appending to vector of any For consistency, since it already works to assign to an index within vector of any. --- CHANGES | 4 ++++ VERSION | 2 +- src/Expr.cc | 5 +++-- .../Baseline/language.vector-any-append/out | 1 + testing/btest/language/vector-any-append.bro | 22 +++++++++++++++++++ 5 files changed, 31 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/language.vector-any-append/out create mode 100644 testing/btest/language/vector-any-append.bro diff --git a/CHANGES b/CHANGES index a0abba26a9..f1747aa825 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-beta2-80 | 2018-11-07 11:46:34 -0600 + + * Support appending to vector of any (Jon Siwek, Corelight) + 2.6-beta2-79 | 2018-11-07 10:27:00 -0600 * Fix coding conventions nits/typos (Vern Paxson, Corelight) diff --git a/VERSION b/VERSION index 48e96e7c51..5d586fae4e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-beta2-79 +2.6-beta2-80 diff --git a/src/Expr.cc b/src/Expr.cc index 3c9be75bf9..84db97b2d6 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -1485,8 +1485,9 @@ AddToExpr::AddToExpr(Expr* arg_op1, Expr* arg_op2) ExprError("appending non-arithmetic to arithmetic vector"); } - else if ( bt1 != bt2 ) - ExprError("incompatible vector append"); + else if ( bt1 != bt2 && bt1 != TYPE_ANY ) + ExprError(fmt("incompatible vector append: %s and %s", + type_name(bt1), type_name(bt2))); else SetType(op1->Type()->Ref()); diff --git a/testing/btest/Baseline/language.vector-any-append/out b/testing/btest/Baseline/language.vector-any-append/out new file mode 100644 index 0000000000..3571ae1207 --- /dev/null +++ b/testing/btest/Baseline/language.vector-any-append/out @@ -0,0 +1 @@ +[0, 1, 2, 3] diff --git a/testing/btest/language/vector-any-append.bro b/testing/btest/language/vector-any-append.bro new file mode 100644 index 0000000000..816627fbf1 --- /dev/null +++ b/testing/btest/language/vector-any-append.bro @@ -0,0 +1,22 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function assign(v: vector of any) + { + v[|v|] = |v|; + } + +function append(v: vector of any) + { + v += |v|; + } + +event bro_init() + { + local v: vector of count; + assign(v); + assign(v); + append(v); + append(v); + print v; + } From 74c6b9f54c0da3792d6060082e772ddd6f1f0093 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Wed, 7 Nov 2018 13:11:15 -0800 Subject: [PATCH 13/14] Remove unnessary check The call to Empty() was originally meant as an optimization in the lookup phase. However, the performance implications are substantial: this check operates in O(f(m/8)) where m is the number of bits in the Bloom filters and f a function that looks for the first non-empty block of bits. As the Bloom filter fills up, the check for Empty() becomes no longer negligible and can lead to serious performance degradations when Bloom filters are used frequently. --- src/probabilistic/bloom-filter.bif | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/probabilistic/bloom-filter.bif b/src/probabilistic/bloom-filter.bif index 46ec4699a0..9af2ae0d33 100644 --- a/src/probabilistic/bloom-filter.bif +++ b/src/probabilistic/bloom-filter.bif @@ -173,9 +173,6 @@ function bloomfilter_lookup%(bf: opaque of bloomfilter, x: any%): count %{ const BloomFilterVal* bfv = static_cast(bf); - if ( bfv->Empty() ) - return new Val(0, TYPE_COUNT); - if ( ! bfv->Type() ) reporter->Error("cannot perform lookup on untyped Bloom filter"); From 01e17b5ea0eaaca138295a79c09d52f79498c79d Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 8 Nov 2018 12:25:21 -0600 Subject: [PATCH 14/14] Fix SumStats "last" plugin in cluster mode --- CHANGES | 4 ++ VERSION | 2 +- .../base/frameworks/sumstats/plugins/last.bro | 18 ++++- .../manager-1..stdout | 9 +++ .../base/frameworks/sumstats/last-cluster.bro | 68 +++++++++++++++++++ 5 files changed, 97 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.last-cluster/manager-1..stdout create mode 100644 testing/btest/scripts/base/frameworks/sumstats/last-cluster.bro diff --git a/CHANGES b/CHANGES index 6720c7c65c..14a5675b22 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.6-beta2-83 | 2018-11-08 12:25:21 -0600 + + * Fix SumStats "last" plugin in cluster mode (Jon Siwek, Corelight) + 2.6-beta2-82 | 2018-11-08 09:38:52 -0600 * Remove unnecessary Bloom filter empty check (Matthias Vallentin) diff --git a/VERSION b/VERSION index 19268ed322..58a2094532 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-beta2-82 +2.6-beta2-83 diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index ca04114f61..b12d854bbb 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -17,7 +17,8 @@ export { }; redef record ResultVal += { - ## This is the queue where elements are maintained. Use the + ## This is the queue where elements are maintained. + ## Don't access this value directly, instead use the ## :bro:see:`SumStats::get_last` function to get a vector of ## the current element values. last_elements: Queue::Queue &optional; @@ -29,10 +30,21 @@ export { function get_last(rv: ResultVal): vector of Observation { - local s: vector of Observation = vector(); + local s: vector of any = vector(); + if ( rv?$last_elements ) Queue::get_vector(rv$last_elements, s); - return s; + + local rval: vector of Observation = vector(); + + for ( i in s ) + # When using the cluster-ized version of SumStats, Queue's + # internal table storage uses "any" type for values, so we need + # to cast them here or else they may be left as Broker::Data from + # the unserialization process. + rval += s[i] as Observation; + + return rval; } hook register_observe_plugins() diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.last-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.last-cluster/manager-1..stdout new file mode 100644 index 0000000000..f4ba3206b0 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.last-cluster/manager-1..stdout @@ -0,0 +1,9 @@ +node up, worker-1 +test thresh crossed, [[num=0, dbl=, str=]] +test thresh crossed, [[num=1, dbl=, str=]] +test thresh crossed, [[num=2, dbl=, str=]] +test thresh crossed, [[num=3, dbl=, str=]] +test thresh crossed, [[num=4, dbl=, str=]] +test thresh crossed, [[num=5, dbl=, str=]] +test thresh crossed, [[num=6, dbl=, str=]] +test thresh crossed, [[num=7, dbl=, str=]] diff --git a/testing/btest/scripts/base/frameworks/sumstats/last-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/last-cluster.bro new file mode 100644 index 0000000000..7e41934cb1 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/sumstats/last-cluster.bro @@ -0,0 +1,68 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT +# @TEST-EXEC: btest-bg-wait 25 + +# @TEST-EXEC: btest-diff manager-1/.stdout +# +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], +}; +@TEST-END-FILE + +global c = 0; + +event do_observe() + { + print "do observe", c; + SumStats::observe("test", + [$str=cat(c)], + [$num=c] + ); + ++c; + schedule 0.1secs { do_observe() }; + } + +event bro_init() + { + local r1 = SumStats::Reducer($stream="test", + $apply=set(SumStats::LAST), + $num_last_elements=1 + ); + + SumStats::create([$name="test", + $epoch=10secs, + $reducers=set(r1), + $threshold_val(key: SumStats::Key, result: SumStats::Result): double = { return 2.0; }, + $threshold = 1.0, + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + local l = SumStats::get_last(result["test"]); + print "test thresh crossed", l; + + if ( l[0]$num == 7 ) + terminate(); + } + ]); + } + +event Cluster::node_up(name: string, id: string) + { + print "node up", name; + + if ( Cluster::node == "worker-1" && name == "manager-1" ) + schedule 0.1secs { do_observe() }; + } + +event Cluster::node_down(name: string, id: string) + { + print "node down", name; + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, id: string) + { + terminate(); + }