From 15ec117da678d8df54d018b93f9460dea782700b Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 18 Dec 2014 11:57:32 -0600 Subject: [PATCH 1/5] Correct a typo in the Notice framework doc --- doc/frameworks/notice.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/frameworks/notice.rst b/doc/frameworks/notice.rst index 2c20149ce5..d8197c13af 100644 --- a/doc/frameworks/notice.rst +++ b/doc/frameworks/notice.rst @@ -271,7 +271,7 @@ script that is generating the notice has indicated to the notice framework how to identify notices that are intrinsically the same. Identification of these "intrinsically duplicate" notices is implemented with an optional field in :bro:see:`Notice::Info` records named ``$identifier`` which is a simple string. -If the ``$identifier`` and ``$type`` fields are the same for two notices, the +If the ``$identifier`` and ``$note`` fields are the same for two notices, the notice framework actually considers them to be the same thing and can use that information to suppress duplicates for a configurable period of time. From 9af5fb1302239a5b88e0ce35857f7005f4e48fb0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 31 Dec 2014 09:14:55 -0800 Subject: [PATCH 2/5] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 90f9ca0ffa..eb1d029e51 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 90f9ca0ffa2306f0d1d2ac208cdbb7787199f890 +Subproject commit eb1d029e5161c4dfff00fd190d8da22c0bf8ba50 From bd8893f0d0d4ce48423c696f6706a985267e6398 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 31 Dec 2014 09:19:09 -0800 Subject: [PATCH 3/5] Changing Makefile's test-all to run test-all for broctl. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9feaecd656..207ce72780 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ test: @( cd testing && make ) test-all: test - test -d aux/broctl && ( cd aux/broctl && make test ) + test -d aux/broctl && ( cd aux/broctl && make test-all ) test -d aux/btest && ( cd aux/btest && make test ) test -d aux/bro-aux && ( cd aux/bro-aux && make test ) test -d aux/plugins && ( cd aux/plugins && make test-all ) From 494545f1ebfd569e39a41863ccfd38b884b50127 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 31 Dec 2014 09:19:34 -0800 Subject: [PATCH 4/5] Updating submodule(s). [nomail] --- CHANGES | 5 +++++ VERSION | 2 +- aux/broctl | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 2d41f9ad1b..4c546131db 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-348 | 2014-12-31 09:19:34 -0800 + + * Changing Makefile's test-all to run test-all for broctl, which now + executes trace-summary tests as well. (Robin Sommer) + 2.3-345 | 2014-12-31 09:06:15 -0800 * Correct a typo in the Notice framework doc. (Daniel Thayer) diff --git a/VERSION b/VERSION index 196c840941..378e5b5ce8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-345 +2.3-348 diff --git a/aux/broctl b/aux/broctl index eb1d029e51..8c9b87bc73 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit eb1d029e5161c4dfff00fd190d8da22c0bf8ba50 +Subproject commit 8c9b87bc73e1ddaa304e3d89028c1e7b95d37a91 From 1971d25a5cb895f039ac809d02d986c9dc118b18 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 5 Jan 2015 15:21:13 -0600 Subject: [PATCH 5/5] Fix race condition in unified2 file analyzer startup. Retrieval of extended alert information from sid-msg.map, gen-msg.map, and classification.config files uses Bro's input framework, but since the unified2 file analyzer also relies on the input framework, coordination is needed to start analysis only after extended info has been read at least once. --- CHANGES | 4 + VERSION | 2 +- scripts/base/files/unified2/main.bro | 95 ++++++++++++++++----- testing/btest/Baseline/plugins.hooks/output | 24 ++++-- 4 files changed, 94 insertions(+), 31 deletions(-) diff --git a/CHANGES b/CHANGES index 4c546131db..60c40cbce6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-349 | 2015-01-05 15:21:13 -0600 + + * Fix race condition in unified2 file analyzer startup. (Jon siwek) + 2.3-348 | 2014-12-31 09:19:34 -0800 * Changing Makefile's test-all to run test-all for broctl, which now diff --git a/VERSION b/VERSION index 378e5b5ce8..f2b1636819 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-348 +2.3-349 diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 2f6ae79f4f..627bcc9fee 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -71,11 +71,50 @@ global classification_map: table[count] of string; global sid_map: table[count] of string; global gen_map: table[count] of string; +global num_classification_map_reads = 0; +global num_sid_map_reads = 0; +global num_gen_map_reads = 0; +global watching = F; + # For reading in config files. type OneLine: record { line: string; }; +function mappings_initialized(): bool + { + return num_classification_map_reads > 0 && + num_sid_map_reads > 0 && + num_gen_map_reads > 0; + } + +function start_watching() + { + if ( watching ) + return; + + watching = T; + + if ( watch_dir != "" ) + { + Dir::monitor(watch_dir, function(fname: string) + { + Input::add_analysis([$source=fname, + $reader=Input::READER_BINARY, + $mode=Input::STREAM, + $name=fname]); + }, 10secs); + } + + if ( watch_file != "" ) + { + Input::add_analysis([$source=watch_file, + $reader=Input::READER_BINARY, + $mode=Input::STREAM, + $name=watch_file]); + } + } + function create_info(ev: IDSEvent): Info { local info = Info($ts=ev$ts, @@ -136,11 +175,33 @@ event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Inp } } +event Input::end_of_data(name: string, source: string) + { + if ( name == classification_config ) + ++num_classification_map_reads; + else if ( name == sid_msg ) + ++num_sid_map_reads; + else if ( name == gen_msg ) + ++num_gen_map_reads; + else + return; + + if ( watching ) + return; + + if ( mappings_initialized() ) + start_watching(); + } + event bro_init() &priority=5 { Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]); - if ( sid_msg != "" ) + if ( sid_msg == "" ) + { + num_sid_map_reads = 1; + } + else { Input::add_event([$source=sid_msg, $reader=Input::READER_RAW, @@ -151,7 +212,11 @@ event bro_init() &priority=5 $ev=Unified2::read_sid_msg_line]); } - if ( gen_msg != "" ) + if ( gen_msg == "" ) + { + num_gen_map_reads = 1; + } + else { Input::add_event([$source=gen_msg, $name=gen_msg, @@ -162,7 +227,11 @@ event bro_init() &priority=5 $ev=Unified2::read_gen_msg_line]); } - if ( classification_config != "" ) + if ( classification_config == "" ) + { + num_classification_map_reads = 1; + } + else { Input::add_event([$source=classification_config, $name=classification_config, @@ -173,24 +242,8 @@ event bro_init() &priority=5 $ev=Unified2::read_classification_line]); } - if ( watch_dir != "" ) - { - Dir::monitor(watch_dir, function(fname: string) - { - Input::add_analysis([$source=fname, - $reader=Input::READER_BINARY, - $mode=Input::STREAM, - $name=fname]); - }, 10secs); - } - - if ( watch_file != "" ) - { - Input::add_analysis([$source=watch_file, - $reader=Input::READER_BINARY, - $mode=Input::STREAM, - $name=watch_file]); - } + if ( mappings_initialized() ) + start_watching(); } event file_new(f: fa_file) diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 31dd415e1b..d18499d5dd 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -189,7 +189,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Conn::LOG)) -> @@ -283,8 +283,8 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T])) -> -0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> @@ -303,6 +303,8 @@ 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, (SumStats::UNIQUE, anonymous-function{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || flattenSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = flattenSumStats::rv$unique_vals})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugin, (SumStats::VARIANCE, anonymous-function{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) -> 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugins, ()) -> +0.000000 MetaHookPost CallFunction(Unified2::mappings_initialized, ()) -> +0.000000 MetaHookPost CallFunction(Unified2::start_watching, ()) -> 0.000000 MetaHookPost CallFunction(bro_init, ()) -> 0.000000 MetaHookPost CallFunction(cat, (Packe, t, _, Filter)) -> 0.000000 MetaHookPost CallFunction(current_time, ()) -> @@ -724,7 +726,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Conn::LOG)) @@ -818,8 +820,8 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T])) -0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) @@ -838,6 +840,8 @@ 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, (SumStats::UNIQUE, anonymous-function{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || flattenSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = flattenSumStats::rv$unique_vals})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugin, (SumStats::VARIANCE, anonymous-function{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average})) 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugins, ()) +0.000000 MetaHookPre CallFunction(Unified2::mappings_initialized, ()) +0.000000 MetaHookPre CallFunction(Unified2::start_watching, ()) 0.000000 MetaHookPre CallFunction(bro_init, ()) 0.000000 MetaHookPre CallFunction(cat, (Packe, t, _, Filter)) 0.000000 MetaHookPre CallFunction(current_time, ()) @@ -1259,7 +1263,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1353,8 +1357,8 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1414788015.369883, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1420492465.686432, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) @@ -1373,6 +1377,8 @@ 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::UNIQUE, anonymous-function{ if (!SumStats::rv?$unique_vals) SumStats::rv$unique_vals = (coerce set() to set[SumStats::Observation])if (SumStats::r?$unique_max) SumStats::rv$unique_max = SumStats::r$unique_maxif (!SumStats::r?$unique_max || flattenSumStats::rv$unique_vals <= SumStats::r$unique_max) add SumStats::rv$unique_vals[SumStats::obs]SumStats::rv$unique = flattenSumStats::rv$unique_vals}) 0.000000 | HookCallFunction SumStats::register_observe_plugin(SumStats::VARIANCE, anonymous-function{ if (1 < SumStats::rv$num) SumStats::rv$var_s += ((SumStats::val - SumStats::rv$prev_avg) * (SumStats::val - SumStats::rv$average))SumStats::calc_variance(SumStats::rv)SumStats::rv$prev_avg = SumStats::rv$average}) 0.000000 | HookCallFunction SumStats::register_observe_plugins() +0.000000 | HookCallFunction Unified2::mappings_initialized() +0.000000 | HookCallFunction Unified2::start_watching() 0.000000 | HookCallFunction bro_init() 0.000000 | HookCallFunction cat(Packe, t, _, Filter) 0.000000 | HookCallFunction current_time()