From cb330287022b506d870cd970e6f0aa1a810bb08e Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Wed, 11 May 2016 23:27:51 +0200 Subject: [PATCH] Added hook to allow extending the intel log. The extension mechanism is basically the one that Seth introduced with his intel extensions. The main difference lies in using a hook instead of an event. An example policy implements whitelisting. --- scripts/base/frameworks/intel/main.bro | 21 +++++++++- scripts/policy/frameworks/intel/whitelist.bro | 30 ++++++++++++++ scripts/test-all-policy.bro | 1 + .../intel.log | 29 ++++++++++++++ .../policy/frameworks/intel/whitelisting.bro | 39 +++++++++++++++++++ 5 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 scripts/policy/frameworks/intel/whitelist.bro create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log create mode 100644 testing/btest/scripts/policy/frameworks/intel/whitelisting.bro diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index b52b30aff0..55494507a7 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -165,6 +165,19 @@ export { ## data within the intelligence framework. global match: event(s: Seen, items: set[Item]); + ## This hook can be used to extend the intel log by adding data to the + ## Info record. The default information is added with a priority of 5. + ## + ## info: The Info record that will be logged. + ## + ## s: Information about the data seen. + ## + ## items: The intel items that match the seen data. + ## + ## In case the hook execution is terminated using break, the match will + ## not be logged. + global extend_match: hook(info: Info, s: Seen, items: set[Item]); + global log_intel: event(rec: Info); } @@ -306,6 +319,12 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 { local info = Info($ts=network_time(), $seen=s, $matched=TypeSet()); + if ( hook extend_match(info, s, items) ) + Log::write(Intel::LOG, info); + } + +hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5 + { if ( s?$f ) { s$fuid = s$f$id; @@ -340,8 +359,6 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 add info$sources[item$meta$source]; add info$matched[item$indicator_type]; } - - Log::write(Intel::LOG, info); } function insert(item: Item) diff --git a/scripts/policy/frameworks/intel/whitelist.bro b/scripts/policy/frameworks/intel/whitelist.bro new file mode 100644 index 0000000000..9061ed2a91 --- /dev/null +++ b/scripts/policy/frameworks/intel/whitelist.bro @@ -0,0 +1,30 @@ + +@load base/frameworks/intel +@load base/frameworks/notice + +module Intel; + +export { + redef record Intel::MetaData += { + ## Add a field to indicate if this is a whitelisted item. + whitelist: bool &default=F; + }; +} + +hook Intel::extend_match(info: Info, s: Seen, items: set[Item]) &priority=9 + { + local whitelisted = F; + for ( item in items ) + { + if ( item$meta$whitelist ) + { + whitelisted = T; + break; + } + } + + if ( whitelisted ) + # Prevent logging + break; + } + diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index f85fdb58b0..02602d1dc6 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -15,6 +15,7 @@ @load frameworks/dpd/detect-protocols.bro @load frameworks/dpd/packet-segment-logging.bro @load frameworks/intel/do_notice.bro +@load frameworks/intel/whitelist.bro @load frameworks/intel/seen/__load__.bro @load frameworks/intel/seen/conn-established.bro @load frameworks/intel/seen/dns.bro diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log new file mode 100644 index 0000000000..2aabd3b2e5 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log @@ -0,0 +1,29 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2016-05-11-19-38-30 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where seen.node matched sources +#types time string addr port addr port string string string string enum enum string set[enum] set[string] +1300475168.853899 CPbrpk1qSsw6ESzHV4 141.142.220.118 43927 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.854837 CIPOse170MGiRM1Qf4 141.142.220.118 40526 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.857956 CMXxB5GvmoxJFXdTa 141.142.220.118 32902 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.858713 Che1bq3i2rO3KD1Syg 141.142.220.118 59714 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.891644 CEle3f3zno26fFZkrh 141.142.220.118 58206 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.892414 CfTOmO0HKorjr8Zp7 141.142.220.118 59746 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.893988 Cab0vO1xNYSS2hJkle 141.142.220.118 45000 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.894787 Cx3C534wEyF3OvvcQe 141.142.220.118 48128 141.142.2.2 53 - - - upload.wikimedia.org Intel::DOMAIN DNS::IN_REQUEST bro Intel::DOMAIN source1 +1300475168.916018 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.916183 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.918358 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.952296 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.952307 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.954820 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.975934 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.976436 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475168.979264 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475169.014593 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475169.014619 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +1300475169.014927 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 - - - upload.wikimedia.org Intel::DOMAIN HTTP::IN_HOST_HEADER bro Intel::DOMAIN source1 +#close 2016-05-11-19-38-30 diff --git a/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro b/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro new file mode 100644 index 0000000000..53acd49aa9 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro @@ -0,0 +1,39 @@ +# @TEST-EXEC: bro -Cr $TRACES/wikipedia.trace %INPUT +# @TEST-EXEC: btest-diff intel.log + +#@TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +upload.wikimedia.org Intel::DOMAIN source1 somehow bad http://some-data-distributor.com/1 +meta.wikimedia.org Intel::DOMAIN source1 also bad http://some-data-distributor.com/1 +#@TEST-END-FILE + +#@TEST-START-FILE whitelist.dat +#fields indicator indicator_type meta.source meta.desc meta.whitelist meta.url +meta.wikimedia.org Intel::DOMAIN source2 also bad T http://some-data-distributor.com/1 +#@TEST-END-FILE + +@load base/frameworks/intel +@load frameworks/intel/whitelist +@load frameworks/intel/seen + +redef Intel::read_files += { + "intel.dat", + "whitelist.dat", +}; + +global total_files_read = 0; + +event bro_init() + { + suspend_processing(); + } + +event Input::end_of_data(name: string, source: string) + { + # Wait until both intel files are read. + if ( /^intel-/ in name && (++total_files_read == 2) ) + { + continue_processing(); + } + } +