diff --git a/CHANGES b/CHANGES index c807dbc422..5d9026291e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.6-186 | 2019-03-25 09:41:57 -0700 + + * Added policy script for intel removal. (Jan Grashoefer) + + * Added Intel::filter_item hook to filter intelligence items. (Jan Grashoefer) + 2.6-178 | 2019-03-21 14:10:44 -0700 * Add support for parsing SMB 3.1.1 NegotiateContextList response values (Mauro Palumbo) diff --git a/NEWS b/NEWS index 89b024a424..3b0b2dbe2d 100644 --- a/NEWS +++ b/NEWS @@ -61,6 +61,9 @@ New Functionality containing capability information found in an SMB 3.1.1 dialect's negotiation message. +- Added a new hook, ``Intel::filter_item``, to assist in filtering and + removal of intelligence items that are about to be inserted. + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 91b521efc5..ddf1957a25 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6-178 +2.6-186 diff --git a/doc b/doc index ee040c581f..ae04aa61a8 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit ee040c581f8a4e1ea82a7ec5f17d5f36f1184324 +Subproject commit ae04aa61a80ff4f436a9c248f9a934a694bb500a diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 592d2b1027..4bc3b296dd 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -173,6 +173,14 @@ export { ## be removed. global item_expired: hook(indicator: string, indicator_type: Type, metas: set[MetaData]); + ## This hook can be used to filter intelligence items that are about to be + ## inserted into the internal data store. In case the hook execution is + ## terminated using break, the item will not be (re)added to the internal + ## data store. + ## + ## item: The intel item that should be inserted. + global filter_item: hook(item: Intel::Item); + global log_intel: event(rec: Info); } @@ -496,8 +504,11 @@ function _insert(item: Item, first_dispatch: bool &default = T) function insert(item: Item) { - # Insert possibly new item. - _insert(item, T); + if ( hook filter_item(item) ) + { + # Insert possibly new item. + _insert(item, T); + } } # Function to check whether an item is present. diff --git a/scripts/policy/frameworks/intel/removal.bro b/scripts/policy/frameworks/intel/removal.bro new file mode 100644 index 0000000000..cc4bb42921 --- /dev/null +++ b/scripts/policy/frameworks/intel/removal.bro @@ -0,0 +1,23 @@ +##! This script enables removal of intelligence items. + +@load base/frameworks/intel + +module Intel; + +export { + redef record Intel::MetaData += { + ## A boolean value to indicate whether the item should be removed. + remove: bool &default=F; + }; +} + +hook Intel::filter_item(item: Item) + { + if ( item$meta$remove ) + { + Intel::remove(item); + # Prevent readding + break; + } + } + diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index c3c2941b06..2e38738527 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -16,6 +16,7 @@ @load frameworks/intel/do_notice.bro @load frameworks/intel/do_expire.bro @load frameworks/intel/whitelist.bro +@load frameworks/intel/removal.bro @load frameworks/intel/seen/__load__.bro @load frameworks/intel/seen/conn-established.bro @load frameworks/intel/seen/dns.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log new file mode 100644 index 0000000000..dfe45974c1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2019-03-24-20-29-18 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc +#types time string addr port addr port string enum enum string set[enum] set[string] string string string +1553459358.205227 - - - - - 1.2.3.42 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - +#close 2019-03-24-20-29-18 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output index dca63f379e..90d390518f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path reporter -#open 2018-02-27-17-25-30 +#open 2019-03-24-20-20-10 #fields ts level message location #types time enum string string -0.000000 Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). /home/jgras/devel/bro/scripts/base/frameworks/intel/./main.bro, lines 552-553 +0.000000 Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). /home/jgras/devel/zeek/scripts/base/frameworks/intel/./main.bro, lines 563-564 0.000000 Reporter::INFO received termination signal (empty) -#close 2018-02-27-17-25-30 +#close 2019-03-24-20-20-10 diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log new file mode 100644 index 0000000000..d43abf187b --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2019-03-24-21-15-06 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc +#types time string addr port addr port string enum enum string set[enum] set[string] string string string +1553462106.131323 - - - - - 10.0.0.2 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - +#close 2019-03-24-21-15-06 diff --git a/testing/btest/scripts/base/frameworks/intel/filter-item.bro b/testing/btest/scripts/base/frameworks/intel/filter-item.bro new file mode 100644 index 0000000000..c598664996 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/filter-item.bro @@ -0,0 +1,43 @@ + +# @TEST-EXEC: btest-bg-run broproc bro %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff broproc/intel.log + +@TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +1.2.3.42 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +10.0.0.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +@TEST-END-FILE + +redef exit_only_after_terminate = T; +redef Site::local_nets += { 10.0.0.0/8 }; +redef Intel::read_files += { "../intel.dat" }; +redef enum Intel::Where += { SOMEWHERE }; + +hook Intel::filter_item(item: Intel::Item) + { + if ( item$indicator_type == Intel::ADDR && + Site::is_local_addr(to_addr(item$indicator)) ) + break; + } + +event do_it() + { + Intel::seen([$host=10.0.0.1, + $where=SOMEWHERE]); + Intel::seen([$host=1.2.3.42, + $where=SOMEWHERE]); + } + +global log_lines = 0; +event Intel::log_intel(rec: Intel::Info) + { + ++log_lines; + if ( log_lines == 1 ) + terminate(); + } + +event bro_init() &priority=-10 + { + schedule 1sec { do_it() }; + } diff --git a/testing/btest/scripts/policy/frameworks/intel/removal.bro b/testing/btest/scripts/policy/frameworks/intel/removal.bro new file mode 100644 index 0000000000..4d7e450da4 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/intel/removal.bro @@ -0,0 +1,46 @@ + +# @TEST-EXEC: btest-bg-run broproc bro %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff broproc/intel.log + +@TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.remove +10.0.0.1 Intel::ADDR source1 T +10.0.0.2 Intel::ADDR source1 F +@TEST-END-FILE + +@load frameworks/intel/removal + +redef exit_only_after_terminate = T; +redef Intel::read_files += { "../intel.dat" }; +redef enum Intel::Where += { SOMEWHERE }; + +hook Intel::filter_item(item: Intel::Item) + { + if ( item$indicator_type == Intel::ADDR && + Site::is_local_addr(to_addr(item$indicator)) ) + break; + } + +event do_it() + { + Intel::seen([$host=10.0.0.1, + $where=SOMEWHERE]); + Intel::seen([$host=10.0.0.2, + $where=SOMEWHERE]); + } + +global log_lines = 0; +event Intel::log_intel(rec: Intel::Info) + { + ++log_lines; + if ( log_lines == 1 ) + terminate(); + } + +event bro_init() &priority=-10 + { + Intel::insert([$indicator="10.0.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); + Intel::insert([$indicator="10.0.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); + schedule 1sec { do_it() }; + } \ No newline at end of file