From 3e31f3d5543ca17ac4ba5237aa9a91dca723b18c Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Sun, 24 Mar 2019 21:31:48 +0100 Subject: [PATCH 1/5] Added hook to filter intelligence items. --- scripts/base/frameworks/intel/main.bro | 15 +++++++++++++-- .../output | 6 +++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 592d2b1027..4bc3b296dd 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -173,6 +173,14 @@ export { ## be removed. global item_expired: hook(indicator: string, indicator_type: Type, metas: set[MetaData]); + ## This hook can be used to filter intelligence items that are about to be + ## inserted into the internal data store. In case the hook execution is + ## terminated using break, the item will not be (re)added to the internal + ## data store. + ## + ## item: The intel item that should be inserted. + global filter_item: hook(item: Intel::Item); + global log_intel: event(rec: Info); } @@ -496,8 +504,11 @@ function _insert(item: Item, first_dispatch: bool &default = T) function insert(item: Item) { - # Insert possibly new item. - _insert(item, T); + if ( hook filter_item(item) ) + { + # Insert possibly new item. + _insert(item, T); + } } # Function to check whether an item is present. diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output index dca63f379e..90d390518f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path reporter -#open 2018-02-27-17-25-30 +#open 2019-03-24-20-20-10 #fields ts level message location #types time enum string string -0.000000 Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). /home/jgras/devel/bro/scripts/base/frameworks/intel/./main.bro, lines 552-553 +0.000000 Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). /home/jgras/devel/zeek/scripts/base/frameworks/intel/./main.bro, lines 563-564 0.000000 Reporter::INFO received termination signal (empty) -#close 2018-02-27-17-25-30 +#close 2019-03-24-20-20-10 From 0f86aaff4b50a9260d18824b915441ed28b326e0 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Sun, 24 Mar 2019 21:32:45 +0100 Subject: [PATCH 2/5] Added test for intel item filtering. --- .../broproc.intel.log | 10 +++++ .../base/frameworks/intel/filter-item.bro | 43 +++++++++++++++++++ 2 files changed, 53 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log create mode 100644 testing/btest/scripts/base/frameworks/intel/filter-item.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log new file mode 100644 index 0000000000..dfe45974c1 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.filter-item/broproc.intel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2019-03-24-20-29-18 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc +#types time string addr port addr port string enum enum string set[enum] set[string] string string string +1553459358.205227 - - - - - 1.2.3.42 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - +#close 2019-03-24-20-29-18 diff --git a/testing/btest/scripts/base/frameworks/intel/filter-item.bro b/testing/btest/scripts/base/frameworks/intel/filter-item.bro new file mode 100644 index 0000000000..4973674d95 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/filter-item.bro @@ -0,0 +1,43 @@ + +# @TEST-EXEC: btest-bg-run broproc bro %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff broproc/intel.log + +@TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.desc meta.url +1.2.3.42 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +10.0.0.1 Intel::ADDR source1 this host is just plain baaad http://some-data-distributor.com/1234 +@TEST-END-FILE + +redef exit_only_after_terminate = T; +redef Site::local_nets += { 10.0.0.0/8 }; +redef Intel::read_files += { "../intel.dat" }; +redef enum Intel::Where += { SOMEWHERE }; + +hook Intel::filter_item(item: Intel::Item) + { + if ( item$indicator_type == Intel::ADDR && + Site::is_local_addr(to_addr(item$indicator)) ) + break; + } + +event do_it() + { + Intel::seen([$host=1.2.3.42, + $where=SOMEWHERE]); + Intel::seen([$host=10.0.0.1, + $where=SOMEWHERE]); + } + +global log_lines = 0; +event Intel::log_intel(rec: Intel::Info) + { + ++log_lines; + if ( log_lines == 1 ) + terminate(); + } + +event bro_init() &priority=-10 + { + schedule 1sec { do_it() }; + } \ No newline at end of file From c301e1c9b444912517d9b57827da3c7575b13912 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Sun, 24 Mar 2019 22:16:13 +0100 Subject: [PATCH 3/5] Added policy script for intel removal. --- scripts/policy/frameworks/intel/removal.bro | 23 +++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 scripts/policy/frameworks/intel/removal.bro diff --git a/scripts/policy/frameworks/intel/removal.bro b/scripts/policy/frameworks/intel/removal.bro new file mode 100644 index 0000000000..cc4bb42921 --- /dev/null +++ b/scripts/policy/frameworks/intel/removal.bro @@ -0,0 +1,23 @@ +##! This script enables removal of intelligence items. + +@load base/frameworks/intel + +module Intel; + +export { + redef record Intel::MetaData += { + ## A boolean value to indicate whether the item should be removed. + remove: bool &default=F; + }; +} + +hook Intel::filter_item(item: Item) + { + if ( item$meta$remove ) + { + Intel::remove(item); + # Prevent readding + break; + } + } + From 2d9b90cddc49bc870b1c8b7a0cd46505585d1687 Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Sun, 24 Mar 2019 22:24:12 +0100 Subject: [PATCH 4/5] Added test for intel removal policy script. --- .../broproc.intel.log | 10 ++++ .../policy/frameworks/intel/removal.bro | 46 +++++++++++++++++++ 2 files changed, 56 insertions(+) create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log create mode 100644 testing/btest/scripts/policy/frameworks/intel/removal.bro diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log new file mode 100644 index 0000000000..d43abf187b --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.removal/broproc.intel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path intel +#open 2019-03-24-21-15-06 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc +#types time string addr port addr port string enum enum string set[enum] set[string] string string string +1553462106.131323 - - - - - 10.0.0.2 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - +#close 2019-03-24-21-15-06 diff --git a/testing/btest/scripts/policy/frameworks/intel/removal.bro b/testing/btest/scripts/policy/frameworks/intel/removal.bro new file mode 100644 index 0000000000..4d7e450da4 --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/intel/removal.bro @@ -0,0 +1,46 @@ + +# @TEST-EXEC: btest-bg-run broproc bro %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff broproc/intel.log + +@TEST-START-FILE intel.dat +#fields indicator indicator_type meta.source meta.remove +10.0.0.1 Intel::ADDR source1 T +10.0.0.2 Intel::ADDR source1 F +@TEST-END-FILE + +@load frameworks/intel/removal + +redef exit_only_after_terminate = T; +redef Intel::read_files += { "../intel.dat" }; +redef enum Intel::Where += { SOMEWHERE }; + +hook Intel::filter_item(item: Intel::Item) + { + if ( item$indicator_type == Intel::ADDR && + Site::is_local_addr(to_addr(item$indicator)) ) + break; + } + +event do_it() + { + Intel::seen([$host=10.0.0.1, + $where=SOMEWHERE]); + Intel::seen([$host=10.0.0.2, + $where=SOMEWHERE]); + } + +global log_lines = 0; +event Intel::log_intel(rec: Intel::Info) + { + ++log_lines; + if ( log_lines == 1 ) + terminate(); + } + +event bro_init() &priority=-10 + { + Intel::insert([$indicator="10.0.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); + Intel::insert([$indicator="10.0.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]); + schedule 1sec { do_it() }; + } \ No newline at end of file From a7b9c1d2584ce5e33365d3c8466e15339adbc33b Mon Sep 17 00:00:00 2001 From: Jan Grashoefer Date: Sun, 24 Mar 2019 22:45:24 +0100 Subject: [PATCH 5/5] Added new intel policy script to policy test. --- scripts/test-all-policy.bro | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index c3c2941b06..2e38738527 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -16,6 +16,7 @@ @load frameworks/intel/do_notice.bro @load frameworks/intel/do_expire.bro @load frameworks/intel/whitelist.bro +@load frameworks/intel/removal.bro @load frameworks/intel/seen/__load__.bro @load frameworks/intel/seen/conn-established.bro @load frameworks/intel/seen/dns.bro