From 62e0dc94db3f606f7ec542a5509ea047af17fdae Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Mon, 22 Jan 2024 13:17:07 +0100 Subject: [PATCH] Intel: Introduce Intel::seen_policy() hook This introduces a new hook into the Intel::seen() function that allows users to directly interact with the result of a find() call via external scripts. This should solve the use-case brought up by @chrisanag1985 in discussion #3256: Recording and acting on "no intel match found". @Canon88 was recently asking on Slack about enabling HTTP logging for a given connection only when an Intel match occurred and found that the Intel::match() event would only occur on the manager. The Intel::match_remote() event might be a workaround, but possibly running a bit too late and also it's just an internal "detail" event that might not be stable. Another internal use case revolved around enabling packet recording based on Intel matches which necessarily needs to happen on the worker where the match happened. The proposed workaround is similar to the above using Intel::match_remote(). This hook also provides an opportunity to rate-limit heavy hitter intel items locally on the worker nodes, or even replacing the event approach currently used with a customized approach. --- NEWS | 5 ++ scripts/base/frameworks/intel/main.zeek | 64 +++++++++++++------ .../output | 2 +- .../intel.log | 4 ++ .../output | 8 +++ .../frameworks/intel/remove-non-existing.zeek | 2 +- .../base/frameworks/intel/seen-policy.zeek | 48 ++++++++++++++ 7 files changed, 112 insertions(+), 21 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output create mode 100644 testing/btest/scripts/base/frameworks/intel/seen-policy.zeek diff --git a/NEWS b/NEWS index a2dd0177d1..8dac5d28ee 100644 --- a/NEWS +++ b/NEWS @@ -182,6 +182,11 @@ New Functionality when encountering unknown/unsupported protocols. Its first use is to indicate ``Tunnel::max_depth`` being exceeded. +- A new ``Intel::seen_policy`` hook has been introduced to allow intercepting + and changing ``Intel::seen` behavior: + + hook Intel::seen_policy(s: Intel::Seen, found: bool) + Changed Functionality --------------------- diff --git a/scripts/base/frameworks/intel/main.zeek b/scripts/base/frameworks/intel/main.zeek index 0c3a05becf..4bfab00546 100644 --- a/scripts/base/frameworks/intel/main.zeek +++ b/scripts/base/frameworks/intel/main.zeek @@ -141,6 +141,8 @@ export { ## ## This is the primary mechanism where a user may take actions based on ## data provided by the intelligence framework. + ## + ## .. zeek::see:: Intel::seen_policy global match: event(s: Seen, items: set[Item]); ## This hook can be used to influence the logging of intelligence hits @@ -157,6 +159,27 @@ export { ## not be logged. global extend_match: hook(info: Info, s: Seen, items: set[Item]); + ## Hook to modify and intercept :zeek:see:`Intel::seen` behavior. + ## + ## This hook is invoked after the Intel datastore was searched for + ## a given :zeek:see:`Intel::Seen` instance. If a matching entry was + ## found, the *found* argument is set to ``T``, else ``F``. + ## + ## Breaking from this hook suppresses :zeek:see:`Intel::match` + ## event generation and any subsequent logging. + ## + ## Note that this hook only runs on the Zeek node where :zeek:seen:`Intel::seen` + ## is invoked. In a cluster configuration that is usually on the worker nodes. + ## This is in contrast to :zeek:see:`Intel::match` that usually runs + ## centrally on the the manager node instead. + ## + ## s: The :zeek:see:`Intel::Seen` instance passed to the :zeek:see:`Intel::seen` function. + ## + ## found: ``T`` if Intel datastore contained *s*, else ``F``. + ## + ## .. zeek::see:: Intel::match + global seen_policy: hook(s: Seen, found: bool); + ## The expiration timeout for intelligence items. Once an item expires, the ## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item ## resets the timeout. A negative value disables expiration of intelligence @@ -352,28 +375,31 @@ function get_items(s: Seen): set[Item] function Intel::seen(s: Seen) { - if ( find(s) ) + local found = find(s); + + if ( ! hook Intel::seen_policy(s, found) ) + return; + + if ( ! found ) + return; + + if ( s?$host ) { - if ( s?$host ) - { - s$indicator = cat(s$host); - s$indicator_type = Intel::ADDR; - } + s$indicator = cat(s$host); + s$indicator_type = Intel::ADDR; + } - if ( ! s?$node ) - { - s$node = peer_description; - } + if ( ! s?$node ) + s$node = peer_description; - if ( have_full_data ) - { - local items = get_items(s); - event Intel::match(s, items); - } - else - { - event Intel::match_remote(s); - } + if ( have_full_data ) + { + local items = get_items(s); + event Intel::match(s, items); + } + else + { + event Intel::match_remote(s); } } diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output index 76d724f57b..78fbf4818f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output @@ -7,6 +7,6 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts level message location #types time enum string string -XXXXXXXXXX.XXXXXX Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). <...>/main.zeek, lines 570-571 +XXXXXXXXXX.XXXXXX Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). <...>/main.zeek, lines xxx-xxx XXXXXXXXXX.XXXXXX Reporter::INFO received termination signal (empty) #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log new file mode 100644 index 0000000000..b90a6acfd9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +seen.indicator seen.indicator_type seen.where seen.node +example.com Intel::DOMAIN Intel::IN_ANYWHERE zeek +root Intel::USER_NAME Intel::IN_ANYWHERE zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output new file mode 100644 index 0000000000..687275a83d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::seen_policy, example.com, Intel::DOMAIN, found, T +Intel::seen_policy, zeek.org, Intel::DOMAIN, found, T +Intel::seen_policy, domain.de, Intel::DOMAIN, found, F +Intel::seen_policy, nobody, Intel::USER_NAME, found, F +Intel::seen_policy, root, Intel::USER_NAME, found, T +Intel::match, example.com, Intel::DOMAIN +Intel::match, root, Intel::USER_NAME diff --git a/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek b/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek index 216e40e835..134316b66d 100644 --- a/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek +++ b/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-bg-wait 30 # @TEST-EXEC: cat zeekproc/reporter.log > output # @TEST-EXEC: cat zeekproc/.stdout >> output -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER='sed -E "s/lines [0-9]+-[0-9]+/lines xxx-xxx/g" | $SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps' btest-diff output # @TEST-START-FILE intel.dat #fields indicator indicator_type meta.source meta.desc meta.url diff --git a/testing/btest/scripts/base/frameworks/intel/seen-policy.zeek b/testing/btest/scripts/base/frameworks/intel/seen-policy.zeek new file mode 100644 index 0000000000..dd56cb1f33 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/seen-policy.zeek @@ -0,0 +1,48 @@ +# @TEST-EXEC: btest-bg-run zeekproc zeek -b %INPUT +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: cp zeekproc/.stdout output +# @TEST-EXEC: zeek-cut -m seen.indicator seen.indicator_type seen.where seen.node < zeekproc/intel.log > intel.log +# @TEST-EXEC: btest-diff intel.log +# @TEST-EXEC: btest-diff output + +@load base/frameworks/intel + +redef exit_only_after_terminate = T; + +event Intel::match(s: Intel::Seen, items: set[Intel::Item]) + { + print "Intel::match", s$indicator, s$indicator_type; + } + +hook Intel::seen_policy(s: Intel::Seen, found: bool) + { + print "Intel::seen_policy", s$indicator, s$indicator_type, "found", found; + + # No event generation for zeek.org + if ( s$indicator == "zeek.org" ) + break; + } + +event seen_policy_test() + { + Intel::seen([$indicator="example.com", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="zeek.org", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="domain.de", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="nobody", $indicator_type=Intel::USER_NAME, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="root", $indicator_type=Intel::USER_NAME, $where=Intel::IN_ANYWHERE]); + + terminate(); + } + +event zeek_init() + { + local meta = Intel::MetaData($source="btest"); + local i0 = Intel::Item($indicator="example.com", $indicator_type=Intel::DOMAIN, $meta=meta); + local i1 = Intel::Item($indicator="zeek.org", $indicator_type=Intel::DOMAIN, $meta=meta); + local i2 = Intel::Item($indicator="root", $indicator_type=Intel::USER_NAME, $meta=meta); + for ( _, i in vector(i0, i1, i2) ) + Intel::insert(i); + + + event seen_policy_test(); + }