diff --git a/CHANGES b/CHANGES index 376d237f39..3e2384c79b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,30 @@ +6.2.0-dev.470 | 2024-01-25 12:25:57 +0100 + + * GH-3256: Intel: Introduce Intel::seen_policy() hook (Arne Welzel, Corelight) + + This introduces a new hook into the Intel::seen() function that allows + users to directly interact with the result of a find() call via external + scripts. + + This should solve the use-case brought up by @chrisanag1985 in + discussion #3256: Recording and acting on "no intel match found". + + @Canon88 was recently asking on Slack about enabling HTTP logging for a + given connection only when an Intel match occurred and found that the + Intel::match() event would only occur on the manager. The + Intel::match_remote() event might be a workaround, but possibly running a + bit too late and also it's just an internal "detail" event that might not + be stable. + + Another internal use case revolved around enabling packet recording + based on Intel matches which necessarily needs to happen on the worker + where the match happened. The proposed workaround is similar to the above + using Intel::match_remote(). + + This hook also provides an opportunity to rate-limit heavy hitter intel + items locally on the worker nodes, or even replacing the event approach + currently used with a customized approach. + 6.2.0-dev.468 | 2024-01-25 12:21:24 +0100 * websocket: Fix opcode for continuation frames (Arne Welzel, Corelight) diff --git a/NEWS b/NEWS index a2dd0177d1..8dac5d28ee 100644 --- a/NEWS +++ b/NEWS @@ -182,6 +182,11 @@ New Functionality when encountering unknown/unsupported protocols. Its first use is to indicate ``Tunnel::max_depth`` being exceeded. +- A new ``Intel::seen_policy`` hook has been introduced to allow intercepting + and changing ``Intel::seen` behavior: + + hook Intel::seen_policy(s: Intel::Seen, found: bool) + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 974f78b7fe..e507950d68 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.2.0-dev.468 +6.2.0-dev.470 diff --git a/scripts/base/frameworks/intel/main.zeek b/scripts/base/frameworks/intel/main.zeek index 0c3a05becf..4bfab00546 100644 --- a/scripts/base/frameworks/intel/main.zeek +++ b/scripts/base/frameworks/intel/main.zeek @@ -141,6 +141,8 @@ export { ## ## This is the primary mechanism where a user may take actions based on ## data provided by the intelligence framework. + ## + ## .. zeek::see:: Intel::seen_policy global match: event(s: Seen, items: set[Item]); ## This hook can be used to influence the logging of intelligence hits @@ -157,6 +159,27 @@ export { ## not be logged. global extend_match: hook(info: Info, s: Seen, items: set[Item]); + ## Hook to modify and intercept :zeek:see:`Intel::seen` behavior. + ## + ## This hook is invoked after the Intel datastore was searched for + ## a given :zeek:see:`Intel::Seen` instance. If a matching entry was + ## found, the *found* argument is set to ``T``, else ``F``. + ## + ## Breaking from this hook suppresses :zeek:see:`Intel::match` + ## event generation and any subsequent logging. + ## + ## Note that this hook only runs on the Zeek node where :zeek:seen:`Intel::seen` + ## is invoked. In a cluster configuration that is usually on the worker nodes. + ## This is in contrast to :zeek:see:`Intel::match` that usually runs + ## centrally on the the manager node instead. + ## + ## s: The :zeek:see:`Intel::Seen` instance passed to the :zeek:see:`Intel::seen` function. + ## + ## found: ``T`` if Intel datastore contained *s*, else ``F``. + ## + ## .. zeek::see:: Intel::match + global seen_policy: hook(s: Seen, found: bool); + ## The expiration timeout for intelligence items. Once an item expires, the ## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item ## resets the timeout. A negative value disables expiration of intelligence @@ -352,28 +375,31 @@ function get_items(s: Seen): set[Item] function Intel::seen(s: Seen) { - if ( find(s) ) + local found = find(s); + + if ( ! hook Intel::seen_policy(s, found) ) + return; + + if ( ! found ) + return; + + if ( s?$host ) { - if ( s?$host ) - { - s$indicator = cat(s$host); - s$indicator_type = Intel::ADDR; - } + s$indicator = cat(s$host); + s$indicator_type = Intel::ADDR; + } - if ( ! s?$node ) - { - s$node = peer_description; - } + if ( ! s?$node ) + s$node = peer_description; - if ( have_full_data ) - { - local items = get_items(s); - event Intel::match(s, items); - } - else - { - event Intel::match_remote(s); - } + if ( have_full_data ) + { + local items = get_items(s); + event Intel::match(s, items); + } + else + { + event Intel::match_remote(s); } } diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output index 76d724f57b..78fbf4818f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output @@ -7,6 +7,6 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts level message location #types time enum string string -XXXXXXXXXX.XXXXXX Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). <...>/main.zeek, lines 570-571 +XXXXXXXXXX.XXXXXX Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). <...>/main.zeek, lines xxx-xxx XXXXXXXXXX.XXXXXX Reporter::INFO received termination signal (empty) #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log new file mode 100644 index 0000000000..b90a6acfd9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +seen.indicator seen.indicator_type seen.where seen.node +example.com Intel::DOMAIN Intel::IN_ANYWHERE zeek +root Intel::USER_NAME Intel::IN_ANYWHERE zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output new file mode 100644 index 0000000000..687275a83d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output @@ -0,0 +1,8 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Intel::seen_policy, example.com, Intel::DOMAIN, found, T +Intel::seen_policy, zeek.org, Intel::DOMAIN, found, T +Intel::seen_policy, domain.de, Intel::DOMAIN, found, F +Intel::seen_policy, nobody, Intel::USER_NAME, found, F +Intel::seen_policy, root, Intel::USER_NAME, found, T +Intel::match, example.com, Intel::DOMAIN +Intel::match, root, Intel::USER_NAME diff --git a/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek b/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek index 216e40e835..134316b66d 100644 --- a/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek +++ b/testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek @@ -2,7 +2,7 @@ # @TEST-EXEC: btest-bg-wait 30 # @TEST-EXEC: cat zeekproc/reporter.log > output # @TEST-EXEC: cat zeekproc/.stdout >> output -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER='sed -E "s/lines [0-9]+-[0-9]+/lines xxx-xxx/g" | $SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps' btest-diff output # @TEST-START-FILE intel.dat #fields indicator indicator_type meta.source meta.desc meta.url diff --git a/testing/btest/scripts/base/frameworks/intel/seen-policy.zeek b/testing/btest/scripts/base/frameworks/intel/seen-policy.zeek new file mode 100644 index 0000000000..dd56cb1f33 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/intel/seen-policy.zeek @@ -0,0 +1,48 @@ +# @TEST-EXEC: btest-bg-run zeekproc zeek -b %INPUT +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: cp zeekproc/.stdout output +# @TEST-EXEC: zeek-cut -m seen.indicator seen.indicator_type seen.where seen.node < zeekproc/intel.log > intel.log +# @TEST-EXEC: btest-diff intel.log +# @TEST-EXEC: btest-diff output + +@load base/frameworks/intel + +redef exit_only_after_terminate = T; + +event Intel::match(s: Intel::Seen, items: set[Intel::Item]) + { + print "Intel::match", s$indicator, s$indicator_type; + } + +hook Intel::seen_policy(s: Intel::Seen, found: bool) + { + print "Intel::seen_policy", s$indicator, s$indicator_type, "found", found; + + # No event generation for zeek.org + if ( s$indicator == "zeek.org" ) + break; + } + +event seen_policy_test() + { + Intel::seen([$indicator="example.com", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="zeek.org", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="domain.de", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="nobody", $indicator_type=Intel::USER_NAME, $where=Intel::IN_ANYWHERE]); + Intel::seen([$indicator="root", $indicator_type=Intel::USER_NAME, $where=Intel::IN_ANYWHERE]); + + terminate(); + } + +event zeek_init() + { + local meta = Intel::MetaData($source="btest"); + local i0 = Intel::Item($indicator="example.com", $indicator_type=Intel::DOMAIN, $meta=meta); + local i1 = Intel::Item($indicator="zeek.org", $indicator_type=Intel::DOMAIN, $meta=meta); + local i2 = Intel::Item($indicator="root", $indicator_type=Intel::USER_NAME, $meta=meta); + for ( _, i in vector(i0, i1, i2) ) + Intel::insert(i); + + + event seen_policy_test(); + }