Merge remote-tracking branch 'origin/topic/awelzel/intel-seen-hook'

* origin/topic/awelzel/intel-seen-hook:
  Intel: Introduce Intel::seen_policy() hook

CHANGES

@@ -1,3 +1,30 @@
+6.2.0-dev.470 | 2024-01-25 12:25:57 +0100
+
+  * GH-3256: Intel: Introduce Intel::seen_policy() hook (Arne Welzel, Corelight)
+
+    This introduces a new hook into the Intel::seen() function that allows
+    users to directly interact with the result of a find() call via external
+    scripts.
+
+    This should solve the use-case brought up by @chrisanag1985 in
+    discussion #3256: Recording and acting on "no intel match found".
+
+    @Canon88 was recently asking on Slack about enabling HTTP logging for a
+    given connection only when an Intel match occurred and found that the
+    Intel::match() event would only occur on the manager. The
+    Intel::match_remote() event might be a workaround, but possibly running a
+    bit too late and also it's just an internal "detail" event that might not
+    be stable.
+
+    Another internal use case revolved around enabling packet recording
+    based on Intel matches which necessarily needs to happen on the worker
+    where the match happened. The proposed workaround is similar to the above
+    using Intel::match_remote().
+
+    This hook also provides an opportunity to rate-limit heavy hitter intel
+    items locally on the worker nodes, or even replacing the event approach
+    currently used with a customized approach.
+
 6.2.0-dev.468 | 2024-01-25 12:21:24 +0100
 
   * websocket: Fix opcode for continuation frames (Arne Welzel, Corelight)

NEWS

@@ -182,6 +182,11 @@ New Functionality
   when encountering unknown/unsupported protocols. Its first use is to indicate
   ``Tunnel::max_depth`` being exceeded.
 
+- A new ``Intel::seen_policy`` hook has been introduced to allow intercepting
+  and changing ``Intel::seen` behavior:
+
+	hook Intel::seen_policy(s: Intel::Seen, found: bool)
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-6.2.0-dev.468
+6.2.0-dev.470

scripts/base/frameworks/intel/main.zeek

@@ -141,6 +141,8 @@ export {
 	##
 	## This is the primary mechanism where a user may take actions based on
 	## data provided by the intelligence framework.
+	##
+	## .. zeek::see:: Intel::seen_policy
 	global match: event(s: Seen, items: set[Item]);
 
 	## This hook can be used to influence the logging of intelligence hits
@@ -157,6 +159,27 @@ export {
 	## not be logged.
 	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
 
+	## Hook to modify and intercept :zeek:see:`Intel::seen` behavior.
+	##
+	## This hook is invoked after the Intel datastore was searched for
+	## a given :zeek:see:`Intel::Seen` instance. If a matching entry was
+	## found, the *found* argument is set to ``T``, else ``F``.
+	##
+	## Breaking from this hook suppresses :zeek:see:`Intel::match`
+	## event generation and any subsequent logging.
+	##
+	## Note that this hook only runs on the Zeek node where :zeek:seen:`Intel::seen`
+	## is invoked. In a cluster configuration that is usually on the worker nodes.
+	## This is in contrast to :zeek:see:`Intel::match` that usually runs
+	## centrally on the the manager node instead.
+	##
+	## s: The :zeek:see:`Intel::Seen` instance passed to the :zeek:see:`Intel::seen` function.
+	##
+	## found: ``T`` if Intel datastore contained *s*, else ``F``.
+	##
+	## .. zeek::see:: Intel::match
+	global seen_policy: hook(s: Seen, found: bool);
+
 	## The expiration timeout for intelligence items. Once an item expires, the
 	## :zeek:id:`Intel::item_expired` hook is called. Reinsertion of an item
 	## resets the timeout. A negative value disables expiration of intelligence
@@ -352,8 +375,14 @@ function get_items(s: Seen): set[Item]
 
 function Intel::seen(s: Seen)
 	{
-	if ( find(s) )
+	local found = find(s);
-		{
+
+	if ( ! hook Intel::seen_policy(s, found) )
+		return;
+
+	if ( ! found )
+		return;
+
 	if ( s?$host )
 		{
 		s$indicator = cat(s$host);
@@ -361,9 +390,7 @@ function Intel::seen(s: Seen)
 		}
 
 	if ( ! s?$node )
-			{
 		s$node = peer_description;
-			}
 
 	if ( have_full_data )
 		{
@@ -375,7 +402,6 @@ function Intel::seen(s: Seen)
 		event Intel::match_remote(s);
 		}
 	}
-	}
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{

testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output

@@ -7,6 +7,6 @@
 #open XXXX-XX-XX-XX-XX-XX
 #fields	ts	level	message	location
 #types	time	enum	string	string
-XXXXXXXXXX.XXXXXX	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	<...>/main.zeek, lines 570-571
+XXXXXXXXXX.XXXXXX	Reporter::INFO	Tried to remove non-existing item '192.168.1.1' (Intel::ADDR).	<...>/main.zeek, lines xxx-xxx
 XXXXXXXXXX.XXXXXX	Reporter::INFO	received termination signal	(empty)
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/intel.log

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+seen.indicator	seen.indicator_type	seen.where	seen.node
+example.com	Intel::DOMAIN	Intel::IN_ANYWHERE	zeek
+root	Intel::USER_NAME	Intel::IN_ANYWHERE	zeek

testing/btest/Baseline/scripts.base.frameworks.intel.seen-policy/output

@@ -0,0 +1,8 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::seen_policy, example.com, Intel::DOMAIN, found, T
+Intel::seen_policy, zeek.org, Intel::DOMAIN, found, T
+Intel::seen_policy, domain.de, Intel::DOMAIN, found, F
+Intel::seen_policy, nobody, Intel::USER_NAME, found, F
+Intel::seen_policy, root, Intel::USER_NAME, found, T
+Intel::match, example.com, Intel::DOMAIN
+Intel::match, root, Intel::USER_NAME

testing/btest/scripts/base/frameworks/intel/remove-non-existing.zeek

@@ -2,7 +2,7 @@
 # @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: cat zeekproc/reporter.log > output
 # @TEST-EXEC: cat zeekproc/.stdout >> output
-# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff output
+# @TEST-EXEC: TEST_DIFF_CANONIFIER='sed -E "s/lines [0-9]+-[0-9]+/lines xxx-xxx/g" | $SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps' btest-diff output
 
 # @TEST-START-FILE intel.dat
 #fields	indicator	indicator_type	meta.source	meta.desc	meta.url

testing/btest/scripts/base/frameworks/intel/seen-policy.zeek

@@ -0,0 +1,48 @@
+# @TEST-EXEC: btest-bg-run zeekproc zeek -b %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: cp zeekproc/.stdout output
+# @TEST-EXEC: zeek-cut -m seen.indicator seen.indicator_type seen.where seen.node < zeekproc/intel.log > intel.log
+# @TEST-EXEC: btest-diff intel.log
+# @TEST-EXEC: btest-diff output
+
+@load base/frameworks/intel
+
+redef exit_only_after_terminate = T;
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print "Intel::match", s$indicator, s$indicator_type;
+	}
+
+hook Intel::seen_policy(s: Intel::Seen, found: bool)
+	{
+	print "Intel::seen_policy", s$indicator, s$indicator_type, "found", found;
+
+	# No event generation for zeek.org
+	if ( s$indicator == "zeek.org" )
+		break;
+	}
+
+event seen_policy_test()
+	{
+	Intel::seen([$indicator="example.com", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]);
+	Intel::seen([$indicator="zeek.org", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]);
+	Intel::seen([$indicator="domain.de", $indicator_type=Intel::DOMAIN, $where=Intel::IN_ANYWHERE]);
+	Intel::seen([$indicator="nobody", $indicator_type=Intel::USER_NAME, $where=Intel::IN_ANYWHERE]);
+	Intel::seen([$indicator="root", $indicator_type=Intel::USER_NAME, $where=Intel::IN_ANYWHERE]);
+
+	terminate();
+	}
+
+event zeek_init()
+	{
+	local meta = Intel::MetaData($source="btest");
+	local i0 = Intel::Item($indicator="example.com", $indicator_type=Intel::DOMAIN, $meta=meta);
+	local i1 = Intel::Item($indicator="zeek.org", $indicator_type=Intel::DOMAIN, $meta=meta);
+	local i2 = Intel::Item($indicator="root", $indicator_type=Intel::USER_NAME, $meta=meta);
+	for ( _, i in vector(i0, i1, i2) )
+		Intel::insert(i);
+
+
+	event seen_policy_test();
+	}