intel/seen/manage-event-groups: Policy script for toggling intel event groups

Co-authored-by: Mohan Dhawan <mohan@corelight.com>

NEWS

@@ -76,6 +76,24 @@ New Functionality
   indicator value is inserted into the store and once it has been completely
   removed from the store.
 
+- The ``frameworks/intel/seen`` scripts have been annotated with event groups
+  and a new ``frameworks/intel/seen/manage-event-groups`` policy script added.
+
+  The motivation is to allow Zeek distributors to load the ``intel/seen`` scripts
+  by default without incurring their event overhead when no Intel indicators are
+  loaded. Corresponding event handlers are enabled once the first Intel indicator
+  of a given ``Intel::Type`` is added. Event handlers are disabled when the last
+  indicator is removed, again.
+
+  Note that the ``manage-event-groups`` script interacts with the ``Intel::seen_policy``
+  hook: If no indicators for a given ``Intel::Type`` are loaded, the ``Intel::seen_policy``
+  will not be invoked as the event handlers extracting indicators aren't executed.
+
+  If you rely on the ``Intel::seen_policy`` hook to be invoked regardless of the
+  contents of the Intel store, do not load the ``manage-event-groups`` or set:
+
+	redef Intel::manage_seen_event_groups = F;
+
 Changed Functionality
 ---------------------
 

scripts/policy/frameworks/intel/seen/manage-event-groups.zeek

@@ -0,0 +1,74 @@
+@load frameworks/intel/seen
+@load base/frameworks/reporter
+
+module Intel;
+
+export {
+	## Whether Intel event groups for the seen scripts are managed.
+	##
+	## When loading this script, by default, all :zeek:see:`Intel::Type`
+	## event groups are disabled at startup and only enabled when indicators
+	## of corresponding types are loaded into the Intel framework's store.
+	## This allows to load the ``frameworks/intel/seen`` scripts without
+	## incurring event handling overhead when no Intel indicators are loaded.
+	##
+	## One caveat is that the :zeek:see:`Intel::seen_policy` hook will not
+	## be invoked for indicator types that are not at all in the Intel
+	## framework's store. If you rely on :zeek:see:`Intel::seen_policy` to
+	## find unmatched indicators, do not not load this script, set this
+	## variable to ``F``, or insert dummy values of the types using
+	## :zeek:see:`Intel::insert`.
+	const manage_seen_event_groups = T &redef;
+}
+
+global intel_type_counts: table[Intel::Type] of count &default=0;
+
+event zeek_init()
+	{
+	# If the feature is disabled, don't act.
+	if ( ! manage_seen_event_groups )
+		return;
+
+	# Disable all Intel related event groups at startup. These
+	# are enabled again as soon as at least one indicator of the
+	# type is inserted.
+	for ( name in enum_names(Intel::Type) )
+		{
+		if ( has_event_group(name) )
+			disable_event_group(name);
+		}
+	}
+
+hook Intel::indicator_inserted(v: string, t: Intel::Type)
+	{
+	++intel_type_counts[t];
+
+	if ( ! manage_seen_event_groups )
+		return;
+
+
+	if ( intel_type_counts[t] == 1 )
+		{
+		local name = cat(t);
+
+		if ( has_event_group(name) )
+			enable_event_group(name);
+		}
+	}
+
+hook Intel::indicator_removed(v: string, t: Intel::Type)
+	{
+	--intel_type_counts[t];
+
+	if ( ! manage_seen_event_groups )
+		return;
+
+
+	if ( intel_type_counts[t] == 0 )
+		{
+		local name = cat(t);
+
+		if ( has_event_group(name) )
+			disable_event_group(name);
+		}
+	}

scripts/test-all-policy.zeek

@@ -59,6 +59,7 @@
 @load frameworks/intel/seen/file-names.zeek
 @load frameworks/intel/seen/http-headers.zeek
 @load frameworks/intel/seen/http-url.zeek
+@load frameworks/intel/seen/manage-event-groups.zeek
 @load frameworks/intel/seen/pubkey-hashes.zeek
 @load frameworks/intel/seen/smb-filenames.zeek
 @load frameworks/intel/seen/smtp-url-extraction.zeek

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout

@@ -0,0 +1,10 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+running do_step(2)
+Intel::indicator_inserted 1.2.3.4 Intel::ADDR
+publish do_step(3) to zeek/cluster/worker
+Intel::match_remote: 1.2.3.4 Intel::ADDR
+Intel::match: 1.2.3.4 Intel::ADDR
+running do_step(4)
+Intel::indicator_removed 1.2.3.4 Intel::ADDR
+publish do_step(5) to zeek/cluster/worker
+running do_step(6)

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	1.1.1.1	1	2.2.2.2	2	1.2.3.4	Intel::ADDR	SMTP::IN_RECEIVED_HEADER	worker-1	Intel::ADDR	source1	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+running do_step(1)
+publish do_step(2) to zeek/cluster/manager
+Intel::indicator_inserted 1.2.3.4 Intel::ADDR
+running do_step(3)
+Intel::seen_policy(1.2.3.4 of Intel::ADDR, T)
+Intel::seen_policy(1.2.3.5 of Intel::ADDR, F)
+Intel::match_remote: 1.2.3.4 Intel::ADDR
+Intel::indicator_removed 1.2.3.4 Intel::ADDR
+running do_step(5)
+publish do_step(6) to zeek/cluster/manager

testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek

@@ -0,0 +1,155 @@
+# @TEST-DOC: Smoke test that the seen/smtp mime_end_entity() only runs when Intel::ADDR indicators are loaded and mime_end_entity() runs for a SMTP connection.
+#
+# @TEST-PORT: BROKER_MANAGER_PORT
+# @TEST-PORT: BROKER_WORKER1_PORT
+#
+# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
+#
+# @TEST-EXEC: zeek --parse-only %INPUT
+#
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
+# @TEST-EXEC: btest-bg-wait 30
+
+# @TEST-EXEC: btest-diff manager/.stdout
+# @TEST-EXEC: btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff manager/intel.log
+
+@load base/frameworks/cluster
+@load base/frameworks/intel
+
+@load frameworks/intel/seen
+@load frameworks/intel/seen/manage-event-groups
+
+@load frameworks/cluster/experimental
+
+redef Log::default_rotation_interval = 0secs;
+
+global addr_indicator = Intel::Item(
+	$indicator="1.2.3.4",
+	$indicator_type=Intel::ADDR,
+	$meta=Intel::MetaData($source="source1"),
+);
+
+function make_conn(): connection
+	{
+	local c = connection(
+			$id = conn_id($orig_h=1.1.1.1, $orig_p=1/tcp,
+			              $resp_h=2.2.2.2, $resp_p=2/tcp, $proto=6),
+			$orig = endpoint($size=1, $state=4, $flow_label=0),
+			$resp = endpoint($size=1, $state=4, $flow_label=0),
+			$start_time=double_to_time(1747323991.0),
+			$duration=1sec,
+			$service=set("smtp"),
+			$history="ShAdDa",
+			$uid="CHhAvVGS1DHFjwGM9",
+	);
+
+	c$smtp = SMTP::Info(
+		$ts=c$start_time,
+		$uid=c$uid,
+		$id=c$id,
+		$trans_depth=1,
+		$path=vector(1.2.3.4)
+	);
+
+	c$smtp$x_originating_ip = 1.2.3.5;
+
+	return c;
+	}
+
+global current_step = 0;
+
+global do_step: event(step: count);
+
+event publish_do_step(step: count)
+	{
+	local topic = Cluster::local_node_type() == Cluster::MANAGER ? Cluster::worker_topic : Cluster::manager_topic;
+	print fmt("publish do_step(%s) to %s", step, topic);
+	Cluster::publish(topic, do_step, step);
+	}
+
+# Log Intel::seen_policy() invocations.
+#
+# The idea here is that if the seen event groups are disabled,
+# the Intel::seen_policy() hook isn't invoked at all. So we can
+# use that to verify that the corresponding events have actually
+# been disabled.
+hook Intel::seen_policy(s: Intel::Seen, found: bool)
+	{
+	print fmt("Intel::seen_policy(%s of %s, %s)",
+	          s?$host ? cat(s$host) : s$indicator,
+	          s?$host ? "Intel::ADDR" : cat(s$indicator_type),
+	          found);
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print fmt("Intel::match: %s %s", s$indicator, s$indicator_type);
+	if ( current_step == 2 )
+		event do_step(4);
+	}
+
+event do_step(step: count)
+	{
+	current_step = step;
+	print fmt("running do_step(%s)", step);
+
+	switch ( step ) {
+	case 1:  # worker
+		local c1 = make_conn();
+		event mime_end_entity(c1);
+		event publish_do_step(2);
+		break;
+	case 2:  # manager, insert a intel indicator
+		Intel::insert(addr_indicator);
+		event publish_do_step(3);
+		break;
+	case 3:  # worker - should have an addr indicator now, match it.
+		local c2 = make_conn();
+		event mime_end_entity(c2);
+		# no publish of step 4, see Intel::match() that drives it
+		break;
+	case 4:  # manager waits for the match
+		Intel::remove(addr_indicator);
+		event publish_do_step(5);
+		break;
+	case 5:  # worker - the ADDR groups are disabled again.
+		local c3 = make_conn();
+		event mime_end_entity(c3);
+		event publish_do_step(6);
+		break;
+	case 6: # manager, done
+		terminate();
+		break;
+	}
+	}
+
+event Cluster::Experimental::cluster_started()
+	{
+	if ( Cluster::node == "worker-1" )
+		event do_step(1);
+	}
+
+event Cluster::node_down(name: string, id: string)
+	{
+	terminate();
+	}
+
+# Output a few internal things for sanity. These aren't testing functionality,
+# but nice to have.
+module Intel;
+event Intel::match_remote(s: Intel::Seen)
+	{
+	print fmt("Intel::match_remote: %s %s", s$indicator, s$indicator_type);
+	}
+
+hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
+	{
+	print fmt("Intel::indicator_inserted %s %s", indicator, indicator_type);
+	}
+
+hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
+	{
+	print fmt("Intel::indicator_removed %s %s", indicator, indicator_type);
+	}

testing/external/scripts/testing-setup.zeek

@@ -17,6 +17,11 @@ redef DPD::track_removed_services_in_connection=T;
 	redef LogAscii::use_json = F;
 @endif
 
+# The tests don't load intel data and so all Intel event groups are disabled
+# due to intel/seen/manage-event-groups being loaded by default. Disable that
+# functionality by default to cover execution in the intel/seen scripts.
+redef Intel::manage_seen_event_groups = F;
+
 # The IMAP analyzer includes absolute filenames in its error messages,
 # exclude it for now from analyzer.log.
 # https://github.com/zeek/zeek/issues/2659