Merge remote-tracking branch 'origin/topic/awelzel/intel-indicator-hooks'

* origin/topic/awelzel/intel-indicator-hooks:
  intel/seen/manage-event-groups: Policy script for toggling intel event groups
  intel: Add indicator_inserted and indicator_removed hooks

CHANGES

@@ -1,3 +1,29 @@
+8.0.0-dev.303 | 2025-06-02 09:51:53 +0200
+
+  * intel/seen/manage-event-groups: Policy script for toggling intel event groups (Arne Welzel, Corelight)
+
+    Co-authored-by: Mohan Dhawan <mohan@corelight.com>
+
+  * intel: Add indicator_inserted and indicator_removed hooks (Arne Welzel, Corelight)
+
+    This change adds two new hooks to the Intel framework that can be used
+    to intercept added and removed indicators and their type.
+
+    These hooks are fairly low-level. One immediate use-case is to count the
+    number of indicators loaded per Intel::Type and enable and disable the
+    corresponding event groups of the intel/seen scripts.
+
+    I attempted to gauge the overhead and while it's definitely there, loading
+    a file with ~500k DOMAIN entries takes somewhere around ~0.5 seconds hooks
+    when populated via the min_data_store store mechanism. While that
+    doesn't sound great, it actually takes the manager on my system 2.5
+    seconds to serialize and Cluster::publish() the min_data_store alone
+    and its doing that serially for every active worker. Mostly to say that
+    the bigger overhead in that area on the manager doing redundant work
+    per worker.
+
+    Co-authored-by: Mohan Dhawan <mohan@corelight.com>
+
 8.0.0-dev.299 | 2025-05-30 13:06:42 -0700
 
   * fix for crash when interpreting transformed ASTs that include multi-field record assignments/additions (Vern Paxson, Corelight)

NEWS

@@ -66,6 +66,34 @@ New Functionality
 
 - The FTP analyzer now supports explicit TLS via AUTH TLS.
 
+- Two new script-level hooks in the Intel framework have been added.
+
+	hook indicator_inserted(indicator_value: string, indicator_type: Intel::Type)
+
+	hook indicator_removed(indicator_value: string, indicator_type: Intel::Type)
+
+  These are reliably invoked on worker and manager nodes the first time an
+  indicator value is inserted into the store and once it has been completely
+  removed from the store.
+
+- The ``frameworks/intel/seen`` scripts have been annotated with event groups
+  and a new ``frameworks/intel/seen/manage-event-groups`` policy script added.
+
+  The motivation is to allow Zeek distributors to load the ``intel/seen`` scripts
+  by default without incurring their event overhead when no Intel indicators are
+  loaded. Corresponding event handlers are enabled once the first Intel indicator
+  of a given ``Intel::Type`` is added. Event handlers are disabled when the last
+  indicator is removed, again.
+
+  Note that the ``manage-event-groups`` script interacts with the ``Intel::seen_policy``
+  hook: If no indicators for a given ``Intel::Type`` are loaded, the ``Intel::seen_policy``
+  will not be invoked as the event handlers extracting indicators aren't executed.
+
+  If you rely on the ``Intel::seen_policy`` hook to be invoked regardless of the
+  contents of the Intel store, do not load the ``manage-event-groups`` or set:
+
+	redef Intel::manage_seen_event_groups = F;
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-8.0.0-dev.299
+8.0.0-dev.303

scripts/base/frameworks/intel/cluster.zeek

@@ -105,10 +105,30 @@ event Intel::insert_indicator(item: Intel::Item) &priority=5
 	Intel::_insert(item, F);
 	}
 
+function invoke_indicator_hook(store: MinDataStore, h: hook(v: string, t: Intel::Type))
+	{
+	for ( a in store$host_data )
+		hook h(cat(a), Intel::ADDR);
+
+	for ( sn in store$subnet_data)
+		hook h(cat(sn), Intel::SUBNET);
+
+	for ( [indicator_value, indicator_type] in store$string_data )
+		hook h(indicator_value, indicator_type);
+	}
+
 # Handling of a complete MinDataStore snapshot
+#
+# Invoke the removed and inserted hooks using the old and new min data store
+# instances, respectively. The way this event is used, the original
+# min_data_store should essentially be empty.
 event new_min_data_store(store: MinDataStore)
 	{
+	invoke_indicator_hook(min_data_store, Intel::indicator_removed);
+
 	min_data_store = store;
+
+	invoke_indicator_hook(min_data_store, Intel::indicator_inserted);
 	}
 @endif
 

scripts/base/frameworks/intel/main.zeek

@@ -207,6 +207,35 @@ export {
 	## item: The intel item that should be inserted.
 	global filter_item: hook(item: Intel::Item);
 
+	## This hook is invoked when a new indicator has been inserted into
+	## the min data store for the first time.
+	##
+	## Calls to :zeek:see:`Intel::seen` with a matching indicator value
+	## and type will result in matches.
+	##
+	## Subsequent inserts of the same indicator type and value do not
+	## invoke this hook. Breaking from this hook has no effect.
+	##
+	## indicator: The indicator value.
+	##
+	## indicator_type: The indicator type.
+	##
+	## .. zeek::see:: Intel::indicator_removed
+	global indicator_inserted: hook(indicator: string, indiator_type: Type);
+
+	## This hook is invoked when an indicator has been removed from
+	## the min data store.
+	##
+	## After this hooks runs, :zeek:see:`Intel::seen` for the indicator
+	## will not return any matches. Breaking from this hook has no effect.
+	##
+	## indicator: The indicator value.
+	##
+	## indicator_type: The indicator type.
+	##
+	## .. zeek::see:: Intel::indicator_inserted
+	global indicator_removed: hook(indicator: string, indiator_type: Type);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -507,18 +536,44 @@ function _insert(item: Item, first_dispatch: bool &default = T)
 	# All intelligence is case insensitive at the moment.
 	local lower_indicator = to_lower(item$indicator);
 
+	# Track if the indicator was inserted into the min_data_store.
+	# It's tempting to just use is_new above, but it seems that only works
+	# correctly on a worker if the manager never spuriously sends a
+	# Intel::insert_item(), so better to determine this locally based
+	# on the actual contents of the min_data_store.
+	local inserted = F;
+	local inserted_value = "";
+
 	# Insert indicator into MinDataStore (might exist already).
 	switch ( item$indicator_type )
 		{
 		case ADDR:
 			local host = to_addr(item$indicator);
+			if ( host !in min_data_store$host_data )
+				{
+				inserted = T;
+				inserted_value = cat(host);
+				}
+
 			add min_data_store$host_data[host];
 			break;
 		case SUBNET:
 			local net = to_subnet(item$indicator);
+			if ( net !in min_data_store$subnet_data )
+				{
+				inserted = T;
+				inserted_value = cat(net);
+				}
+
 			add min_data_store$subnet_data[net];
 			break;
 		default:
+			if ( [lower_indicator, item$indicator_type] !in min_data_store$string_data )
+				{
+				inserted = T;
+				inserted_value = lower_indicator;
+				}
+
 			add min_data_store$string_data[lower_indicator, item$indicator_type];
 			break;
 		}
@@ -533,6 +588,9 @@ function _insert(item: Item, first_dispatch: bool &default = T)
 		# Announce a (possibly) new item if this is the first dispatch and
 		# we know it is new or have to assume that on a worker.
 		event Intel::new_item(item);
+
+	if ( inserted )
+		hook Intel::indicator_inserted(inserted_value, item$indicator_type);
 	}
 
 function insert(item: Item)
@@ -632,18 +690,43 @@ function remove(item: Item, purge_indicator: bool)
 # Handling of indicator removal in minimal data stores.
 event remove_indicator(item: Item)
 	{
+	local removed = F;
+	local removed_value = "";
+
 	switch ( item$indicator_type )
 		{
 		case ADDR:
 			local host = to_addr(item$indicator);
+			if ( host in min_data_store$host_data )
+				{
+				removed = T;
+				removed_value = cat(host);
+				}
+
 			delete min_data_store$host_data[host];
 			break;
 		case SUBNET:
 			local net = to_subnet(item$indicator);
+			if ( net in min_data_store$subnet_data )
+				{
+				removed = T;
+				removed_value = cat(net);
+				}
+
 			delete min_data_store$subnet_data[net];
 			break;
 		default:
-			delete min_data_store$string_data[to_lower(item$indicator), item$indicator_type];
+			local indicator_value = to_lower(item$indicator);
+			if ( [indicator_value, item$indicator_type] in min_data_store$string_data )
+				{
+				removed = T;
+				removed_value = indicator_value;
+				}
+
+			delete min_data_store$string_data[indicator_value, item$indicator_type];
 			break;
 		}
+
+	if ( removed )
+		hook Intel::indicator_removed(removed_value, item$indicator_type);
 	}

scripts/policy/frameworks/intel/seen/manage-event-groups.zeek

@@ -0,0 +1,74 @@
+@load frameworks/intel/seen
+@load base/frameworks/reporter
+
+module Intel;
+
+export {
+	## Whether Intel event groups for the seen scripts are managed.
+	##
+	## When loading this script, by default, all :zeek:see:`Intel::Type`
+	## event groups are disabled at startup and only enabled when indicators
+	## of corresponding types are loaded into the Intel framework's store.
+	## This allows to load the ``frameworks/intel/seen`` scripts without
+	## incurring event handling overhead when no Intel indicators are loaded.
+	##
+	## One caveat is that the :zeek:see:`Intel::seen_policy` hook will not
+	## be invoked for indicator types that are not at all in the Intel
+	## framework's store. If you rely on :zeek:see:`Intel::seen_policy` to
+	## find unmatched indicators, do not not load this script, set this
+	## variable to ``F``, or insert dummy values of the types using
+	## :zeek:see:`Intel::insert`.
+	const manage_seen_event_groups = T &redef;
+}
+
+global intel_type_counts: table[Intel::Type] of count &default=0;
+
+event zeek_init()
+	{
+	# If the feature is disabled, don't act.
+	if ( ! manage_seen_event_groups )
+		return;
+
+	# Disable all Intel related event groups at startup. These
+	# are enabled again as soon as at least one indicator of the
+	# type is inserted.
+	for ( name in enum_names(Intel::Type) )
+		{
+		if ( has_event_group(name) )
+			disable_event_group(name);
+		}
+	}
+
+hook Intel::indicator_inserted(v: string, t: Intel::Type)
+	{
+	++intel_type_counts[t];
+
+	if ( ! manage_seen_event_groups )
+		return;
+
+
+	if ( intel_type_counts[t] == 1 )
+		{
+		local name = cat(t);
+
+		if ( has_event_group(name) )
+			enable_event_group(name);
+		}
+	}
+
+hook Intel::indicator_removed(v: string, t: Intel::Type)
+	{
+	--intel_type_counts[t];
+
+	if ( ! manage_seen_event_groups )
+		return;
+
+
+	if ( intel_type_counts[t] == 0 )
+		{
+		local name = cat(t);
+
+		if ( has_event_group(name) )
+			disable_event_group(name);
+		}
+	}

scripts/test-all-policy.zeek

@@ -59,6 +59,7 @@
 @load frameworks/intel/seen/file-names.zeek
 @load frameworks/intel/seen/http-headers.zeek
 @load frameworks/intel/seen/http-url.zeek
+@load frameworks/intel/seen/manage-event-groups.zeek
 @load frameworks/intel/seen/pubkey-hashes.zeek
 @load frameworks/intel/seen/smb-filenames.zeek
 @load frameworks/intel/seen/smtp-url-extraction.zeek

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stderr

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/manager..stdout

@@ -0,0 +1,6 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
+Intel::indicator_inserted, putty, Intel::SOFTWARE
+Intel::indicator_inserted, putty2, Intel::SOFTWARE

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-1..stdout

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
+Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
+Intel::insert_indicator, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::insert_indicator, 1.2.3.5, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET
+Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
+Intel::insert_indicator, putty, Intel::SOFTWARE
+Intel::indicator_inserted, putty, Intel::SOFTWARE
+Intel::insert_indicator, putty2, Intel::SOFTWARE
+Intel::indicator_inserted, putty2, Intel::SOFTWARE

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-1-run-2..stdout

@@ -0,0 +1,8 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
+Intel::indicator_inserted, putty, Intel::SOFTWARE
+Intel::indicator_inserted, putty2, Intel::SOFTWARE
+Intel::new_min_data_store post, [host_data={\x0a\x091.2.3.5,\x0a\x091.2.3.4\x0a}, subnet_data={\x0a\x09192.168.0.0/16\x0a}, string_data={\x0a\x09[putty, Intel::SOFTWARE] ,\x0a\x09[putty2, Intel::SOFTWARE] \x0a}]

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted-new-min-store/worker-2..stdout

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::new_min_data_store pre, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
+Intel::new_min_data_store post, [host_data={\x0a\x0a}, subnet_data={\x0a\x0a}, string_data={\x0a\x0a}]
+Intel::insert_indicator, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::insert_indicator, 1.2.3.5, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+Intel::insert_indicator, 192.168.0.0/16, Intel::SUBNET
+Intel::indicator_inserted, 192.168.0.0/16, Intel::SUBNET
+Intel::insert_indicator, putty, Intel::SOFTWARE
+Intel::indicator_inserted, putty, Intel::SOFTWARE
+Intel::insert_indicator, putty2, Intel::SOFTWARE
+Intel::indicator_inserted, putty2, Intel::SOFTWARE

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.manager..stdout

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+insert_addr, 1.2.3.4, from-manager
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+Intel::indicator_removed, 1.2.3.4, Intel::ADDR
+Intel::indicator_removed, 1.2.3.5, Intel::ADDR
+====
+insert_addr, 1.2.3.4, from-manager
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+Intel::indicator_removed, 1.2.3.4, Intel::ADDR
+Intel::indicator_removed, 1.2.3.5, Intel::ADDR
+publish_do_terminate()

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-1..stdout

@@ -0,0 +1,16 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+insert_addr, 1.2.3.4, from-worker
+insert_addr, 1.2.3.5, from-worker
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+insert_addr, 1.2.3.5, from-manager
+Intel::indicator_removed, 1.2.3.4, Intel::ADDR
+Intel::indicator_removed, 1.2.3.5, Intel::ADDR
+====
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+insert_addr, 1.2.3.4, from-worker
+insert_addr, 1.2.3.5, from-worker
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+insert_addr, 1.2.3.5, from-manager
+Intel::indicator_removed, 1.2.3.4, Intel::ADDR
+Intel::indicator_removed, 1.2.3.5, Intel::ADDR

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/addr-indicator.worker-2..stdout

@@ -0,0 +1,18 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+remove_addr, 1.2.3.4, from-manager
+remove_addr, 1.2.3.5, from-manager
+remove_addr, 1.2.3.4, from-worker
+remove_addr, 1.2.3.5, from-worker
+Intel::indicator_removed, 1.2.3.4, Intel::ADDR
+Intel::indicator_removed, 1.2.3.5, Intel::ADDR
+====
+Intel::indicator_inserted, 1.2.3.4, Intel::ADDR
+Intel::indicator_inserted, 1.2.3.5, Intel::ADDR
+remove_addr, 1.2.3.4, from-manager
+remove_addr, 1.2.3.5, from-manager
+remove_addr, 1.2.3.4, from-worker
+remove_addr, 1.2.3.5, from-worker
+Intel::indicator_removed, 1.2.3.4, Intel::ADDR
+Intel::indicator_removed, 1.2.3.5, Intel::ADDR

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.manager..stdout

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+insert_software, software-1.2.3.4, from-manager
+Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
+====
+insert_software, software-1.2.3.4, from-manager
+Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
+publish_do_terminate()

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-1..stdout

@@ -0,0 +1,16 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
+insert_software, software-1.2.3.4, from-worker
+insert_software, software-1.2.3.5, from-worker
+Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
+insert_software, software-1.2.3.5, from-manager
+Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
+====
+Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
+insert_software, software-1.2.3.4, from-worker
+insert_software, software-1.2.3.5, from-worker
+Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
+insert_software, software-1.2.3.5, from-manager
+Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/software-indicator.worker-2..stdout

@@ -0,0 +1,18 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
+remove_software, software-1.2.3.4, from-manager
+remove_software, software-1.2.3.5, from-manager
+remove_software, software-1.2.3.4, from-worker
+remove_software, software-1.2.3.5, from-worker
+Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE
+====
+Intel::indicator_inserted, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_inserted, software-1.2.3.5, Intel::SOFTWARE
+remove_software, software-1.2.3.4, from-manager
+remove_software, software-1.2.3.5, from-manager
+remove_software, software-1.2.3.4, from-worker
+remove_software, software-1.2.3.5, from-worker
+Intel::indicator_removed, software-1.2.3.4, Intel::SOFTWARE
+Intel::indicator_removed, software-1.2.3.5, Intel::SOFTWARE

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.manager..stdout

@@ -0,0 +1,13 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+insert_subnet, 1.2.3.4/32, from-manager
+Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
+====
+insert_subnet, 1.2.3.4/32, from-manager
+Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
+publish_do_terminate()

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-1..stdout

@@ -0,0 +1,16 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
+insert_subnet, 1.2.3.4/32, from-worker
+insert_subnet, 1.2.3.5/32, from-worker
+Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
+insert_subnet, 1.2.3.5/32, from-manager
+Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
+====
+Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
+insert_subnet, 1.2.3.4/32, from-worker
+insert_subnet, 1.2.3.5/32, from-worker
+Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
+insert_subnet, 1.2.3.5/32, from-manager
+Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-indicator-inserted/subnet-indicator.worker-2..stdout

@@ -0,0 +1,18 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
+remove_subnet, 1.2.3.4/32, from-manager
+remove_subnet, 1.2.3.5/32, from-manager
+remove_subnet, 1.2.3.4/32, from-worker
+remove_subnet, 1.2.3.5/32, from-worker
+Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET
+====
+Intel::indicator_inserted, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_inserted, 1.2.3.5/32, Intel::SUBNET
+remove_subnet, 1.2.3.4/32, from-manager
+remove_subnet, 1.2.3.5/32, from-manager
+remove_subnet, 1.2.3.4/32, from-worker
+remove_subnet, 1.2.3.5/32, from-worker
+Intel::indicator_removed, 1.2.3.4/32, Intel::SUBNET
+Intel::indicator_removed, 1.2.3.5/32, Intel::SUBNET

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager..stdout

@@ -0,0 +1,10 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+running do_step(2)
+Intel::indicator_inserted 1.2.3.4 Intel::ADDR
+publish do_step(3) to zeek/cluster/worker
+Intel::match_remote: 1.2.3.4 Intel::ADDR
+Intel::match: 1.2.3.4 Intel::ADDR
+running do_step(4)
+Intel::indicator_removed 1.2.3.4 Intel::ADDR
+publish do_step(5) to zeek/cluster/worker
+running do_step(6)

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/manager.intel.log

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open XXXX-XX-XX-XX-XX-XX
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+XXXXXXXXXX.XXXXXX	CHhAvVGS1DHFjwGM9	1.1.1.1	1	2.2.2.2	2	1.2.3.4	Intel::ADDR	SMTP::IN_RECEIVED_HEADER	worker-1	Intel::ADDR	source1	-	-	-
+#close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.manage-event-groups.basic-smtp/worker-1..stdout

@@ -0,0 +1,11 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+running do_step(1)
+publish do_step(2) to zeek/cluster/manager
+Intel::indicator_inserted 1.2.3.4 Intel::ADDR
+running do_step(3)
+Intel::seen_policy(1.2.3.4 of Intel::ADDR, T)
+Intel::seen_policy(1.2.3.5 of Intel::ADDR, F)
+Intel::match_remote: 1.2.3.4 Intel::ADDR
+Intel::indicator_removed 1.2.3.4 Intel::ADDR
+running do_step(5)
+publish do_step(6) to zeek/cluster/manager

testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted-new-min-store.zeek

@@ -0,0 +1,121 @@
+# @TEST-DOC: Verify behavior of Intel:indicator_inserted() and Intel::indicator_removed() when a worker node restarts.
+#
+# @TEST-PORT: BROKER_MANAGER_PORT
+# @TEST-PORT: BROKER_WORKER1_PORT
+# @TEST-PORT: BROKER_WORKER2_PORT
+#
+# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
+#
+# @TEST-EXEC: zeek --parse-only %INPUT
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT
+# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
+# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b %INPUT
+# @TEST-EXEC: wait-for-file worker-1/DONE 30
+# @TEST-EXEC: mv worker-1 worker-1-run-1
+# @TEST-EXEC: rm -rf worker-1
+# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
+# @TEST-EXEC: wait-for-file worker-1/DONE 30
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: mv worker-1 worker-1-run-2
+
+# @TEST-EXEC: btest-diff manager/.stdout
+# @TEST-EXEC: btest-diff manager/.stderr
+# @TEST-EXEC: btest-diff worker-1-run-1/.stdout
+# @TEST-EXEC: btest-diff worker-1-run-2/.stdout
+# @TEST-EXEC: btest-diff worker-2/.stdout
+#
+@load policy/frameworks/cluster/experimental
+@load base/frameworks/intel
+
+# Ttoal of 5 unique indicators
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.4	Intel::ADDR	source1	this host is bad	http://some-data-distributor.com/1
+1.2.3.4	Intel::ADDR	source2	this host is bad	http://some-data-distributor.com/1
+1.2.3.5	Intel::ADDR	source1	this host is bad	http://some-data-distributor.com/1
+192.168.0.0/16	Intel::SUBNET	source1	this network is bad	http://some-data-distributor.com/2
+putty	Intel::SOFTWARE	source1	this software is bad	http://some-data-distributor.com/2
+putty	Intel::SOFTWARE	source2	this software is bad	http://some-data-distributor.com/2
+putty2	Intel::SOFTWARE	source3	this software is bad	http://some-data-distributor.com/2
+# @TEST-END-FILE
+
+redef Log::default_rotation_interval = 0sec;
+
+event Cluster::Experimental::cluster_started()
+	{
+	# Start reading the intel file on the manager once all workers are up.
+	if ( Cluster::node == "manager" )
+		{
+		local source = "../intel.dat";
+		Input::add_event([$source=source,
+		                  $reader=Input::READER_ASCII,
+		                  $mode=Input::REREAD,
+		                  $name=cat("intel-", source),
+		                  $fields=Intel::Item,
+		                  $ev=Intel::read_entry,
+		                  $error_ev=Intel::read_error]);
+		}
+	}
+
+# Send by manager for termination purposes.
+event do_terminate()
+	{
+	terminate();
+	}
+
+global worker1_down = 0;
+global nodes_down = 0;
+
+event Cluster::node_down(name: string, id: string)
+	{
+	++nodes_down;
+	if ( name == "worker-1")
+		++worker1_down;
+
+	if ( worker1_down == 2 )
+		Cluster::publish(Cluster::worker_topic, do_terminate);
+
+	if ( nodes_down >= 3 )
+		terminate();
+	}
+
+global total_indicators = 0;
+
+hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
+	{
+	print "Intel::indicator_inserted", indicator, indicator_type;
+	++total_indicators;
+
+	# Once worker-1 has seen all the 5 indicators, write a DONE file and terminate()
+	if ( Cluster::node == "worker-1" && total_indicators == 5 )
+		{
+		if ( ! piped_exec("touch DONE", "") )
+			exit(1);
+
+		terminate();
+		}
+	}
+
+hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
+	{
+	print "Intel::indicator_removed", indicator, indicator_type;
+	}
+
+module Intel;
+
+# Internal events for easier grasping behavior.
+
+event Intel::insert_indicator(item: Intel::Item) &priority=10
+	{
+	print "Intel::insert_indicator", item$indicator, item$indicator_type;
+	}
+
+event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=10
+	{
+	print "Intel::new_min_data_store pre", cat(Intel::min_data_store);
+	}
+
+event Intel::new_min_data_store(store: Intel::MinDataStore) &priority=-10
+	{
+	print "Intel::new_min_data_store post", cat(Intel::min_data_store);
+	}

testing/btest/scripts/base/frameworks/intel/cluster-indicator-inserted.zeek

@@ -0,0 +1,193 @@
+# @TEST-DOC: Verify Intel::indicator_inserted() and Intel::indicator_removed() in a cluster setup with three different types of indicators.
+#
+# @TEST-PORT: BROKER_MANAGER_PORT
+# @TEST-PORT: BROKER_WORKER1_PORT
+# @TEST-PORT: BROKER_WORKER2_PORT
+#
+# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
+#
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../addr-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../addr-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../addr-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: mkdir addr-indicator/; cp -R manager worker-1 worker-2 ./addr-indicator/
+# @TEST-EXEC: btest-diff addr-indicator/manager/.stdout
+# @TEST-EXEC: btest-diff addr-indicator/worker-1/.stdout
+# @TEST-EXEC: btest-diff addr-indicator/worker-2/.stdout
+#
+# @TEST-EXEC: rm -rf manager worker-1 worker-2
+#
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../subnet-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../subnet-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../subnet-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: mkdir subnet-indicator/; cp -R manager worker-1 worker-2 ./subnet-indicator/
+# @TEST-EXEC: btest-diff subnet-indicator/manager/.stdout
+# @TEST-EXEC: btest-diff subnet-indicator/worker-1/.stdout
+# @TEST-EXEC: btest-diff subnet-indicator/worker-2/.stdout
+
+# @TEST-EXEC: rm -rf manager worker-1 worker-2
+#
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b ../software-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b ../software-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-run worker-2 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b ../software-indicator.zeek %INPUT
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: mkdir software-indicator/; cp -R manager worker-1 worker-2 ./software-indicator/
+# @TEST-EXEC: btest-diff software-indicator/manager/.stdout
+# @TEST-EXEC: btest-diff software-indicator/worker-1/.stdout
+# @TEST-EXEC: btest-diff software-indicator/worker-2/.stdout
+#
+# @TEST-EXEC: rm -rf manager worker-1 worker-2
+
+@load policy/frameworks/cluster/experimental
+@load base/frameworks/intel
+
+redef Log::default_rotation_interval = 0sec;
+
+# @TEST-START-FILE addr-indicator.zeek
+@load base/frameworks/intel
+
+const test_intel_type = Intel::ADDR;
+
+event remove_indicator(value: string, source: string)
+	{
+	print"remove_addr", value, source;
+	Intel::remove([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]], F);
+	}
+
+event insert_indicator(value: string, source: string)
+	{
+	print "insert_addr", value, source;
+	Intel::insert([$indicator=value, $indicator_type=Intel::ADDR, $meta=[$source=source]]);
+	}
+# @TEST-END-FILE
+
+# @TEST-START-FILE subnet-indicator.zeek
+@load base/frameworks/intel
+
+const test_intel_type = Intel::SUBNET;
+
+event remove_indicator(value: string, source: string)
+	{
+	value = value + "/32";  # make the IP value from generic code a valid subnet
+	print"remove_subnet", value, source;
+	Intel::remove([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]], F);
+	}
+
+event insert_indicator(value: string, source: string)
+	{
+	value = value + "/32";  # make the IP value from generic code a valid subnet
+	print "insert_subnet", value , source;
+	Intel::insert([$indicator=value, $indicator_type=Intel::SUBNET, $meta=[$source=source]]);
+	}
+# @TEST-END-FILE
+
+# @TEST-START-FILE software-indicator.zeek
+@load base/frameworks/intel
+
+const test_intel_type = Intel::SOFTWARE;
+
+event remove_indicator(value: string, source: string)
+	{
+	value = "software-" + value;
+	print"remove_software", value, source;
+	Intel::remove([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]], F);
+	}
+
+event insert_indicator(value: string, source: string)
+	{
+	value = "software-" + value;
+	print "insert_software", value , source;
+	Intel::insert([$indicator=value, $indicator_type=Intel::SOFTWARE, $meta=[$source=source]]);
+	}
+# @TEST-END-FILE
+
+# Helper event for printing on manager and worker.
+event next_round()
+	{
+	print "====";
+	if ( Cluster::node == "manager" )
+		Cluster::publish(Cluster::worker_topic, next_round);
+	}
+
+# Send by manager for termination purposes.
+event do_terminate()
+	{
+	terminate();
+	}
+
+event publish_do_terminate()
+	{
+	print "publish_do_terminate()";
+	Cluster::publish(Cluster::worker_topic, do_terminate);
+	}
+
+# Ensure the manager terminates eventually.
+global nodes_down = 0;
+event Cluster::node_down(name: string, id: string)
+	{
+	++nodes_down;
+	if ( nodes_down >= 2)
+		terminate();
+	}
+
+# Gets the ball rolling
+event Cluster::Experimental::cluster_started()
+	{
+	if ( Cluster::local_node_type() == Cluster::MANAGER )
+		event insert_indicator("1.2.3.4", "from-manager");
+	}
+
+global indicators_inserted: table[Intel::Type] of count &default=0;
+global indicator_removed: table[Intel::Type] of count &default=0;
+
+hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
+	{
+	print "Intel::indicator_inserted", indicator, indicator_type;
+	++indicators_inserted[indicator_type];
+
+	# If worker-1 sees the first addr indicator (1.2.3.4), it re-inserts
+	# it with a different source (from-worker) and also inserts a second
+	# indicatore 1.2.3.5 with with sources "from-worker" and "from-manager";
+	if ( Cluster::node == "worker-1" )
+		{
+		if ( indicators_inserted[test_intel_type] == 1 || indicators_inserted[test_intel_type] == 3 )
+			{
+			event insert_indicator("1.2.3.4", "from-worker");
+			event insert_indicator("1.2.3.5", "from-worker");
+			event insert_indicator("1.2.3.5", "from-manager");
+			}
+		}
+
+	# Once worker-2 has observed two or four indicators, it removes
+	# all of them again!
+	if ( Cluster::node == "worker-2" )
+		{
+		if ( indicators_inserted[test_intel_type] == 2 || indicators_inserted[test_intel_type] == 4 )
+			{
+			event remove_indicator("1.2.3.4", "from-manager");
+			event remove_indicator("1.2.3.5", "from-manager");
+			event remove_indicator("1.2.3.4", "from-worker");
+			event remove_indicator("1.2.3.5", "from-worker");
+			}
+		}
+	}
+
+hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
+	{
+	print "Intel::indicator_removed", indicator, indicator_type;
+	++indicator_removed[indicator_type];
+	if ( Cluster::node == "manager" )
+		{
+		if ( indicator_removed[test_intel_type] == 2 )
+			{
+			# Trigger another round of inserts at the workers!
+			event next_round();
+			event insert_indicator("1.2.3.4", "from-manager");
+			}
+		else if ( indicator_removed[test_intel_type] == 4 )
+			{
+			event publish_do_terminate();
+			}
+		}
+	}

testing/btest/scripts/policy/frameworks/intel/seen/manage-event-groups/basic-smtp.zeek

@@ -0,0 +1,155 @@
+# @TEST-DOC: Smoke test that the seen/smtp mime_end_entity() only runs when Intel::ADDR indicators are loaded and mime_end_entity() runs for a SMTP connection.
+#
+# @TEST-PORT: BROKER_MANAGER_PORT
+# @TEST-PORT: BROKER_WORKER1_PORT
+#
+# @TEST-EXEC: cp $FILES/broker/cluster-layout.zeek .
+#
+# @TEST-EXEC: zeek --parse-only %INPUT
+#
+# @TEST-EXEC: btest-bg-run manager ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager zeek -b %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
+# @TEST-EXEC: btest-bg-wait 30
+
+# @TEST-EXEC: btest-diff manager/.stdout
+# @TEST-EXEC: btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff manager/intel.log
+
+@load base/frameworks/cluster
+@load base/frameworks/intel
+
+@load frameworks/intel/seen
+@load frameworks/intel/seen/manage-event-groups
+
+@load frameworks/cluster/experimental
+
+redef Log::default_rotation_interval = 0secs;
+
+global addr_indicator = Intel::Item(
+	$indicator="1.2.3.4",
+	$indicator_type=Intel::ADDR,
+	$meta=Intel::MetaData($source="source1"),
+);
+
+function make_conn(): connection
+	{
+	local c = connection(
+			$id = conn_id($orig_h=1.1.1.1, $orig_p=1/tcp,
+			              $resp_h=2.2.2.2, $resp_p=2/tcp, $proto=6),
+			$orig = endpoint($size=1, $state=4, $flow_label=0),
+			$resp = endpoint($size=1, $state=4, $flow_label=0),
+			$start_time=double_to_time(1747323991.0),
+			$duration=1sec,
+			$service=set("smtp"),
+			$history="ShAdDa",
+			$uid="CHhAvVGS1DHFjwGM9",
+	);
+
+	c$smtp = SMTP::Info(
+		$ts=c$start_time,
+		$uid=c$uid,
+		$id=c$id,
+		$trans_depth=1,
+		$path=vector(1.2.3.4)
+	);
+
+	c$smtp$x_originating_ip = 1.2.3.5;
+
+	return c;
+	}
+
+global current_step = 0;
+
+global do_step: event(step: count);
+
+event publish_do_step(step: count)
+	{
+	local topic = Cluster::local_node_type() == Cluster::MANAGER ? Cluster::worker_topic : Cluster::manager_topic;
+	print fmt("publish do_step(%s) to %s", step, topic);
+	Cluster::publish(topic, do_step, step);
+	}
+
+# Log Intel::seen_policy() invocations.
+#
+# The idea here is that if the seen event groups are disabled,
+# the Intel::seen_policy() hook isn't invoked at all. So we can
+# use that to verify that the corresponding events have actually
+# been disabled.
+hook Intel::seen_policy(s: Intel::Seen, found: bool)
+	{
+	print fmt("Intel::seen_policy(%s of %s, %s)",
+	          s?$host ? cat(s$host) : s$indicator,
+	          s?$host ? "Intel::ADDR" : cat(s$indicator_type),
+	          found);
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print fmt("Intel::match: %s %s", s$indicator, s$indicator_type);
+	if ( current_step == 2 )
+		event do_step(4);
+	}
+
+event do_step(step: count)
+	{
+	current_step = step;
+	print fmt("running do_step(%s)", step);
+
+	switch ( step ) {
+	case 1:  # worker
+		local c1 = make_conn();
+		event mime_end_entity(c1);
+		event publish_do_step(2);
+		break;
+	case 2:  # manager, insert a intel indicator
+		Intel::insert(addr_indicator);
+		event publish_do_step(3);
+		break;
+	case 3:  # worker - should have an addr indicator now, match it.
+		local c2 = make_conn();
+		event mime_end_entity(c2);
+		# no publish of step 4, see Intel::match() that drives it
+		break;
+	case 4:  # manager waits for the match
+		Intel::remove(addr_indicator);
+		event publish_do_step(5);
+		break;
+	case 5:  # worker - the ADDR groups are disabled again.
+		local c3 = make_conn();
+		event mime_end_entity(c3);
+		event publish_do_step(6);
+		break;
+	case 6: # manager, done
+		terminate();
+		break;
+	}
+	}
+
+event Cluster::Experimental::cluster_started()
+	{
+	if ( Cluster::node == "worker-1" )
+		event do_step(1);
+	}
+
+event Cluster::node_down(name: string, id: string)
+	{
+	terminate();
+	}
+
+# Output a few internal things for sanity. These aren't testing functionality,
+# but nice to have.
+module Intel;
+event Intel::match_remote(s: Intel::Seen)
+	{
+	print fmt("Intel::match_remote: %s %s", s$indicator, s$indicator_type);
+	}
+
+hook Intel::indicator_inserted(indicator: string, indicator_type: Intel::Type)
+	{
+	print fmt("Intel::indicator_inserted %s %s", indicator, indicator_type);
+	}
+
+hook Intel::indicator_removed(indicator: string, indicator_type: Intel::Type)
+	{
+	print fmt("Intel::indicator_removed %s %s", indicator, indicator_type);
+	}

testing/external/scripts/testing-setup.zeek

@@ -17,6 +17,11 @@ redef DPD::track_removed_services_in_connection=T;
 	redef LogAscii::use_json = F;
 @endif
 
+# The tests don't load intel data and so all Intel event groups are disabled
+# due to intel/seen/manage-event-groups being loaded by default. Disable that
+# functionality by default to cover execution in the intel/seen scripts.
+redef Intel::manage_seen_event_groups = F;
+
 # The IMAP analyzer includes absolute filenames in its error messages,
 # exclude it for now from analyzer.log.
 # https://github.com/zeek/zeek/issues/2659