Merge branch 'topic/jgras/intel-update' of https://github.com/J-Gras/bro into topic/seth/intel-update-merge

# Conflicts:
#	testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
#	testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log

scripts/base/frameworks/intel/__load__.bro

@@ -1,5 +1,8 @@
 @load ./main
 
+# File analysis framework integration.
+@load ./files
+
 # The cluster framework must be loaded first.
 @load base/frameworks/cluster
 

scripts/base/frameworks/intel/cluster.bro

@@ -1,8 +1,8 @@
 ##! Cluster transparency support for the intelligence framework.  This is mostly
 ##! oriented toward distributing intelligence information across clusters.
 
+@load ./main
 @load base/frameworks/cluster
-@load ./input
 
 module Intel;
 
@@ -17,19 +17,17 @@ redef record Item += {
 redef have_full_data = F;
 @endif
 
+# Internal event for cluster data distribution.
 global cluster_new_item: event(item: Item);
 
-# Primary intelligence distribution comes from manager.
+# Primary intelligence management is done by the manager:
-redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
+# The manager informs the workers about new items and item removal.
-# If a worker finds intelligence and adds it, it should share it back to the manager.
+redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/;
-redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
+# A worker queries the manager to insert, remove or indicate the match of an item.
+redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
-event Intel::match_no_items(s: Seen) &priority=5
+# Handling of new worker nodes.
-	{
-	event Intel::match(s, Intel::get_items(s));
-	}
-
 event remote_connection_handshake_done(p: event_peer)
 	{
 	# When a worker connects, send it the complete minimal data store.
@@ -39,15 +37,22 @@ event remote_connection_handshake_done(p: event_peer)
 		send_id(p, "Intel::min_data_store");
 		}
 	}
-@endif
 
-event Intel::cluster_new_item(item: Intel::Item) &priority=5
+# Handling of matches triggered by worker nodes.
+event Intel::match_no_items(s: Seen) &priority=5
 	{
-	# Ignore locally generated events to avoid event storms.
+	if ( Intel::find(s) )
-	if ( is_remote_event() )
+		event Intel::match(s, Intel::get_items(s));
-		Intel::insert(item);
 	}
 
+# Handling of item removal triggered by worker nodes.
+event Intel::remove_item(item: Item, purge_indicator: bool)
+	{
+	remove(item, purge_indicator);
+	}
+@endif
+
+# Handling of item insertion.
 event Intel::new_item(item: Intel::Item) &priority=5
 	{
 	# The cluster manager always rebroadcasts intelligence.
@@ -59,3 +64,11 @@ event Intel::new_item(item: Intel::Item) &priority=5
 		event Intel::cluster_new_item(item);
 		}
 	}
+
+# Handling of item insertion by remote node.
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}

scripts/base/frameworks/intel/files.bro

@@ -0,0 +1,84 @@
+##! File analysis framework integration for the intelligence framework. This
+##! script manages file information in intelligence framework datastructures.
+
+@load ./main
+
+module Intel;
+
+export {
+	## Enum type to represent various types of intelligence data.
+	redef enum Type += {
+		## File hash which is non-hash type specific.  It's up to the
+		## user to query for any relevant hash types.
+		FILE_HASH,
+		## File name.  Typically with protocols with definite
+		## indications of a file name.
+		FILE_NAME,
+	};
+
+	## Information about a piece of "seen" data.
+	redef record Seen += {
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:              fa_file  &optional;
+		## If the data was discovered within a file, the file uid should
+		## go here to provide context to the data. If the file record *f*
+		## is provided, this will be automatically filled out.
+		fuid:           string   &optional;
+	};
+
+	## Record used for the logging framework representing a positive
+	## hit within the intelligence framework.
+	redef record Info += {
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string   &log &optional;
+		## A mime type if the intelligence hit is related to a file.
+		## If the $f field is provided this will be automatically filled
+		## out.
+		file_mime_type: string   &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically
+		## filled out.
+		file_desc:      string   &log &optional;
+	};
+}
+
+# Add file information to matches if available.
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+	{
+	if ( s?$f )
+		{
+		s$fuid = s$f$id;
+
+		if ( s$f?$conns && |s$f$conns| == 1 )
+			{
+			for ( cid in s$f$conns )
+				s$conn = s$f$conns[cid];
+			}
+
+		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
+			info$file_mime_type = s$f$info$mime_type;
+
+		if ( ! info?$file_desc )
+			info$file_desc = Files::describe(s$f);
+		}
+
+	if ( s?$fuid )
+		info$fuid = s$fuid;
+
+	if ( s?$conn )
+		{
+		s$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	if ( s?$uid )
+		info$uid = s$uid;
+
+	for ( item in items )
+		{
+		add info$sources[item$meta$source];
+		add info$matched[item$indicator_type];
+		}
+	}

scripts/base/frameworks/intel/input.bro

@@ -1,11 +1,14 @@
+##! Input handling for the intelligence framework. This script implements the
+##! import of intelligence data from files using the input framework.
+
 @load ./main
 
 module Intel;
 
 export {
-	## Intelligence files that will be read off disk.  The files are
+	## Intelligence files that will be read off disk. The files are
-	## reread every time they are updated so updates must be atomic with
+	## reread every time they are updated so updates must be atomic
-	## "mv" instead of writing the file in place.
+	## with "mv" instead of writing the file in place.
 	const read_files: set[string] = {} &redef;
 }
 

scripts/base/frameworks/intel/main.bro

@@ -1,7 +1,6 @@
-##! The intelligence framework provides a way to store and query IP addresses,
+##! The intelligence framework provides a way to store and query intelligence data
-##! and strings (with a str_type).  Metadata can
+##! (e.g. IP addresses, URLs and hashes). The intelligence items can be associated
-##! also be associated with the intelligence, like for making more informed
+##! with metadata to allow informed decisions about matching and handling.
-##! decisions about matching and handling of intelligence.
 
 @load base/frameworks/notice
 
@@ -14,6 +13,8 @@ export {
 	type Type: enum {
 		## An IP address.
 		ADDR,
+		## A subnet in CIDR notation.
+		SUBNET,
 		## A complete URL without the prefix ``"http://"``.
 		URL,
 		## Software name.
@@ -24,24 +25,20 @@ export {
 		DOMAIN,
 		## A user name.
 		USER_NAME,
-		## File hash which is non-hash type specific.  It's up to the
-		## user to query for any relevant hash types.
-		FILE_HASH,
-		## File name.  Typically with protocols with definite
-		## indications of a file name.
-		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
 		## Public key MD5 hash. (SSH server host keys are a good example.)
 		PUBKEY_HASH,
 	};
-	
+
+	## Set of intelligence data types.
+	type TypeSet: set[Type];
+
 	## Data about an :bro:type:`Intel::Item`.
 	type MetaData: record {
-		## An arbitrary string value representing the data source.
+		## An arbitrary string value representing the data source. This
-		## Typically, the convention for this field will be the source
+		## value is used as unique key to identify a metadata record in
-		## name and feed name separated by a hyphen.
+		## the scope of a single intelligence item.
-		## For example: "source1-c&c".
 		source:      string;
 		## A freeform description for the data.
 		desc:        string      &optional;
@@ -57,7 +54,7 @@ export {
 		## The type of data that the indicator field represents.
 		indicator_type: Type;
 		
-		## Metadata for the item.  Typically represents more deeply
+		## Metadata for the item. Typically represents more deeply
 		## descriptive data for a piece of intelligence.
 		meta:           MetaData;
 	};
@@ -96,15 +93,6 @@ export {
 		## If the *conn* field is provided, this will be automatically
 		## filled out.
 		uid:             string        &optional;
-
-		## If the data was discovered within a file, the file record
-		## should go here to provide context to the data.
-		f:               fa_file       &optional;
-
-		## If the data was discovered within a file, the file uid should
-		## go here to provide context to the data. If the *f* field is
-		## provided, this will be automatically filled out.
-		fuid:            string        &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -120,41 +108,69 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
-		## If a file was associated with this intelligence hit,
-		## this is the uid for the file.
-		fuid:           string   &log &optional;
-
-		## A mime type if the intelligence hit is related to a file.
-		## If the $f field is provided this will be automatically filled
-		## out.
-		file_mime_type: string   &log &optional;
-		## Frequently files can be "described" to give a bit more context.
-		## If the $f field is provided this field will be automatically
-		## filled out.
-		file_desc:      string   &log &optional;
-
 		## Where the data was seen.
 		seen:     Seen           &log;
+		## Which indicator types matched.
+		matched:  TypeSet        &log;
 		## Sources which supplied data that resulted in this match.
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation function.
+	## Function to insert intelligence data. If the indicator is already
+	## present, the associated metadata will be added to the indicator. If
+	## the indicator already contains a metadata record from the same source,
+	## the existing metadata record will be updated.
 	global insert: function(item: Item);
 
+	## Function to remove intelligence data. If purge_indicator is set, the
+	## given metadata is ignored and the indicator is removed completely.
+	global remove: function(item: Item, purge_indicator: bool &default = F);
+
 	## Function to declare discovery of a piece of data in order to check
 	## it against known intelligence for matches.
 	global seen: function(s: Seen);
 
 	## Event to represent a match in the intelligence data from data that
-	## was seen.  On clusters there is no assurance as to where this event
+	## was seen. On clusters there is no assurance as to when this event
 	## will be generated so do not assume that arbitrary global state beyond
 	## the given data will be available.
 	##
-	## This is the primary mechanism where a user will take actions based on
+	## This is the primary mechanism where a user may take actions based on
-	## data within the intelligence framework.
+	## data provided by the intelligence framework.
 	global match: event(s: Seen, items: set[Item]);
 
+	## This hook can be used to influence the logging of intelligence hits
+	## (e.g. by adding data to the Info record). The default information is
+	## added with a priority of 5.
+	##
+	## info: The Info record that will be logged.
+	##
+	## s: Information about the data seen.
+	##
+	## items: The intel items that match the seen data.
+	##
+	## In case the hook execution is terminated using break, the match will
+	## not be logged.
+	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
+
+	## The expiration timeout for intelligence items. Once an item expires, the
+	## :bro:id:`item_expired` hook is called. Reinsertion of an item resets the
+	## timeout. A negative value disables expiration of intelligence items.
+	const item_expiration = -1 min &redef;
+
+	## This hook can be used to handle expiration of intelligence items.
+	##
+	## indicator: The indicator of the expired item.
+	##
+	## indicator_type: The indicator type of the expired item.
+	##
+	## metas: The set of metadata describing the expired item.
+	##
+	## If all hook handlers are executed, the expiration timeout will be reset.
+	## Otherwise, if one of the handlers terminates using break, the item will
+	## be removed.
+	global item_expired: hook(indicator: string, indicator_type: Type, metas: set[MetaData]);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -163,16 +179,26 @@ global match_no_items: event(s: Seen);
 
 # Internal events for cluster data distribution.
 global new_item: event(item: Item);
-global updated_item: event(item: Item);
+global remove_item: event(item: Item, purge_indicator: bool);
+global purge_item: event(item: Item);
 
 # Optionally store metadata.  This is used internally depending on
 # if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
+# Table of metadata, indexed by source string.
+type MetaDataTable: table[string] of MetaData;
+
+# Expiration handlers.
+global expire_host_data: function(data: table[addr] of MetaDataTable, idx: addr): interval;
+global expire_subnet_data: function(data: table[subnet] of MetaDataTable, idx: subnet): interval;
+global expire_string_data: function(data: table[string, Type] of MetaDataTable, idx: any): interval;
+
 # The in memory data structure for holding intelligence.
 type DataStore: record {
-	host_data:    table[addr] of set[MetaData];
+	host_data:    table[addr] of MetaDataTable &write_expire=item_expiration &expire_func=expire_host_data;
-	string_data:  table[string, Type] of set[MetaData];
+	subnet_data:  table[subnet] of MetaDataTable &write_expire=item_expiration &expire_func=expire_subnet_data;
+	string_data:  table[string, Type] of MetaDataTable &write_expire=item_expiration &expire_func=expire_string_data;
 };
 global data_store: DataStore &redef;
 
@@ -181,6 +207,7 @@ global data_store: DataStore &redef;
 # a minimal amount of data for the full match to happen on the manager.
 type MinDataStore: record {
 	host_data:    set[addr];
+	subnet_data:  set[subnet];
 	string_data:  set[string, Type];
 };
 global min_data_store: MinDataStore &redef;
@@ -191,33 +218,78 @@ event bro_init() &priority=5
 	Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]);
 	}
 
+# Function that abstracts expiration of different types.
+function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval
+	{
+	if ( hook item_expired(indicator, indicator_type, metas) )
+		return item_expiration;
+	else
+		remove([$indicator=indicator, $indicator_type=indicator_type, $meta=[$source=""]], T);
+	return 0 sec;
+	}
+
+# Expiration handler definitions.
+function expire_host_data(data: table[addr] of MetaDataTable, idx: addr): interval
+	{
+	local meta_tbl: MetaDataTable = data[idx];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(cat(idx), ADDR, metas);
+	}
+
+function expire_subnet_data(data: table[subnet] of MetaDataTable, idx: subnet): interval
+	{
+	local meta_tbl: MetaDataTable = data[idx];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(cat(idx), ADDR, metas);
+	}
+
+function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any): interval
+	{
+	local indicator: string;
+	local indicator_type: Type;
+	[indicator, indicator_type] = idx;
+
+	local meta_tbl: MetaDataTable = data[indicator, indicator_type];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(indicator, indicator_type, metas);
+	}
+
+# Function to check for intelligence hits.
 function find(s: Seen): bool
 	{
+	local ds = have_full_data ? data_store : min_data_store;
+
 	if ( s?$host )
 		{
-		return ((s$host in min_data_store$host_data) || 
+		return ((s$host in ds$host_data) ||
-		        (have_full_data && s$host in data_store$host_data));
+		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
-		}
-	else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
-	           (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) )
-		{
-		return T;
 		}
 	else
 		{
-		return F;
+		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
 		}
 	}
 
+# Function to retrieve intelligence items while abstracting from different
+# data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
 	local return_data: set[Item];
+	local mt: MetaDataTable;
 
 	if ( ! have_full_data )
 		{
-		# A reporter warning should be generated here because this function
+		Reporter::warning(fmt("Intel::get_items was called from a host (%s) that doesn't have the full data.",
-		# should never be called from a host that doesn't have the full data.
+			peer_description));
-		# TODO: do a reporter warning.
 		return return_data;
 		}
 
@@ -226,11 +298,23 @@ function get_items(s: Seen): set[Item]
 		# See if the host is known about and it has meta values
 		if ( s$host in data_store$host_data )
 			{
-			for ( m in data_store$host_data[s$host] )
+			mt = data_store$host_data[s$host];
+			for ( m in mt )
 				{
-				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)];
+				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
 				}
 			}
+		# See if the host is part of a known subnet, which has meta values
+		local nets: table[subnet] of MetaDataTable;
+		nets = filter_subnet_table(addr_to_subnet(s$host), data_store$subnet_data);
+		for ( n in nets )
+			{
+				mt = nets[n];
+				for ( m in mt )
+					{
+					add return_data[Item($indicator=cat(n), $indicator_type=SUBNET, $meta=mt[m])];
+					}
+			}
 		}
 	else
 		{
@@ -238,9 +322,10 @@ function get_items(s: Seen): set[Item]
 		# See if the string is known about and it has meta values
 		if ( [lower_indicator, s$indicator_type] in data_store$string_data )
 			{
-			for ( m in data_store$string_data[lower_indicator, s$indicator_type] )
+			mt = data_store$string_data[lower_indicator, s$indicator_type];
+			for ( m in mt )
 				{
-				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)];
+				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=mt[m])];
 				}
 			}
 		}
@@ -275,64 +360,20 @@ function Intel::seen(s: Seen)
 		}
 	}
 
-
-function has_meta(check: MetaData, metas: set[MetaData]): bool
-	{
-	local check_hash = md5_hash(check);
-	for ( m in metas )
-		{
-		if ( check_hash == md5_hash(m) )
-			return T;
-		}
-
-	# The records must not be equivalent if we made it this far.
-	return F;
-	}
-
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info = Info($ts=network_time(), $seen=s);
+	local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
 
-	if ( s?$f )
+	if ( hook extend_match(info, s, items) )
-		{
+		Log::write(Intel::LOG, info);
-		s$fuid = s$f$id;
-
-		if ( s$f?$conns && |s$f$conns| == 1 )
-			{
-			for ( cid in s$f$conns )
-				s$conn = s$f$conns[cid];
-			}
-
-		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
-			info$file_mime_type = s$f$info$mime_type;
-
-		if ( ! info?$file_desc )
-			info$file_desc = Files::describe(s$f);
-		}
-
-	if ( s?$fuid )
-		info$fuid = s$fuid;
-
-	if ( s?$conn )
-		{
-		s$uid = s$conn$uid;
-		info$id  = s$conn$id;
-		}
-
-	if ( s?$uid )
-		info$uid = s$uid;
-
-	for ( item in items )
-		add info$sources[item$meta$source];
-
-	Log::write(Intel::LOG, info);
 	}
 
 function insert(item: Item)
 	{
-	# Create and fill out the meta data item.
+	# Create and fill out the metadata item.
 	local meta = item$meta;
-	local metas: set[MetaData];
+	local meta_tbl: table [string] of MetaData;
+	local is_new: bool = T;
 
 	# All intelligence is case insensitive at the moment.
 	local lower_indicator = to_lower(item$indicator);
@@ -343,51 +384,133 @@ function insert(item: Item)
 		if ( have_full_data )
 			{
 			if ( host !in data_store$host_data )
-				data_store$host_data[host] = set();
+				data_store$host_data[host] = table();
+			else
+				is_new = F;
 
-			metas = data_store$host_data[host];
+			meta_tbl = data_store$host_data[host];
 			}
 
 		add min_data_store$host_data[host];
 		}
+	else if ( item$indicator_type == SUBNET )
+		{
+		local net = to_subnet(item$indicator);
+		if ( have_full_data )
+			{
+			if ( !check_subnet(net, data_store$subnet_data) )
+				data_store$subnet_data[net] = table();
+			else
+				is_new = F;
+
+			meta_tbl = data_store$subnet_data[net];
+			}
+
+		add min_data_store$subnet_data[net];
+		}
 	else
 		{
 		if ( have_full_data )
 			{
 			if ( [lower_indicator, item$indicator_type] !in data_store$string_data )
-				data_store$string_data[lower_indicator, item$indicator_type] = set();
+				data_store$string_data[lower_indicator, item$indicator_type] = table();
+			else
+				is_new = F;
 
-			metas = data_store$string_data[lower_indicator, item$indicator_type];
+			meta_tbl = data_store$string_data[lower_indicator, item$indicator_type];
 			}
 
 		add min_data_store$string_data[lower_indicator, item$indicator_type];
 		}
 
-	local updated = F;
 	if ( have_full_data )
 		{
-		for ( m in metas )
+		# Insert new metadata or update if already present
-			{
+		meta_tbl[meta$source] = meta;
-			if ( meta$source == m$source )
-				{
-				if ( has_meta(meta, metas) )
-					{
-					# It's the same item being inserted again.
-					return;
-					}
-				else
-					{
-					# Same source, different metadata means updated item.
-					updated = T;
-					}
-				}
-			}
-		add metas[item$meta];
 		}
-	
+
-	if ( updated )
+	if ( is_new )
-		event Intel::updated_item(item);
+		# Trigger insert for cluster in case the item is new
-	else
+		# or insert was called on a worker
 		event Intel::new_item(item);
 	}
-	
+
+# Function to remove metadata of an item. The function returns T
+# if there is no metadata left for the given indicator.
+function remove_meta_data(item: Item): bool
+	{
+	if ( ! have_full_data )
+		{
+		Reporter::warning(fmt("Intel::remove_meta_data was called from a host (%s) that doesn't have the full data.",
+			peer_description));
+		return F;
+		}
+
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			local host = to_addr(item$indicator);
+			delete data_store$host_data[host][item$meta$source];
+			return (|data_store$host_data[host]| == 0);
+		case SUBNET:
+			local net = to_subnet(item$indicator);
+			delete data_store$subnet_data[net][item$meta$source];
+			return (|data_store$subnet_data[net]| == 0);
+		default:
+			delete data_store$string_data[item$indicator, item$indicator_type][item$meta$source];
+			return (|data_store$string_data[item$indicator, item$indicator_type]| == 0);
+		}
+	}
+
+function remove(item: Item, purge_indicator: bool)
+	{
+	# Delegate removal if we are on a worker
+	if ( !have_full_data )
+		{
+		event Intel::remove_item(item, purge_indicator);
+		return;
+		}
+
+	# Remove metadata from manager's data store
+	local no_meta_data = remove_meta_data(item);
+	# Remove whole indicator if necessary
+	if ( no_meta_data || purge_indicator )
+		{
+		switch ( item$indicator_type )
+			{
+			case ADDR:
+				local host = to_addr(item$indicator);
+				delete data_store$host_data[host];
+				break;
+			case SUBNET:
+				local net = to_subnet(item$indicator);
+				delete data_store$subnet_data[net];
+				break;
+			default:
+				delete data_store$string_data[item$indicator, item$indicator_type];
+				break;
+			}
+		# Trigger deletion in minimal data stores
+		event Intel::purge_item(item);
+		}
+	}
+
+# Handling of indicator removal in minimal data stores.
+event purge_item(item: Item)
+	{
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			local host = to_addr(item$indicator);
+			delete min_data_store$host_data[host];
+			break;
+		case SUBNET:
+			local net = to_subnet(item$indicator);
+			delete min_data_store$subnet_data[net];
+			break;
+		default:
+			delete min_data_store$string_data[item$indicator, item$indicator_type];
+			break;
+		}
+	}
+

scripts/policy/frameworks/intel/do_expire.bro

@@ -0,0 +1,14 @@
+##! This script enables expiration for intelligence items.
+
+@load base/frameworks/intel
+
+module Intel;
+
+redef item_expiration = 10min;
+
+hook item_expired(indicator: string, indicator_type: Type,
+	metas: set[MetaData]) &priority=-10
+	{
+	# Trigger removal of the expired item.
+	break;
+	}

scripts/policy/frameworks/intel/do_notice.bro

@@ -1,3 +1,4 @@
+##! This script enables notice generation for intelligence matches.
 
 @load base/frameworks/intel
 @load base/frameworks/notice
@@ -6,14 +7,14 @@ module Intel;
 
 export {
 	redef enum Notice::Type += {
-		## Intel::Notice is a notice that happens when an intelligence 
+		## Intel::Notice is a notice that happens when an intelligence
 		## indicator is denoted to be notice-worthy.
 		Intel::Notice
 	};
 
 	redef record Intel::MetaData += {
 		## A boolean value to allow the data itself to represent
-		## if the indicator that this metadata is attached to 
+		## if the indicator that this metadata is attached to
 		## is notice worthy.
 		do_notice: bool &default=F;
 
@@ -29,15 +30,42 @@ event Intel::match(s: Seen, items: set[Item])
 	for ( item in items )
 		{
 		if ( item$meta$do_notice &&
-		     (! item$meta?$if_in || s$where == item$meta$if_in) )
+			(! item$meta?$if_in || s$where == item$meta$if_in) )
 			{
 			local n = Notice::Info($note=Intel::Notice,
-			                       $msg=fmt("Intel hit on %s at %s", s$indicator, s$where),
+				$msg = fmt("Intel hit on %s at %s", s$indicator, s$where),
-			                       $sub=s$indicator);
+				$sub = s$indicator);
+			local service_str = "";
 
 			if ( s?$conn )
+				{
 				n$conn = s$conn;
 
+				# Add identifier composed of indicator, originator's and responder's IP,
+				# without considering the direction of the flow.
+				local intel_id = s$indicator;
+				if( s$conn?$id )
+					{
+					if( s$conn$id$orig_h < s$conn$id$resp_h)
+						intel_id = cat(intel_id, s$conn$id$orig_h, s$conn$id$resp_h);
+					else
+						intel_id = cat(intel_id, s$conn$id$resp_h, s$conn$id$orig_h);
+					}
+				n$identifier = intel_id;
+
+				if ( s$conn?$service )
+					{
+					for ( service in s$conn$service )
+						service_str = cat(service_str, service, " ");
+					}
+				}
+
+			# Add additional information to the generated mail
+			local mail_ext = vector(
+				fmt("Service: %s\n", service_str),
+				fmt("Intel source: %s\n", item$meta$source));
+			n$email_body_sections = mail_ext;
+
 			NOTICE(n);
 			}
 		}

scripts/policy/frameworks/intel/whitelist.bro

@@ -0,0 +1,30 @@
+##! This script enables whitelisting for intelligence items.
+
+@load base/frameworks/intel
+
+module Intel;
+
+export {
+	redef record Intel::MetaData += {
+		## A boolean value to indicate whether the item is whitelisted.
+		whitelist: bool &default=F;
+	};
+}
+
+hook Intel::extend_match(info: Info, s: Seen, items: set[Item]) &priority=9
+	{
+	local whitelisted = F;
+	for ( item in items )
+		{
+		if ( item$meta$whitelist )
+			{
+			whitelisted = T;
+			break;
+			}
+		}
+	
+	if ( whitelisted )
+		# Prevent logging
+		break;
+	}
+

scripts/test-all-policy.bro

@@ -15,6 +15,8 @@
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
 @load frameworks/intel/do_notice.bro
+@load frameworks/intel/do_expire.bro
+@load frameworks/intel/whitelist.bro
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-08-01-16-08-53
+#open	2016-06-15-19-16-09
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -203,6 +203,7 @@ scripts/base/init-default.bro
     scripts/base/frameworks/communication/main.bro
   scripts/base/frameworks/intel/__load__.bro
     scripts/base/frameworks/intel/main.bro
+    scripts/base/frameworks/intel/files.bro
     scripts/base/frameworks/intel/input.bro
   scripts/base/frameworks/sumstats/__load__.bro
     scripts/base/frameworks/sumstats/main.bro
@@ -312,4 +313,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-08-01-16-08-53
+#close	2016-06-15-19-16-09

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-13-39
+#open	2016-06-15-19-11-27
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1411488819.555114	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	worker-1
+1466017887.060652	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1	-	-	-
-#close	2014-09-23-16-13-49
+#close	2016-06-15-19-11-36

testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output

@@ -0,0 +1,22 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-15-19-11-06
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017866.348490	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017867.349583	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017868.349656	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2016-06-15-19-11-12
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Expired: 1.2.3.4
+Trigger: 1.2.3.4
+Trigger: 1.2.3.4
+Trigger: 1.2.3.4

testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-14-49
+#open	2016-06-15-19-12-26
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1411488889.571819	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	source1
+1466017946.413077	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1	-	-	-
-1411488889.571819	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
+1466017946.413077	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-#close	2014-09-23-16-14-49
+#close	2016-06-15-19-12-26

testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output

@@ -0,0 +1,24 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-22-19-12-08
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466622728.846581	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466622728.846581	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1466622728.846581	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
+#close	2016-06-22-19-12-08
+
+Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
+Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
+
+Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
+Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
+
+Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
+Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
+Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
+Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
+Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]

testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log

@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-15-00
+#open	2016-06-15-19-14-30
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1411488900.900403	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	source1
+1466018070.494693	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1	-	-	-
-1411488900.900403	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	source1
+1466018070.494693	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1	-	-	-
-1411488901.923543	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	source1
+1466018071.505800	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1	-	-	-
-1411488901.923543	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	source1
+1466018071.505800	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1	-	-	-
-#close	2014-09-23-16-15-09
+#close	2016-06-15-19-14-39

testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout

@@ -0,0 +1,6 @@
+Purging 192.168.0.1.
+Purging 192.168.0.2.
+Removing 192.168.1.2 (source: source1).
+Removing 192.168.1.2 (source: source2).
+Purging 192.168.1.2.
+Logging intel hit!

testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-15-19-10-09
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017809.810005	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end	-	-	-
+#close	2016-06-15-19-10-19

testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout

@@ -0,0 +1,5 @@
+Removing 192.168.1.2 (source: source1).
+Removing 192.168.1.2 (source: source2).
+Purging 192.168.0.1.
+Purging 192.168.0.2.
+Purging 192.168.1.2.

testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output

@@ -0,0 +1,25 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-15-19-09-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017751.936022	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017754.938975	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
+1466017754.938975	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+1466017757.941783	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
+1466017757.941783	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+#close	2016-06-15-19-09-18
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2016-06-15-19-09-18
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
+1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-06-15-19-09-18

testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log

@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-07-13-16-17-18
+#open	2016-06-15-19-08-03
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1416942644.593119	CHhAvVGS1DHFjwGM9	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
-#close	2016-07-13-16-17-18
+#close	2016-06-15-19-08-03
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-07-13-16-17-19
+#open	2016-06-15-19-08-03
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1170717505.735416	CHhAvVGS1DHFjwGM9	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	source1
+1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
-1170717505.934612	CHhAvVGS1DHFjwGM9	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
-1170717508.883051	ClEkJM2Vm5giqnMf4h	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	source1
+1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
-1170717509.082241	ClEkJM2Vm5giqnMf4h	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
-1170717511.909717	C4J4Th3PJpwUYZZ6gc	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	source1
+1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
-1170717512.108799	C4J4Th3PJpwUYZZ6gc	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
-#close	2016-07-13-16-17-19
+#close	2016-06-15-19-08-03

testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log

@@ -0,0 +1,29 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-15-19-06-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+#close	2016-06-15-19-06-02

testing/btest/scripts/base/frameworks/intel/expire-item.bro

@@ -0,0 +1,46 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 7
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.4	Intel::ADDR	source1	this host is bad	http://some-data-distributor.com/1
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+@load frameworks/intel/do_expire
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+redef Intel::item_expiration = 3sec;
+redef table_expire_interval = 1sec;
+
+global runs = 0;
+event do_it()
+	{
+	print "Trigger: 1.2.3.4";
+	Intel::seen([$host=1.2.3.4,
+	             $where=SOMEWHERE]);
+
+	++runs;
+	if ( runs < 6 )
+		schedule 1sec { do_it() };
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print fmt("Seen: %s", s$indicator);
+	}
+
+hook Intel::item_expired(indicator: string, indicator_type: Intel::Type,
+	metas: set[Intel::MetaData])
+	{
+	print fmt("Expired: %s", indicator);
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

testing/btest/scripts/base/frameworks/intel/match-subnet.bro

@@ -0,0 +1,51 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+192.168.1.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1
+192.168.2.0/24	Intel::SUBNET	source1	this subnetwork is just plain baaad	http://some-data-distributor.com/2
+192.168.142.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/3
+192.168.142.0/24	Intel::SUBNET	source1	this subnetwork is baaad	http://some-data-distributor.com/4
+192.168.142.0/26	Intel::SUBNET	source1	this subnetwork is inside	http://some-data-distributor.com/4
+192.168.128.0/18	Intel::SUBNET	source1	this subnetwork might be baaad	http://some-data-distributor.com/5
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+event do_it()
+	{
+	Intel::seen([$host=192.168.1.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=192.168.2.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=192.168.142.1,
+	             $where=SOMEWHERE]);
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 2 )
+		terminate();
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print "";
+	print fmt("Seen: %s", s);
+	for ( item in items )
+		print fmt("Item: %s", item);
+	}

testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro

@@ -0,0 +1,88 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff manager-1/.stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff manager-1/intel.log
+
+# @TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"],
+};
+# @TEST-END-FILE
+
+@load base/frameworks/control
+
+module Intel;
+
+redef Log::default_rotation_interval=0sec;
+
+event test_manager()
+	{
+	Intel::remove([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::seen([$host=192.168.0.1, $where=Intel::IN_ANYWHERE]);
+	Intel::remove([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]], T);
+	Intel::seen([$host=192.168.0.2, $where=Intel::IN_ANYWHERE]);
+	}
+
+event test_worker()
+	{
+	Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+	Intel::seen([$host=192.168.1.2, $where=Intel::IN_ANYWHERE]);
+	# Trigger shutdown by matching data that should be present
+	Intel::seen([$host=10.10.10.10, $where=Intel::IN_ANYWHERE]);
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# Insert the data once all workers are connected.
+	if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 1 )
+		{
+		Intel::insert([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+		Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+		Intel::insert([$indicator="10.10.10.10", $indicator_type=Intel::ADDR, $meta=[$source="end"]]);
+
+		event test_manager();
+		}
+	}
+
+global worker_data = 0;
+event Intel::cluster_new_item(item: Intel::Item)
+	{
+	# Run test on worker-1 when all items have been inserted
+	if ( Cluster::node == "worker-1" )
+		{
+		++worker_data;
+		if ( worker_data == 4 )
+			event test_worker();
+		}
+	}
+
+event Intel::remove_item(item: Item, purge_indicator: bool)
+	{
+	print fmt("Removing %s (source: %s).", item$indicator, item$meta$source);
+	}
+
+event purge_item(item: Item)
+	{
+	print fmt("Purging %s.", item$indicator);
+	}
+
+event Intel::log_intel(rec: Intel::Info)
+	{
+	print "Logging intel hit!";
+	event Control::shutdown_request();
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	# Cascading termination
+	terminate_communication();
+	}

testing/btest/scripts/base/frameworks/intel/updated-match.bro

@@ -0,0 +1,62 @@
+# @TEST-SERIALIZE: comm
+
+# @TEST-EXEC: cp intel1.dat intel.dat
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp intel2.dat intel.dat
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp intel3.dat intel.dat
+# @TEST-EXEC: btest-bg-wait 6
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/notice.log >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel1.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234	F
+# @TEST-END-FILE
+
+# @TEST-START-FILE intel2.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source2	this host is just plain baaad	http://some-data-distributor.com/1234	F
+4.3.2.1	Intel::ADDR	source2	this host might also be baaad	http://some-data-distributor.com/4321	F
+# @TEST-END-FILE
+
+# @TEST-START-FILE intel3.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source2	this host is just plain baaad	http://some-data-distributor.com/1234	T
+4.3.2.1	Intel::ADDR	source2	this host might also be baaad	http://some-data-distributor.com/4321	T
+# @TEST-END-FILE
+
+@load base/frameworks/communication # let network-time run
+@load frameworks/intel/do_notice
+
+redef exit_only_after_terminate = T;
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+global runs = 0;
+event do_it()
+	{
+	Intel::seen([$host=1.2.3.4,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=4.3.2.1,
+	             $where=SOMEWHERE]);
+
+	++runs;
+	if ( runs < 3 )
+		schedule 3sec { do_it() };
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 5 )
+		terminate();
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

testing/btest/scripts/policy/frameworks/intel/seen/certs.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro -r $TRACES/tls/ecdsa-cert.pcap %INPUT
+# @TEST-EXEC: bro -Cr $TRACES/tls/ecdsa-cert.pcap %INPUT
 # @TEST-EXEC: cat intel.log > intel-all.log
 # @TEST-EXEC: bro -r $TRACES/tls/ssl.v3.trace %INPUT
 # @TEST-EXEC: cat intel.log >> intel-all.log

testing/btest/scripts/policy/frameworks/intel/whitelisting.bro

@@ -0,0 +1,39 @@
+# @TEST-EXEC: bro -Cr $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-diff intel.log
+
+#@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+upload.wikimedia.org	Intel::DOMAIN	source1	somehow bad	http://some-data-distributor.com/1
+meta.wikimedia.org	Intel::DOMAIN	source1	also bad	http://some-data-distributor.com/1
+#@TEST-END-FILE
+
+#@TEST-START-FILE whitelist.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.whitelist	meta.url
+meta.wikimedia.org	Intel::DOMAIN	source2	also bad	T	http://some-data-distributor.com/1
+#@TEST-END-FILE
+
+@load base/frameworks/intel
+@load frameworks/intel/whitelist
+@load frameworks/intel/seen
+
+redef Intel::read_files += {
+	"intel.dat",
+	"whitelist.dat",
+};
+
+global total_files_read = 0;
+
+event bro_init()
+	{
+	suspend_processing();
+	}
+
+event Input::end_of_data(name: string, source: string)
+	{
+	# Wait until both intel files are read.
+	if ( /^intel-/ in name && (++total_files_read == 2) )
+		{
+		continue_processing();
+		}
+	}
+