Functional intelligence framework.

- All 5 intelligence tests pass.
- Some initial memory optimizations done.
  - More work needs done to reduce duplicate data in memory.
- Input framework integration.
  - Define files to read in the "Bro intelligence format" in Intel::read_files.
- Cluster transparency.
- DNS Zones are a fully supported data type.
  - Queries for Intel::DOMAIN values will automatically check in DNS_ZONE intelligence.

scripts/base/frameworks/intel/__load__.bro

@@ -1 +1,11 @@
 @load ./main
+@load ./input
+
+# The cluster framework must be loaded first.
+@load base/frameworks/cluster
+
+@if ( Cluster::is_enabled() )
+@load ./cluster
+@endif
+
+@load ./plugins/dns_zones

scripts/base/frameworks/intel/cluster.bro

@@ -0,0 +1,59 @@
+##! Cluster transparency support for the intelligence framework.  This is mostly oriented
+##! toward distributing intelligence information across clusters.
+
+@load base/frameworks/cluster
+
+module Intel;
+
+export {
+	global cluster_new_item: event(item: Item);
+	global cluster_updated_item: event(item: Item);
+
+	redef record Item += {
+		## This field is solely used internally for cluster transparency with
+		## the intelligence framework to avoid storms of intelligence data 
+		## swirling forever.  It allows data to propagate only a single time.
+		first_dispatch: bool &default=T;
+	};
+}
+
+# Primary intelligence distribution comes from manager.
+redef Cluster::manager2worker_events += /Intel::cluster_(new|updated)_item/;
+# If a worker finds intelligence and adds it, it should share it back to the manager.
+redef Cluster::worker2manager_events += /Intel::cluster_(new|updated)_item/;
+
+event Intel::cluster_new_item(item: Intel::Item)
+	{
+	# Ignore locally generated events.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
+
+event Intel::cluster_updated_item(item: Intel::Item)
+	{
+	# Ignore locally generated events.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
+
+event Intel::new_item(item: Intel::Item)
+	{
+	# If this is the first time this item has been dispatched, 
+	# send it over the cluster.
+	if ( item$first_dispatch )
+		{
+		item$first_dispatch = F;
+		event Intel::cluster_new_item(item);
+		}
+	}
+
+event Intel::updated_item(item: Intel::Item)
+	{
+	# If this is the first time this item has been dispatched, 
+	# send it over the cluster.
+	if ( item$first_dispatch )
+		{
+		item$first_dispatch = F;
+		event Intel::cluster_updated_item(item);
+		}
+	}

scripts/base/frameworks/intel/input.bro

@@ -0,0 +1,28 @@
+@load ./main
+
+module Intel;
+
+export {
+	## Files that will be read off disk
+	const read_files: set[string] = {} &redef;
+
+	global entry: event(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item);
+}
+
+event Intel::entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item)
+	{
+	Intel::insert(item);
+	}
+
+event bro_init() &priority=5
+	{
+	for ( a_file in read_files )
+		{
+		Input::add_event([$source=a_file,
+		                  $reader=Input::READER_ASCII,
+		                  $mode=Input::REREAD,
+		                  $name=cat("intel-", a_file),
+		                  $fields=Intel::Item,
+		                  $ev=Intel::entry]);
+		}
+	}

scripts/base/frameworks/intel/main.bro

@@ -1,31 +1,19 @@
 ##! The intelligence framework provides a way to store and query IP addresses,
-##! strings (with a subtype), and numeric (with a subtype) data.  Metadata 
+##! and strings (with a subtype).  Metadata can
-##! also be associated with the intelligence like tags which are arbitrary
+##! also be associated with the intelligence like for making more informated
-##! strings, time values, and longer descriptive strings.
+##! decisions about matching and handling of intelligence.
-
+#
-# Example string subtypes:
+# TODO: 
-#   url
+#   Comments
-#   email
+#   Better Intel::Item comparison (same_meta)
-#   domain
+#   Generate a notice when messed up data is discovered.
-#   software
+#   Complete "net" support as an intelligence type.
-#   user_name
-#   file_name
-#   file_md5
-#   x509_md5
-
-# Example tags: 
-#   infrastructure
-#   malicious
-#   sensitive
-#   canary
-#   friend
 
 @load base/frameworks/notice
 
 module Intel;
 
 export {
-	## The intel logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
 	redef enum Notice::Type += {
@@ -34,158 +22,171 @@ export {
 		Detection,
 	};
 	
-	## Record type used for logging information from the intelligence framework.
+	type Classification: enum {
-	## Primarily for problems or oddities with inserting and querying data.  
+		MALICIOUS,
-	## This is important since the content of the intelligence framework can 
+		INFRASTRUCTURE,
-	## change quite dramatically during runtime and problems may be introduced 
+		SENSITIVE,
-	## into the data.
+		FRIEND,
+		CANARY,
+		WHITELIST,
+	};
+
+	type SubType: enum {
+		URL,
+		EMAIL,
+		DOMAIN,
+		USER_NAME,
+		FILE_HASH, # (non hash type specific, md5, sha1, sha256)
+		CERT_HASH,
+		ASN,
+	};
+	
 	type Info: record {
-		## The current network time.
 		ts:      time   &log;
-		## Represents the severity of the message. 
 		## This value should be one of: "info", "warn", "error"
 		level:   string &log;
-		## The message.
 		message: string &log;
 	};
 	
-	## Record to represent metadata associated with a single piece of
-	## intelligence.
 	type MetaData: record {
-		## A description for the data.
+		source:      string;
+		class:       Classification;
 		desc:        string      &optional;
-		## A URL where more information may be found about the intelligence.
 		url:         string      &optional;
-		## The time at which the data was first declared to be intelligence.
+		tags:        set[string] &optional;
-		first_seen:  time        &optional;
-		## When this data was most recent inserted into the framework.
-		latest_seen: time        &optional;
-		## Arbitrary text tags for the data.
-		tags:        set[string];
 	};
 	
-	## Record to represent a singular piece of intelligence.
 	type Item: record {
-		## If the data is an IP address, this hold the address.
+		ip:          addr           &optional;
-		ip:          addr        &optional;
+		net:         subnet         &optional;
-		## If the data is textual, this holds the text.
-		str:         string      &optional;
-		## If the data is numeric, this holds the number.
-		num:         int         &optional;
-		## The subtype of the data for when either the $str or $num fields are
-		## given.  If one of those fields are given, this field must be present.
-		subtype:     string      &optional;
 
-		## The next five fields are temporary until a better model for 
+		str:         string         &optional;
-		## attaching metadata to an intelligence item is created.
+		subtype:     SubType        &optional;
-		desc:        string      &optional;
-		url:         string      &optional;
-		first_seen:  time        &optional;
-		latest_seen: time        &optional;
-		tags:        set[string];
 		
-		## These single string tags are throw away until pybroccoli supports sets.
+		meta:        MetaData;
-		tag1: string &optional;
-		tag2: string &optional;
-		tag3: string &optional;
 	};
 
-	## Record model used for constructing queries against the intelligence 
+	type Query: record {
-	## framework.
+		ip:          addr           &optional;
-	type QueryItem: record {
-		## If an IP address is being queried for, this field should be given.
-		ip:        addr        &optional;
-		## If a string is being queried for, this field should be given.
-		str:       string      &optional;
-		## If numeric data is being queried for, this field should be given.
-		num:       int         &optional;
-		## If either a string or number is being queried for, this field should
-		## indicate the subtype of the data.
-		subtype:   string      &optional;
 
-		## A set of tags where if a single metadata record attached to an item
+		str:         string         &optional;
-		## has any one of the tags defined in this field, it will match.
+		subtype:     SubType        &optional;
-		or_tags:   set[string] &optional;
+		
-		## A set of tags where a single metadata record attached to an item 
+		class:       Classification &optional;
-		## must have all of the tags defined in this field.
+
-		and_tags:  set[string] &optional; 
+		or_tags:     set[string]    &optional;
+		and_tags:    set[string]    &optional;
 		
 		## The predicate can be given when searching for a match.  It will
-		## be tested against every :bro:type:`Intel::MetaData` item associated
+		## be tested against every :bro:type:`MetaData` item associated with 
-		## with the data being matched on.  If it returns T a single time, the 
+		## the data being matched on.  If it returns T a single time, the 
-		## matcher will consider that the item has matched.  This field can
+		## matcher will consider that the item has matched.
-		## be used for constructing arbitrarily complex queries that may not
+		pred:        function(meta: Intel::Item): bool &optional;
-		## be possible with the $or_tags or $and_tags fields.
+	};
-		pred:      function(meta: Intel::MetaData): bool &optional;
+	
+	type Importer: enum {
+		NULL_IMPORTER
 	};
 
-	## Function to insert data into the intelligence framework.
-	## 
-	## item: The data item.
-	##
-	## Returns: T if the data was successfully inserted into the framework,
-	##          otherwise it returns F.
 	global insert: function(item: Item): bool;
-	
-	## A wrapper for the :bro:id:`Intel::insert` function.  This is primarily
-	## used as the external API for inserting data into the intelligence 
-	## using Broccoli.
 	global insert_event: event(item: Item);
+	global delete_item: function(item: Item): bool;
 
-	## Function for matching data within the intelligence framework.
+	global matcher: function(query: Query): bool;
-	global matcher: function(item: QueryItem): bool;
+	global lookup: function(query: Query): set[Item];
+
+	global register_custom_matcher: function(subtype: SubType, 
+	                                         func: function(query: Query): bool);
+	global register_custom_lookup: function(subtype: SubType,
+	                                        func: function(query: Query): set[Item]);
+
+	global new_item: event(item: Item);
+	global updated_item: event(item: Item);
 }
 
-type MetaDataStore: table[count] of MetaData;
+## Store collections of :bro:type:`MetaData` records indexed by a source name.
+type IndexedItems: table[string, Classification] of MetaData;
 type DataStore: record {
-	ip_data:     table[addr] of MetaDataStore;
+	ip_data:     table[addr] of IndexedItems;
-	# The first string is the actual value and the second string is the subtype.
+	string_data: table[string, SubType] of IndexedItems;
-	string_data: table[string, string] of MetaDataStore;
-	int_data:    table[int, string] of MetaDataStore;
 };
 global data_store: DataStore;
 
-event bro_init()
+global custom_matchers: table[SubType] of set[function(query: Query): bool];
+global custom_lookup: table[SubType] of set[function(query: Query): set[Item]];
+
+event bro_init() &priority=5
 	{
 	Log::create_stream(Intel::LOG, [$columns=Info]);
 	}
 
+function register_custom_matcher(subtype: SubType, func: function(query: Query): bool)
+	{
+	if ( subtype !in custom_matchers )
+		custom_matchers[subtype] = set();
+	add custom_matchers[subtype][func];
+	}
+
+function register_custom_lookup(subtype: SubType, func: function(query: Query): set[Item])
+	{
+	if ( subtype !in custom_lookup )
+		custom_lookup[subtype] = set();
+	add custom_lookup[subtype][func];
+	}
+
+
+
+function same_meta(meta1: MetaData, meta2: MetaData): bool
+	{
+	# "any" type values can't be compared so this generic implementation doesn't work.
+	#local rf1 = record_fields(item1);
+	#local rf2 = record_fields(item2);
+	#for ( field in rf1 )
+	#	{
+	#	if ( ((rf1[field]?$value && rf1[field]?$value) &&
+	#	       rf1[field]$value != rf2[field]$value) ||
+	#	      ! (rf1[field]?$value && rf1[field]?$value) )
+	#		return F; 
+	#	}
+
+	if ( meta1$source == meta2$source &&
+	     meta1$class  == meta2$class &&
+	     ((!meta1?$desc && !meta2?$desc) || (meta1?$desc && meta2?$desc && meta1$desc == meta2$desc)) &&
+	     ((!meta1?$url && !meta2?$url) || (meta1?$url && meta2?$url && meta1$url == meta2$url)) &&
+	     ((!meta1?$tags && !meta2?$tags) || (meta1?$tags && meta2?$tags && |meta1$tags| == |meta2$tags|)) )
+		{
+		# TODO: match on all of the tag values
+		return T;
+		}
+
+	# The records must not be equivalent if we made it this far.
+	return F;
+	}
 
 function insert(item: Item): bool
 	{
 	local err_msg = "";
-	if ( (item?$str || item?$num) && ! item?$subtype )
+	if ( item?$str && ! item?$subtype )
-		err_msg = "You must provide a subtype to insert_sync or this item doesn't make sense.";
+		err_msg = "You must provide a subtype for strings or this item doesn't make sense.";
 	
 	if ( err_msg == "" )
 		{
 		# Create and fill out the meta data item.
-		local meta: MetaData;
+		local meta = item$meta;
-		if ( item?$first_seen )
-			meta$first_seen = item$first_seen;
-		if ( item?$latest_seen )
-			meta$latest_seen = item$latest_seen;
-		if ( item?$tags )
-			meta$tags = item$tags;
-		if ( item?$desc )
-			meta$desc = item$desc;
-		if ( item?$url )
-			meta$url = item$url;
-		
-		
-		# This is hopefully only temporary until pybroccoli supports sets.
-		if ( item?$tag1 )
-			add item$tags[item$tag1];
-		if ( item?$tag2 )
-			add item$tags[item$tag2];
-		if ( item?$tag3 )
-			add item$tags[item$tag3];
 
 		if ( item?$ip )
 			{
 			if ( item$ip !in data_store$ip_data )
 				data_store$ip_data[item$ip] = table();
-			data_store$ip_data[item$ip][|data_store$ip_data[item$ip]|] = meta;
+			
+			if ( [meta$source, meta$class] !in data_store$ip_data[item$ip] )
+				event Intel::new_item(item);
+			else if ( ! same_meta(data_store$ip_data[item$ip][meta$source, meta$class], meta) )
+				event Intel::updated_item(item);
+			else 
+				return F;
+
+			data_store$ip_data[item$ip][meta$source, meta$class] = item$meta;
 			return T;
 			}
 		else if ( item?$str )
@@ -193,15 +194,14 @@ function insert(item: Item): bool
 			if ( [item$str, item$subtype] !in data_store$string_data )
 				data_store$string_data[item$str, item$subtype] = table();
 			
-			data_store$string_data[item$str, item$subtype][|data_store$string_data[item$str, item$subtype]|] = meta;
+			if ( [meta$source, meta$class] !in data_store$string_data[item$str, item$subtype] )
-			return T;
+				event Intel::new_item(item);
-			}
+			else if ( ! same_meta(data_store$string_data[item$str, item$subtype][meta$source, meta$class], meta) )
-		else if ( item?$num )
+				event Intel::updated_item(item);
-			{
+			else 
-			if ( [item$num, item$subtype] !in data_store$int_data )
+				return F;
-				data_store$int_data[item$num, item$subtype] = table();
 
-			data_store$int_data[item$num, item$subtype][|data_store$int_data[item$num, item$subtype]|] = meta;
+			data_store$string_data[item$str, item$subtype][meta$source, meta$class] = item$meta;
 			return T;
 			}
 		else
@@ -218,106 +218,160 @@ event insert_event(item: Item)
 	insert(item);
 	}
 
-function match_item_with_metadata(item: QueryItem, meta: MetaData): bool
+function match_item_with_query(item: Item, query: Query): bool
 	{
-	if ( item?$and_tags )
+	if ( ! query?$and_tags && ! query?$or_tags && ! query?$pred )
+		return T;
+
+	if ( query?$and_tags )
 		{
 		local matched = T;
 		# Every tag given has to match in a single MetaData entry.
-		for ( tag in item$and_tags )
+		for ( tag in query$and_tags )
 			{
-			if ( tag !in meta$tags )
+			if ( item$meta?$tags && tag !in item$meta$tags )
 				matched = F;
 			}
 		if ( matched )
 			return T;
 		}
-	else if ( item?$or_tags )
+	else if ( query?$or_tags )
 		{
 		# For OR tags, only a single tag has to match.
-		for ( tag in item$or_tags )
+		for ( tag in query$or_tags )
 			{
-			if ( tag in meta$tags )
+			if ( item$meta?$tags && tag in item$meta$tags )
 				return T;
 			}
 		}
-	else if ( item?$pred )
+	else if ( query?$pred )
-		return item$pred(meta);
+		return query$pred(item);
 
 	# This indicates some sort of failure in the query
 	return F;
 	}
 	
-function matcher(item: QueryItem): bool
+function lookup(query: Query): set[Item]
+	{
+	local meta: MetaData;
+	local item: Item;
+	local return_data: set[Item] = set();
+
+	if ( query?$ip )
+		{
+		if ( query$ip in data_store$ip_data )
+			{
+			for ( [source, class] in data_store$ip_data[query$ip] )
+				{
+				meta = data_store$ip_data[query$ip][source, class];
+				item = [$ip=query$ip,$meta=meta];
+				if ( match_item_with_query(item, query) )
+					add return_data[item];
+				}
+			}
+		}
+	
+	else if ( query?$str )
+		{
+		if ( [query$str, query$subtype] in data_store$string_data )
+			{
+			for ( [source, class] in data_store$string_data[query$str, query$subtype] )
+				{
+				meta = data_store$string_data[query$str, query$subtype][source, class];
+				item = [$str=query$str,$subtype=query$subtype,$meta=meta];
+				if ( match_item_with_query(item, query) )
+					add return_data[item];
+				}
+			}
+
+		# Check if there are any custom subtype lookup functons and add the values to 
+		# the result set.
+		if ( query$subtype in custom_lookup )
+			{
+			for ( lookup_func in custom_lookup[query$subtype] )
+				{
+				# Iterating here because there is no way to merge sets generically.
+				for ( custom_lookup_item in lookup_func(query) )
+					add return_data[custom_lookup_item];
+				}
+			}
+		}
+	
+	return return_data;
+	}
+
+	
+function matcher(query: Query): bool
 	{
 	local err_msg = "";
-	if ( ! (item?$ip || item?$str || item?$num) )
+	if ( (query?$or_tags || query?$and_tags) && query?$pred )
-		err_msg = "You must supply one of the $ip, $str, or $num fields to search on";
-	else if ( (item?$or_tags || item?$and_tags) && item?$pred )
 		err_msg = "You can't match with both tags and a predicate.";
-	else if ( item?$or_tags && item?$and_tags )
+	else if ( query?$or_tags && query?$and_tags )
 		err_msg = "You can't match with both OR'd together tags and AND'd together tags";
-	else if ( (item?$str || item?$num) && ! item?$subtype )
+	else if ( query?$str && ! query?$subtype )
-		err_msg = "You must provide a subtype to matcher or this item doesn't make sense.";
+		err_msg = "You must provide a subtype to matcher or this query doesn't make sense.";
-	else if ( item?$str && item?$num )
-		err_msg = "You must only provide $str or $num, not both.";
 		
+	local item: Item;
 	local meta: MetaData;
 
 	if ( err_msg == "" )
 		{
-		if ( item?$ip )
+		if ( query?$ip )
 			{
-			if ( item$ip in data_store$ip_data )
+			if ( query$ip in data_store$ip_data )
 				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
+				if ( ! query?$and_tags && ! query?$or_tags && ! query?$pred )
 					return T;
 				
-				for ( i in data_store$ip_data[item$ip] )
+				for ( [source, class] in data_store$ip_data[query$ip] )
 					{
-					meta = data_store$ip_data[item$ip][i];
+					meta = data_store$ip_data[query$ip][source, class];
-					if ( match_item_with_metadata(item, meta) )
+					item = [$ip=query$ip,$meta=meta];
+					if ( match_item_with_query(item, query) )
 						return T;
 					}
 				}
 			}
 		
-		else if ( item?$str )
+		else if ( query?$str )
 			{
-			if ( [item$str, item$subtype] in data_store$string_data )
+			if ( [query$str, query$subtype] in data_store$string_data )
 				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
+				if ( ! query?$and_tags && ! query?$or_tags && ! query?$pred )
 					return T;
 
-				for ( i in data_store$string_data[item$str, item$subtype] )
+				for ( [source, class] in data_store$string_data[query$str, query$subtype] )
 					{
-					meta = data_store$string_data[item$str, item$subtype][i];
+					meta = data_store$string_data[query$str, query$subtype][source, class];
-					if ( match_item_with_metadata(item, meta) )
+					item = [$str=query$str,$subtype=query$subtype,$meta=meta];
+					if ( match_item_with_query(item, query) )
+						return T;
+					}
+				}
+
+			# Check if there are any custom subtype matchers in case we haven't matched yet.
+			if ( query$subtype in custom_matchers )
+				{
+				for ( match_func in custom_matchers[query$subtype] )
+					{
+					if ( match_func(query) )
 						return T;
 					}
 				}
 			}
 
-		else if ( item?$num )
-			{
-			if ( [item$num, item$subtype] in data_store$int_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$int_data[item$num, item$subtype] )
-					{
-					meta = data_store$int_data[item$num, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
 		else
-			err_msg = "Failed to query intelligence data for some unknown reason.";
+			err_msg = "You must supply one of the $ip or $str fields to search on";
 		}
 		
 	if ( err_msg != "" )
 		Log::write(Intel::LOG, [$ts=network_time(), $level="error", $message=fmt(err_msg)]);
 	return F;
 	}
+
+module GLOBAL;
+
+function INTEL(item: Intel::Query): bool
+	{
+	return Intel::matcher(item);
+	}

scripts/base/frameworks/intel/plugins/dns_zones.bro

@@ -0,0 +1,53 @@
+
+module Intel;
+
+export {
+	redef enum SubType += {
+		DNS_ZONE,
+	};
+}
+
+function dns_zone_ripper(query: Query): Query
+	{
+	local query_copy = copy(query);
+	# We can assume that we're getting a string and subtype because
+	# this function is only registered for DOMAIN and DNS_ZONE data.
+	local dns_name = sub(query_copy$str, /^[^\.]*\./, "");
+	query_copy$str = dns_name;
+	# We are doing a literal search for a DNS zone at this point
+	query_copy$subtype = Intel::DNS_ZONE;
+	return query_copy;
+	}
+
+# This matcher extension adds additional matchers for domain names.
+function dns_zone_matcher(query: Query): bool
+	{
+	local query_copy = dns_zone_ripper(query);
+	if ( query$str == query_copy$str )
+		return F;
+
+	return Intel::matcher(query_copy);
+	}
+
+function dns_zone_lookup(query: Query): set[Item]
+	{
+	local result_set: set[Item] = set();
+	local query_copy = dns_zone_ripper(query);
+	if ( query$str == query_copy$str )
+		return result_set;
+
+	for ( item in Intel::lookup(query_copy) )
+		add result_set[item];
+	return result_set;
+	}
+
+event bro_init() &priority=10
+	{
+	register_custom_matcher(DOMAIN, dns_zone_matcher);
+	# The DNS_ZONE subtype needs added because it's ultimately 
+	# a subset of DOMAIN and will need to be searched as well.
+	register_custom_matcher(DNS_ZONE, dns_zone_matcher);
+
+	register_custom_lookup(DOMAIN, dns_zone_lookup);
+	register_custom_lookup(DNS_ZONE, dns_zone_lookup);
+	}

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1..stdout

@@ -0,0 +1,7 @@
+1.2.3.4
+{
+b,
+c,
+a
+}
+foobar

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/worker-1..stdout

@@ -0,0 +1,7 @@
+1.2.3.4
+{
+b,
+c,
+a
+}
+foobar

testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/worker-2..stdout

@@ -0,0 +1,7 @@
+1.2.3.4
+{
+b,
+c,
+a
+}
+foobar

testing/btest/Baseline/scripts.base.frameworks.intel.dns-zone-plugin/out

@@ -0,0 +1,3 @@
+It matched!
+bad.com
+Intel::DNS_ZONE

testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/out

@@ -0,0 +1 @@
+Matched it!

testing/btest/Baseline/scripts.base.frameworks.intel.item-merge/out

@@ -0,0 +1,3 @@
+Number of matching intel items: 2 (should be 2)
+Number of matching intel items: 2 (should still be 2)
+Number of matching intel items: 3 (should be 3)

testing/btest/Baseline/scripts.base.frameworks.intel.matching/out

@@ -0,0 +1,3 @@
+VALID
+VALID
+VALID

testing/btest/scripts/base/frameworks/intel/cluster-transparency.bro

@@ -0,0 +1,44 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 3
+# @TEST-EXEC: btest-diff manager-1/.stdout
+# @TEST-EXEC: btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff worker-2/.stdout
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1",$interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1",$interface="eth1"],
+};
+@TEST-END-FILE
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# Insert the data once both workers are connected.
+	if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 2 )
+		{
+		Intel::insert([$ip=1.2.3.4,$meta=[$source="foobar", $class=Intel::MALICIOUS, $tags=set("a","b","c")]]);
+		}
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 0 )
+		terminate_communication();
+	}
+
+# This should print out a single time on the manager and each worker 
+# due to the cluster transparency.
+event Intel::new_item(item: Intel::Item)
+	{
+	print item$ip;
+	print item$meta$tags;
+	print item$meta$source;
+
+	if ( Cluster::local_node_type() == Cluster::WORKER )
+		terminate_communication();
+	}

testing/btest/scripts/base/frameworks/intel/dns-zone-plugin.bro

@@ -0,0 +1,18 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	Intel::insert([$str="bad.com", $subtype=Intel::DNS_ZONE, $meta=[$source="src1", $class=Intel::MALICIOUS]]);
+	local query: Intel::Query = [$str="some.host.bad.com", $subtype=Intel::DOMAIN, $class=Intel::MALICIOUS];
+	if ( Intel::matcher(query) )
+		{
+		print "It matched!";
+		local items = Intel::lookup(query);
+		for ( item in items )
+			{
+			print item$str;
+			print item$subtype;
+			}
+		}
+	}

testing/btest/scripts/base/frameworks/intel/input-and-match.bro

@@ -0,0 +1,36 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE intel.dat
+#fields	ip	net	str	subtype	meta.source	meta.class	meta.desc	meta.url	meta.tags
+1.2.3.4	-	-	-	source1	Intel::MALICIOUS	this host is just plain baaad	http://some-data-distributor.com/1234	foo,bar
+1.2.3.4	-	-	-	source1	Intel::MALICIOUS	this host is just plain baaad	http://some-data-distributor.com/1234	foo,bar
+-	-	e@mail.com	Intel::EMAIL	source1	Intel::MALICIOUS	Phishing email source	http://some-data-distributor.com/100000	-
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef Intel::read_files += { "intel.dat" };
+
+event do_it(allowed_loops: count)
+	{
+	if ( Intel::matcher([$str="e@mail.com", $subtype=Intel::EMAIL, $class=Intel::MALICIOUS]) &&
+	     Intel::matcher([$ip=1.2.3.4, $class=Intel::MALICIOUS]) )
+		{
+		# Once the match happens a single time we print and shutdown.
+		print "Matched it!";
+		terminate_communication();
+		return;
+		}
+	
+	if ( allowed_loops > 0 )
+		schedule 100msecs { do_it(allowed_loops-1) };
+	else
+		terminate_communication();
+	}
+
+
+event bro_init()
+	{
+	event do_it(20);
+	}

testing/btest/scripts/base/frameworks/intel/insert-and-matcher.bro

@@ -1,34 +0,0 @@
-#
-# @TEST-EXEC: bro %INPUT >out
-# @TEST-EXEC: btest-diff out
-
-event bro_init()
-	{
-	Intel::insert([$ip=1.2.3.4, $tags=set("zeustracker.abuse.ch", "malicious")]);
-	Intel::insert([$str="http://www.google.com/", $subtype="url", $tags=set("infrastructure", "google")]);
-	Intel::insert([$str="Ab439G32F...", $subtype="x509_cert", $tags=set("bad")]);
-	Intel::insert([$str="Ab439G32F...", $tags=set("bad")]);
-	}
-
-event bro_done()
-	{
-	local orig_h = 1.2.3.4;
-	
-	if ( Intel::matcher([$ip=orig_h, $and_tags=set("malicious")]) )
-		print "VALID";
-	
-	if ( Intel::matcher([$ip=orig_h, $and_tags=set("don't match")]) )
-		print "INVALID";
-	
-	if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::MetaData): bool { return T; } ]) )
-		print "VALID";
-		
-	if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::MetaData): bool { return F; } ]) )
-		print "INVALID";
-		
-	if ( Intel::matcher([$str="http://www.google.com/", $subtype="url", $tags=set("google")]) )
-		print "VALID";
-		
-	if ( Intel::matcher([$str="http://www.example.com", $subtype="url"]) )
-		print "INVALID";
-	}

testing/btest/scripts/base/frameworks/intel/item-merge.bro

@@ -0,0 +1,23 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	Intel::insert([$ip=1.2.3.4, $meta=[$source="source1-feed1", $class=Intel::MALICIOUS, $tags=set("foo")]]);
+	Intel::insert([$ip=1.2.3.4, $meta=[$source="source2-special-sauce", $class=Intel::MALICIOUS, $tags=set("foo","bar")]]);
+
+	# Lookup should return the items matching the query.
+	local items = Intel::lookup([$ip=1.2.3.4]);
+	print fmt("Number of matching intel items: %d (should be 2)", |items|);
+
+	# This can be considered an update of a previous value since the
+	# data, source, and class are the matching points for determining sameness.
+	Intel::insert([$ip=1.2.3.4, $meta=[$source="source2-special-sauce", $class=Intel::MALICIOUS, $tags=set("foobar", "testing")]]);
+	items = Intel::lookup([$ip=1.2.3.4]);
+	print fmt("Number of matching intel items: %d (should still be 2)", |items|);
+
+	# This is a new value.
+	Intel::insert([$ip=1.2.3.4, $meta=[$source="source3", $class=Intel::MALICIOUS]]);
+	items = Intel::lookup([$ip=1.2.3.4]);
+	print fmt("Number of matching intel items: %d (should be 3)", |items|);
+	}

testing/btest/scripts/base/frameworks/intel/matching.bro

@@ -0,0 +1,38 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	Intel::insert([$ip=1.2.3.4, $meta=[$source="zeus-tracker", $class=Intel::MALICIOUS, $tags=set("example-tag1", "example-tag2")]]);
+	Intel::insert([$str="http://www.google.com/", $subtype=Intel::URL, $meta=[$source="source2", $class=Intel::MALICIOUS, $tags=set("infrastructure", "google")]]);
+	}
+
+event bro_done()
+	{
+	local orig_h = 1.2.3.4;
+	
+	if ( Intel::matcher([$ip=orig_h, $and_tags=set("example-tag1", "example-tag2")]) )
+		print "VALID";
+	
+	if ( Intel::matcher([$ip=orig_h, $and_tags=set("don't match")]) )
+		print "INVALID";
+	
+	if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::Item): bool { return T; } ]) )
+		print "VALID";
+		
+	if ( Intel::matcher([$ip=4.3.2.1, $pred=function(meta: Intel::Item): bool { return T; } ]) )
+		print "INVALID";
+
+	if ( Intel::matcher([$ip=orig_h, $pred=function(meta: Intel::Item): bool { return F; } ]) )
+		print "INVALID";
+		
+	if ( Intel::matcher([$str="http://www.google.com/", $subtype=Intel::URL, $and_tags=set("google")]) )
+		print "VALID";
+		
+	if ( Intel::matcher([$str="http://www.google.com/", $subtype=Intel::URL, $and_tags=set("woah")]) )
+		print "INVALID";
+
+	if ( Intel::matcher([$str="http://www.example.com", $subtype=Intel::URL]) )
+		print "INVALID";
+	}