From c5c650b486e5acc2594bdf45e73f7160629df0d0 Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Sat, 19 Mar 2016 17:02:52 +0100
Subject: [PATCH 01/11] Added testcase for intel updates.

By addind debug output to Intel::insert() the testcase reveals that
updating an intel item will cause its metadata to be inserted again,
without the old being deleted.
---
 .../output                                    | 25 ++++++++
 .../base/frameworks/intel/updated-match.bro   | 62 +++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
 create mode 100644 testing/btest/scripts/base/frameworks/intel/updated-match.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
new file mode 100644
index 0000000000..3e7fa97f40
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -0,0 +1,25 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-03-19-16-01-52
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+1458403312.669166	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
+1458403315.672095	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
+1458403315.672095	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
+1458403318.675592	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
+1458403318.675592	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
+#close	2016-03-19-16-01-58
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2016-03-19-16-01-58
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
+1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-03-19-16-01-58
diff --git a/testing/btest/scripts/base/frameworks/intel/updated-match.bro b/testing/btest/scripts/base/frameworks/intel/updated-match.bro
new file mode 100644
index 0000000000..75063d4b8f
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/updated-match.bro
@@ -0,0 +1,62 @@
+# @TEST-SERIALIZE: comm
+
+# @TEST-EXEC: cp intel1.dat intel.dat
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp intel2.dat intel.dat
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp intel3.dat intel.dat
+# @TEST-EXEC: btest-bg-wait 6
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/notice.log >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel1.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234	F
+# @TEST-END-FILE
+
+# @TEST-START-FILE intel2.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source2	this host is just plain baaad	http://some-data-distributor.com/1234	F
+4.3.2.1	Intel::ADDR	source2	this host might also be baaad	http://some-data-distributor.com/4321	F
+# @TEST-END-FILE
+
+# @TEST-START-FILE intel3.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source2	this host is just plain baaad	http://some-data-distributor.com/1234	T
+4.3.2.1	Intel::ADDR	source2	this host might also be baaad	http://some-data-distributor.com/4321	T
+# @TEST-END-FILE
+
+@load base/frameworks/communication # let network-time run
+@load frameworks/intel/do_notice
+
+redef exit_only_after_terminate = T;
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+global runs = 0;
+event do_it()
+	{
+	Intel::seen([$host=1.2.3.4,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=4.3.2.1,
+	             $where=SOMEWHERE]);
+
+	++runs;
+	if ( runs < 3 )
+		schedule 3sec { do_it() };
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 5 )
+		terminate();
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

From 0146e85c41613190b542d28e549d0c3974e08fa2 Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Sat, 19 Mar 2016 17:12:06 +0100
Subject: [PATCH 02/11] Refactoring of meta data handling for intel.

To simplify meta data handling inside the intel framework and avoid
duplicate insertion of meta data on update, meta data is stored in a
table indexed by meta data source.
---
 scripts/base/frameworks/intel/main.bro | 90 ++++++++++----------------
 1 file changed, 33 insertions(+), 57 deletions(-)

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index eba27ca56a..32b48a089b 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -151,16 +151,18 @@ global match_no_items: event(s: Seen);
 
 # Internal events for cluster data distribution.
 global new_item: event(item: Item);
-global updated_item: event(item: Item);
 
 # Optionally store metadata.  This is used internally depending on
 # if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
+# Table of meta data, indexed by source string.
+type MetaDataTable: table[string] of MetaData;
+
 # The in memory data structure for holding intelligence.
 type DataStore: record {
-	host_data:    table[addr] of set[MetaData];
-	string_data:  table[string, Type] of set[MetaData];
+	host_data:    table[addr] of MetaDataTable;
+	string_data:  table[string, Type] of MetaDataTable;
 };
 global data_store: DataStore &redef;
 
@@ -186,26 +188,23 @@ function find(s: Seen): bool
 		return ((s$host in min_data_store$host_data) || 
 		        (have_full_data && s$host in data_store$host_data));
 		}
-	else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
-	           (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) )
-		{
-		return T;
-		}
 	else
 		{
-		return F;
+		return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
+		        (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data));
 		}
 	}
 
+# Function to abstract from different data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
 	local return_data: set[Item];
+	local mt: MetaDataTable;
 
 	if ( ! have_full_data )
 		{
-		# A reporter warning should be generated here because this function
-		# should never be called from a host that doesn't have the full data.
-		# TODO: do a reporter warning.
+		Reporter::warning(fmt("Intel::get_items was called from a host (%s) that doesn't have the full data.",
+			peer_description));
 		return return_data;
 		}
 
@@ -214,9 +213,10 @@ function get_items(s: Seen): set[Item]
 		# See if the host is known about and it has meta values
 		if ( s$host in data_store$host_data )
 			{
-			for ( m in data_store$host_data[s$host] )
+			mt = data_store$host_data[s$host];
+			for ( m in mt )
 				{
-				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)];
+				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
 				}
 			}
 		}
@@ -226,9 +226,10 @@ function get_items(s: Seen): set[Item]
 		# See if the string is known about and it has meta values
 		if ( [lower_indicator, s$indicator_type] in data_store$string_data )
 			{
-			for ( m in data_store$string_data[lower_indicator, s$indicator_type] )
+			mt = data_store$string_data[lower_indicator, s$indicator_type];
+			for ( m in mt )
 				{
-				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)];
+				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=mt[m])];
 				}
 			}
 		}
@@ -263,20 +264,6 @@ function Intel::seen(s: Seen)
 		}
 	}
 
-
-function has_meta(check: MetaData, metas: set[MetaData]): bool
-	{
-	local check_hash = md5_hash(check);
-	for ( m in metas )
-		{
-		if ( check_hash == md5_hash(m) )
-			return T;
-		}
-
-	# The records must not be equivalent if we made it this far.
-	return F;
-	}
-
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
 	local info = Info($ts=network_time(), $seen=s);
@@ -315,7 +302,8 @@ function insert(item: Item)
 	{
 	# Create and fill out the meta data item.
 	local meta = item$meta;
-	local metas: set[MetaData];
+	local meta_tbl: table [string] of MetaData;
+	local is_new: bool = T;
 
 	# All intelligence is case insensitive at the moment.
 	local lower_indicator = to_lower(item$indicator);
@@ -326,9 +314,11 @@ function insert(item: Item)
 		if ( have_full_data )
 			{
 			if ( host !in data_store$host_data )
-				data_store$host_data[host] = set();
+				data_store$host_data[host] = table();
+			else
+				is_new = F;
 
-			metas = data_store$host_data[host];
+			meta_tbl = data_store$host_data[host];
 			}
 
 		add min_data_store$host_data[host];
@@ -338,39 +328,25 @@ function insert(item: Item)
 		if ( have_full_data )
 			{
 			if ( [lower_indicator, item$indicator_type] !in data_store$string_data )
-				data_store$string_data[lower_indicator, item$indicator_type] = set();
+				data_store$string_data[lower_indicator, item$indicator_type] = table();
+			else
+				is_new = F;
 
-			metas = data_store$string_data[lower_indicator, item$indicator_type];
+			meta_tbl = data_store$string_data[lower_indicator, item$indicator_type];
 			}
 
 		add min_data_store$string_data[lower_indicator, item$indicator_type];
 		}
 
-	local updated = F;
 	if ( have_full_data )
 		{
-		for ( m in metas )
-			{
-			if ( meta$source == m$source )
-				{
-				if ( has_meta(meta, metas) )
-					{
-					# It's the same item being inserted again.
-					return;
-					}
-				else
-					{
-					# Same source, different metadata means updated item.
-					updated = T;
-					}
-				}
-			}
-		add metas[item$meta];
+		# Insert new meta data or update if already present
+		meta_tbl[meta$source] = meta;
 		}
-	
-	if ( updated )
-		event Intel::updated_item(item);
-	else
+
+	if ( is_new )
+		# Trigger insert for cluster in case the item is new
+		# or insert was called on a worker
 		event Intel::new_item(item);
 	}
 	

From cafae5351bbfea8f767bf82fbb97b5317abd74df Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Tue, 22 Mar 2016 19:16:51 +0100
Subject: [PATCH 03/11] Added support for subnets to intel-framework.

The intel-framework now supports the new indicator type Intel::SUBNET.
As subnets are matched against seen addresses, the field matched was
introduced to indicate which indicator types caused the hit. A testcase
for subents was added and the old ones have been updated accordingly.
---
 scripts/base/frameworks/intel/main.bro        | 46 +++++++++++++++--
 .../manager-1.intel.log                       | 10 ++--
 .../broproc.intel.log                         | 12 ++---
 .../output                                    | 23 +++++++++
 .../manager-1.intel.log                       | 16 +++---
 .../output                                    | 26 +++++-----
 .../intel-all.log                             | 24 ++++-----
 .../base/frameworks/intel/match-subnet.bro    | 50 +++++++++++++++++++
 8 files changed, 158 insertions(+), 49 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
 create mode 100644 testing/btest/scripts/base/frameworks/intel/match-subnet.bro

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 32b48a089b..f3bceec25e 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -14,6 +14,8 @@ export {
 	type Type: enum {
 		## An IP address.
 		ADDR,
+		## A subnet in CIDR notation.
+		SUBNET,
 		## A complete URL without the prefix ``"http://"``.
 		URL,
 		## Software name.
@@ -35,7 +37,9 @@ export {
 		## Public key MD5 hash. (SSH server host keys are a good example.)
 		PUBKEY_HASH,
 	};
-	
+	## Set of intelligence data types.
+	type TypeSet: set[Type];
+
 	## Data about an :bro:type:`Intel::Item`.
 	type MetaData: record {
 		## An arbitrary string value representing the data source.
@@ -123,6 +127,8 @@ export {
 
 		## Where the data was seen.
 		seen:     Seen           &log;
+		## Which indicator types matched.
+		matched:  TypeSet        &log;
 		## Sources which supplied data that resulted in this match.
 		sources:  set[string]    &log &default=string_set();
 	};
@@ -162,6 +168,7 @@ type MetaDataTable: table[string] of MetaData;
 # The in memory data structure for holding intelligence.
 type DataStore: record {
 	host_data:    table[addr] of MetaDataTable;
+	subnet_data:  table[subnet] of MetaDataTable;
 	string_data:  table[string, Type] of MetaDataTable;
 };
 global data_store: DataStore &redef;
@@ -171,6 +178,7 @@ global data_store: DataStore &redef;
 # a minimal amount of data for the full match to happen on the manager.
 type MinDataStore: record {
 	host_data:    set[addr];
+	subnet_data:  set[subnet];
 	string_data:  set[string, Type];
 };
 global min_data_store: MinDataStore &redef;
@@ -186,12 +194,11 @@ function find(s: Seen): bool
 	if ( s?$host )
 		{
 		return ((s$host in min_data_store$host_data) || 
-		        (have_full_data && s$host in data_store$host_data));
+		        (|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0));
 		}
 	else
 		{
-		return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
-		        (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data));
+		return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data);
 		}
 	}
 
@@ -219,6 +226,17 @@ function get_items(s: Seen): set[Item]
 				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
 				}
 			}
+		# See if the host is part of a known subnet, which has meta values
+		local nets: table[subnet] of MetaDataTable;
+		nets = filter_subnet_table(addr_to_subnet(s$host), data_store$subnet_data);
+		for ( n in nets )
+			{
+				mt = nets[n];
+				for ( m in mt )
+					{
+					add return_data[Item($indicator=cat(n), $indicator_type=SUBNET, $meta=mt[m])];
+					}
+			}
 		}
 	else
 		{
@@ -266,7 +284,7 @@ function Intel::seen(s: Seen)
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info = Info($ts=network_time(), $seen=s);
+	local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
 
 	if ( s?$f )
 		{
@@ -293,7 +311,10 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		}
 
 	for ( item in items )
+		{
 		add info$sources[item$meta$source];
+		add info$matched[item$indicator_type];
+		}
 
 	Log::write(Intel::LOG, info);
 	}
@@ -323,6 +344,21 @@ function insert(item: Item)
 
 		add min_data_store$host_data[host];
 		}
+	else if ( item$indicator_type == SUBNET )
+		{
+		local net = to_subnet(item$indicator);
+		if ( have_full_data )
+			{
+			if ( net !in data_store$subnet_data )
+				data_store$subnet_data[net] = table();
+			else
+				is_new = F;
+
+			meta_tbl = data_store$subnet_data[net];
+			}
+
+		add min_data_store$subnet_data[net];
+		}
 	else
 		{
 		if ( have_full_data )
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
index ba19f4e8d7..015d2f21bd 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-13-39
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1411488819.555114	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	worker-1
-#close	2014-09-23-16-13-49
+#open	2016-03-22-18-11-20
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670280.078658	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1
+#close	2016-03-22-18-11-29
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
index 33c97c0c1e..4436253d96 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-14-49
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1411488889.571819	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	source1
-1411488889.571819	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
-#close	2014-09-23-16-14-49
+#open	2016-03-22-18-11-32
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670292.167298	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1
+1458670292.167298	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+#close	2016-03-22-18-11-32
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
new file mode 100644
index 0000000000..3fbd90949f
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-03-22-18-11-34
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670294.227182	-	-	-	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1458670294.227182	-	-	-	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1
+1458670294.227182	-	-	-	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1
+#close	2016-03-22-18-11-34
+
+Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
+Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
+
+Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
+Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
+
+Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
+Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
+Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
+Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
index d8e2d43674..074e6a0122 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-15-00
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1411488900.900403	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	source1
-1411488900.900403	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	source1
-1411488901.923543	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	source1
-1411488901.923543	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	source1
-#close	2014-09-23-16-15-09
+#open	2016-03-22-18-11-40
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670300.363597	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1
+1458670300.363597	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1
+1458670301.370555	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1
+1458670301.370555	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1
+#close	2016-03-22-18-11-49
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
index 3e7fa97f40..be3604f541 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-19-16-01-52
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1458403312.669166	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
-1458403315.672095	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
-1458403315.672095	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
-1458403318.675592	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
-1458403318.675592	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
-#close	2016-03-19-16-01-58
+#open	2016-03-22-18-11-51
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670311.505318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1458670314.509318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
+1458670314.509318	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
+1458670317.513183	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
+1458670317.513183	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
+#close	2016-03-22-18-11-57
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2016-03-19-16-01-58
+#open	2016-03-22-18-11-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
-1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2016-03-19-16-01-58
+1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-03-22-18-11-57
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
index ba1afe4239..ac88efb3d6 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
@@ -3,20 +3,20 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2015-03-14-01-47-46
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	source1
-#close	2015-03-14-01-47-46
+#open	2016-03-22-18-09-35
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+#close	2016-03-22-18-09-35
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2015-03-14-01-47-46
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-#close	2015-03-14-01-47-46
+#open	2016-03-22-18-09-35
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+#close	2016-03-22-18-09-35
diff --git a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
new file mode 100644
index 0000000000..924fa947b6
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
@@ -0,0 +1,50 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+192.168.1.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1
+192.168.2.0/24	Intel::SUBNET	source1	this subnetwork is just plain baaad	http://some-data-distributor.com/2
+192.168.142.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/3
+192.168.142.0/24	Intel::SUBNET	source1	this subnetwork is baaad	http://some-data-distributor.com/4
+192.168.128.0/18	Intel::SUBNET	source1	this subnetwork might be baaad	http://some-data-distributor.com/5
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+event do_it()
+	{
+	Intel::seen([$host=192.168.1.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=192.168.2.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=192.168.142.1,
+	             $where=SOMEWHERE]);
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 2 )
+		terminate();
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print "";
+	print fmt("Seen: %s", s);
+	for ( item in items )
+		print fmt("Item: %s", item);
+	}
\ No newline at end of file

From 2ebac7078251cee2ef3f4607dcbe28c12e9165ad Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 30 Mar 2016 20:03:07 +0200
Subject: [PATCH 04/11] Added remove function to intel-framework.

---
 scripts/base/frameworks/intel/cluster.bro     | 20 ++--
 scripts/base/frameworks/intel/main.bro        | 95 ++++++++++++++++++-
 .../manager-1..stdout                         |  6 ++
 .../manager-1.intel.log                       | 10 ++
 .../worker-1..stdout                          |  5 +
 .../frameworks/intel/remove-item-cluster.bro  | 88 +++++++++++++++++
 6 files changed, 213 insertions(+), 11 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout
 create mode 100644 testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro

diff --git a/scripts/base/frameworks/intel/cluster.bro b/scripts/base/frameworks/intel/cluster.bro
index 7791c334d5..e8fae8327c 100644
--- a/scripts/base/frameworks/intel/cluster.bro
+++ b/scripts/base/frameworks/intel/cluster.bro
@@ -20,16 +20,11 @@ redef have_full_data = F;
 global cluster_new_item: event(item: Item);
 
 # Primary intelligence distribution comes from manager.
-redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
+redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/;
 # If a worker finds intelligence and adds it, it should share it back to the manager.
-redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
+redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
-event Intel::match_no_items(s: Seen) &priority=5
-	{
-	event Intel::match(s, Intel::get_items(s));
-	}
-
 event remote_connection_handshake_done(p: event_peer)
 	{
 	# When a worker connects, send it the complete minimal data store.
@@ -39,6 +34,17 @@ event remote_connection_handshake_done(p: event_peer)
 		send_id(p, "Intel::min_data_store");
 		}
 	}
+
+event Intel::match_no_items(s: Seen) &priority=5
+	{
+	if ( Intel::find(s) )
+		event Intel::match(s, Intel::get_items(s));
+	}
+
+event Intel::remove_item(item: Item, purge_indicator: bool)
+	{
+	remove(item, purge_indicator);
+	}
 @endif
 
 event Intel::cluster_new_item(item: Intel::Item) &priority=5
diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index f3bceec25e..8e387f71f6 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -136,6 +136,10 @@ export {
 	## Intelligence data manipulation function.
 	global insert: function(item: Item);
 
+	## Function to remove intelligence data. If purge_indicator is set, the
+	## given meta data is ignored and the indicator is removed completely.
+	global remove: function(item: Item, purge_indicator: bool &default = F);
+
 	## Function to declare discovery of a piece of data in order to check
 	## it against known intelligence for matches.
 	global seen: function(s: Seen);
@@ -157,6 +161,8 @@ global match_no_items: event(s: Seen);
 
 # Internal events for cluster data distribution.
 global new_item: event(item: Item);
+global remove_item: event(item: Item, purge_indicator: bool);
+global purge_item: event(item: Item);
 
 # Optionally store metadata.  This is used internally depending on
 # if this is a cluster deployment or not.
@@ -191,14 +197,16 @@ event bro_init() &priority=5
 
 function find(s: Seen): bool
 	{
+	local ds = have_full_data ? data_store : min_data_store;
+
 	if ( s?$host )
 		{
-		return ((s$host in min_data_store$host_data) || 
-		        (|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0));
+		return ((s$host in ds$host_data) ||
+		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
 		}
 	else
 		{
-		return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data);
+		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
 		}
 	}
 
@@ -385,4 +393,83 @@ function insert(item: Item)
 		# or insert was called on a worker
 		event Intel::new_item(item);
 	}
-	
+
+# Function to remove meta data of an item. The function returns T
+# if there is no meta data left for the given indicator.
+function remove_meta_data(item: Item): bool
+	{
+	if ( ! have_full_data )
+		{
+		Reporter::warning(fmt("Intel::remove_meta_data was called from a host (%s) that doesn't have the full data.",
+			peer_description));
+		return F;
+		}
+
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			local host = to_addr(item$indicator);
+			delete data_store$host_data[host][item$meta$source];
+			return (|data_store$host_data[host]| == 0);
+		case SUBNET:
+			local net = to_subnet(item$indicator);
+			delete data_store$subnet_data[net][item$meta$source];
+			return (|data_store$subnet_data[net]| == 0);
+		default:
+			delete data_store$string_data[item$indicator, item$indicator_type][item$meta$source];
+			return (|data_store$string_data[item$indicator, item$indicator_type]| == 0);
+		}
+	}
+
+function remove(item: Item, purge_indicator: bool)
+	{
+	# Delegate removal if we are on a worker
+	if ( !have_full_data )
+		{
+		event Intel::remove_item(item, purge_indicator);
+		return;
+		}
+
+	# Remove meta data from manager's data store
+	local no_meta_data = remove_meta_data(item);
+	# Remove whole indicator if necessary
+	if ( no_meta_data || purge_indicator )
+		{
+		switch ( item$indicator_type )
+			{
+			case ADDR:
+				local host = to_addr(item$indicator);
+				delete data_store$host_data[host];
+				break;
+			case SUBNET:
+				local net = to_subnet(item$indicator);
+				delete data_store$subnet_data[net];
+				break;
+			default:
+				delete data_store$string_data[item$indicator, item$indicator_type];
+				break;
+			}
+		# Trigger deletion in min data stores
+		event Intel::purge_item(item);
+		}
+	}
+
+event purge_item(item: Item)
+	{
+	# Remove data from min data store
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			local host = to_addr(item$indicator);
+			delete min_data_store$host_data[host];
+			break;
+		case SUBNET:
+			local net = to_subnet(item$indicator);
+			delete min_data_store$subnet_data[net];
+			break;
+		default:
+			delete min_data_store$string_data[item$indicator, item$indicator_type];
+			break;
+		}
+	}
+
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout
new file mode 100644
index 0000000000..17862ce14b
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout
@@ -0,0 +1,6 @@
+Purging 192.168.0.1.
+Purging 192.168.0.2.
+Removing 192.168.1.2 (source: source1).
+Removing 192.168.1.2 (source: source2).
+Purging 192.168.1.2.
+Logging intel hit!
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
new file mode 100644
index 0000000000..bb3541ba32
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-03-30-16-01-31
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1459353691.470304	-	-	-	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end
+#close	2016-03-30-16-01-41
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout
new file mode 100644
index 0000000000..042032cb9d
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout
@@ -0,0 +1,5 @@
+Removing 192.168.1.2 (source: source1).
+Removing 192.168.1.2 (source: source2).
+Purging 192.168.0.1.
+Purging 192.168.0.2.
+Purging 192.168.1.2.
diff --git a/testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro b/testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro
new file mode 100644
index 0000000000..d13536a015
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro
@@ -0,0 +1,88 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff manager-1/.stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff manager-1/intel.log
+
+# @TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"],
+};
+# @TEST-END-FILE
+
+@load base/frameworks/control
+
+module Intel;
+
+redef Log::default_rotation_interval=0sec;
+
+event test_manager()
+	{
+	Intel::remove([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::seen([$host=192.168.0.1, $where=Intel::IN_ANYWHERE]);
+	Intel::remove([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]], T);
+	Intel::seen([$host=192.168.0.2, $where=Intel::IN_ANYWHERE]);
+	}
+
+event test_worker()
+	{
+	Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+	Intel::seen([$host=192.168.1.2, $where=Intel::IN_ANYWHERE]);
+	# Trigger shutdown by matching data that should be present
+	Intel::seen([$host=10.10.10.10, $where=Intel::IN_ANYWHERE]);
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# Insert the data once all workers are connected.
+	if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 1 )
+		{
+		Intel::insert([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+		Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+		Intel::insert([$indicator="10.10.10.10", $indicator_type=Intel::ADDR, $meta=[$source="end"]]);
+
+		event test_manager();
+		}
+	}
+
+global worker_data = 0;
+event Intel::cluster_new_item(item: Intel::Item)
+	{
+	# Run test on worker-1 when all items have been inserted
+	if ( Cluster::node == "worker-1" )
+		{
+		++worker_data;
+		if ( worker_data == 4 )
+			event test_worker();
+		}
+	}
+
+event Intel::remove_item(item: Item, purge_indicator: bool)
+	{
+	print fmt("Removing %s (source: %s).", item$indicator, item$meta$source);
+	}
+
+event purge_item(item: Item)
+	{
+	print fmt("Purging %s.", item$indicator);
+	}
+
+event Intel::log_intel(rec: Intel::Info)
+	{
+	print "Logging intel hit!";
+	event Control::shutdown_request();
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	# Cascading termination
+	terminate_communication();
+	}

From cb330287022b506d870cd970e6f0aa1a810bb08e Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 11 May 2016 23:27:51 +0200
Subject: [PATCH 05/11] Added hook to allow extending the intel log.

The extension mechanism is basically the one that Seth introduced with
his intel extensions. The main difference lies in using a hook instead
of an event. An example policy implements whitelisting.
---
 scripts/base/frameworks/intel/main.bro        | 21 +++++++++-
 scripts/policy/frameworks/intel/whitelist.bro | 30 ++++++++++++++
 scripts/test-all-policy.bro                   |  1 +
 .../intel.log                                 | 29 ++++++++++++++
 .../policy/frameworks/intel/whitelisting.bro  | 39 +++++++++++++++++++
 5 files changed, 118 insertions(+), 2 deletions(-)
 create mode 100644 scripts/policy/frameworks/intel/whitelist.bro
 create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
 create mode 100644 testing/btest/scripts/policy/frameworks/intel/whitelisting.bro

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index b52b30aff0..55494507a7 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -165,6 +165,19 @@ export {
 	## data within the intelligence framework.
 	global match: event(s: Seen, items: set[Item]);
 
+	## This hook can be used to extend the intel log by adding data to the
+	## Info record. The default information is added with a priority of 5.
+	##
+	## info: The Info record that will be logged.
+	##
+	## s: Information about the data seen.
+	##
+	## items: The intel items that match the seen data.
+	##
+	## In case the hook execution is terminated using break, the match will
+	## not be logged.
+	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -306,6 +319,12 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
 	local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
 
+	if ( hook extend_match(info, s, items) )
+		Log::write(Intel::LOG, info);
+	}
+
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+	{
 	if ( s?$f )
 		{
 		s$fuid = s$f$id;
@@ -340,8 +359,6 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		add info$sources[item$meta$source];
 		add info$matched[item$indicator_type];
 		}
-
-	Log::write(Intel::LOG, info);
 	}
 
 function insert(item: Item)
diff --git a/scripts/policy/frameworks/intel/whitelist.bro b/scripts/policy/frameworks/intel/whitelist.bro
new file mode 100644
index 0000000000..9061ed2a91
--- /dev/null
+++ b/scripts/policy/frameworks/intel/whitelist.bro
@@ -0,0 +1,30 @@
+
+@load base/frameworks/intel
+@load base/frameworks/notice
+
+module Intel;
+
+export {
+	redef record Intel::MetaData += {
+		## Add a field to indicate if this is a whitelisted item.
+		whitelist: bool &default=F;
+	};
+}
+
+hook Intel::extend_match(info: Info, s: Seen, items: set[Item]) &priority=9
+	{
+	local whitelisted = F;
+	for ( item in items )
+		{
+		if ( item$meta$whitelist )
+			{
+			whitelisted = T;
+			break;
+			}
+		}
+	
+	if ( whitelisted )
+		# Prevent logging
+		break;
+	}
+
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index f85fdb58b0..02602d1dc6 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -15,6 +15,7 @@
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
 @load frameworks/intel/do_notice.bro
+@load frameworks/intel/whitelist.bro
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
new file mode 100644
index 0000000000..2aabd3b2e5
--- /dev/null
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
@@ -0,0 +1,29 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-05-11-19-38-30
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+#close	2016-05-11-19-38-30
diff --git a/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro b/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro
new file mode 100644
index 0000000000..53acd49aa9
--- /dev/null
+++ b/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro
@@ -0,0 +1,39 @@
+# @TEST-EXEC: bro -Cr $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-diff intel.log
+
+#@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+upload.wikimedia.org	Intel::DOMAIN	source1	somehow bad	http://some-data-distributor.com/1
+meta.wikimedia.org	Intel::DOMAIN	source1	also bad	http://some-data-distributor.com/1
+#@TEST-END-FILE
+
+#@TEST-START-FILE whitelist.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.whitelist	meta.url
+meta.wikimedia.org	Intel::DOMAIN	source2	also bad	T	http://some-data-distributor.com/1
+#@TEST-END-FILE
+
+@load base/frameworks/intel
+@load frameworks/intel/whitelist
+@load frameworks/intel/seen
+
+redef Intel::read_files += {
+	"intel.dat",
+	"whitelist.dat",
+};
+
+global total_files_read = 0;
+
+event bro_init()
+	{
+	suspend_processing();
+	}
+
+event Input::end_of_data(name: string, source: string)
+	{
+	# Wait until both intel files are read.
+	if ( /^intel-/ in name && (++total_files_read == 2) )
+		{
+		continue_processing();
+		}
+	}
+

From a9ad41cdccdfecffc4dda453da3223d203f48658 Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Thu, 19 May 2016 21:03:36 +0200
Subject: [PATCH 06/11] Improved intel notices.

Intel notices are identified by a direction independent 3-tuple
(indicator, originator IP, responder IP). This allows notice
suppression. Additionally service and intel source are added to the
notice mail.
---
 scripts/policy/frameworks/intel/do_notice.bro | 37 ++++++++++++++++---
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/scripts/policy/frameworks/intel/do_notice.bro b/scripts/policy/frameworks/intel/do_notice.bro
index 89910ede32..ed859b468b 100644
--- a/scripts/policy/frameworks/intel/do_notice.bro
+++ b/scripts/policy/frameworks/intel/do_notice.bro
@@ -6,14 +6,14 @@ module Intel;
 
 export {
 	redef enum Notice::Type += {
-		## Intel::Notice is a notice that happens when an intelligence 
+		## Intel::Notice is a notice that happens when an intelligence
 		## indicator is denoted to be notice-worthy.
 		Intel::Notice
 	};
 
 	redef record Intel::MetaData += {
 		## A boolean value to allow the data itself to represent
-		## if the indicator that this metadata is attached to 
+		## if the indicator that this metadata is attached to
 		## is notice worthy.
 		do_notice: bool &default=F;
 
@@ -29,15 +29,42 @@ event Intel::match(s: Seen, items: set[Item])
 	for ( item in items )
 		{
 		if ( item$meta$do_notice &&
-		     (! item$meta?$if_in || s$where == item$meta$if_in) )
+			(! item$meta?$if_in || s$where == item$meta$if_in) )
 			{
 			local n = Notice::Info($note=Intel::Notice,
-			                       $msg=fmt("Intel hit on %s at %s", s$indicator, s$where),
-			                       $sub=s$indicator);
+				$msg = fmt("Intel hit on %s at %s", s$indicator, s$where),
+				$sub = s$indicator);
+			local service_str = "";
 
 			if ( s?$conn )
+				{
 				n$conn = s$conn;
 
+				# Add identifier composed of indicator, originator's and responder's IP,
+				# without considering the direction of the flow.
+				local intel_id = s$indicator;
+				if( s$conn?$id )
+					{
+					if( s$conn$id$orig_h < s$conn$id$resp_h)
+						intel_id = cat(intel_id, s$conn$id$orig_h, s$conn$id$resp_h);
+					else
+						intel_id = cat(intel_id, s$conn$id$resp_h, s$conn$id$orig_h);
+					}
+				n$identifier = intel_id;
+
+				if ( s$conn?$service )
+					{
+					for ( service in s$conn$service )
+						service_str = cat(service_str, service, " ");
+					}
+				}
+
+			# Add additional information to the generated mail
+			local mail_ext = vector(
+				fmt("Service: %s\n", service_str),
+				fmt("Intel source: %s\n", item$meta$source));
+			n$email_body_sections = mail_ext;
+
 			NOTICE(n);
 			}
 		}

From 5d340e669c9d41bc9f7dd3aea0a52624de115cde Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 15 Jun 2016 19:19:13 +0200
Subject: [PATCH 07/11] Added expiration for intelligence items.

Expiration of intelligence items can be configured using
Intel::item_expiration. Expiration can be handled using the
Intel::item_expired hook.
---
 scripts/base/frameworks/intel/main.bro        | 74 ++++++++++++++++++-
 scripts/policy/frameworks/intel/do_expire.bro | 10 +++
 scripts/test-all-policy.bro                   |  1 +
 .../output                                    | 22 ++++++
 .../base/frameworks/intel/expire-item.bro     | 46 ++++++++++++
 5 files changed, 150 insertions(+), 3 deletions(-)
 create mode 100644 scripts/policy/frameworks/intel/do_expire.bro
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
 create mode 100644 testing/btest/scripts/base/frameworks/intel/expire-item.bro

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 55494507a7..6d4e19b222 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -178,6 +178,24 @@ export {
 	## not be logged.
 	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
 
+	## The expiration timeout for intelligence items. Once an item expires, the
+	## :bro:id:`item_expired` hook is called. Reinsertion of an item resets the
+	## timeout. A negative value disables expiration of intelligence items.
+	const item_expiration = -1 min &redef;
+
+	## This hook can be used to handle expiration of intelligence items.
+	##
+	## indicator: The indicator of the expired item.
+	##
+	## indicator_type: The indicator type of the expired item.
+	##
+	## metas: The set of meta data describing the expired item.
+	##
+	## If all hook handlers are executed, the expiration timeout will be reset.
+	## Otherwise, if one of the handlers terminates using break, the item will
+	## be removed.
+	global item_expired: hook(indicator: string, indicator_type: Type, metas: set[MetaData]);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -196,11 +214,16 @@ const have_full_data = T &redef;
 # Table of meta data, indexed by source string.
 type MetaDataTable: table[string] of MetaData;
 
+# Expiration handlers.
+global expire_host_data: function(data: table[addr] of MetaDataTable, idx: addr): interval;
+global expire_subnet_data: function(data: table[subnet] of MetaDataTable, idx: subnet): interval;
+global expire_string_data: function(data: table[string, Type] of MetaDataTable, idx: any): interval;
+
 # The in memory data structure for holding intelligence.
 type DataStore: record {
-	host_data:    table[addr] of MetaDataTable;
-	subnet_data:  table[subnet] of MetaDataTable;
-	string_data:  table[string, Type] of MetaDataTable;
+	host_data:    table[addr] of MetaDataTable &write_expire=item_expiration &expire_func=expire_host_data;
+	subnet_data:  table[subnet] of MetaDataTable &write_expire=item_expiration &expire_func=expire_subnet_data;
+	string_data:  table[string, Type] of MetaDataTable &write_expire=item_expiration &expire_func=expire_string_data;
 };
 global data_store: DataStore &redef;
 
@@ -235,6 +258,51 @@ function find(s: Seen): bool
 		}
 	}
 
+# Function that abstracts expiration of different types.
+function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval
+	{
+	if ( hook item_expired(indicator, indicator_type, metas) )
+		return item_expiration;
+	else
+		remove([$indicator=indicator, $indicator_type=indicator_type, $meta=[$source=""]], T);
+	return 0 sec;
+	}
+
+# Expiration handler definitions.
+function expire_host_data(data: table[addr] of MetaDataTable, idx: addr): interval
+	{
+	local meta_tbl: MetaDataTable = data[idx];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(cat(idx), ADDR, metas);
+	}
+
+function expire_subnet_data(data: table[subnet] of MetaDataTable, idx: subnet): interval
+	{
+	local meta_tbl: MetaDataTable = data[idx];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(cat(idx), ADDR, metas);
+	}
+
+function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any): interval
+	{
+	local indicator: string;
+	local indicator_type: Type;
+	[indicator, indicator_type] = idx;
+
+	local meta_tbl: MetaDataTable = data[indicator, indicator_type];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(indicator, indicator_type, metas);
+	}
+
 # Function to abstract from different data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro
new file mode 100644
index 0000000000..b5f47c2ade
--- /dev/null
+++ b/scripts/policy/frameworks/intel/do_expire.bro
@@ -0,0 +1,10 @@
+
+@load base/frameworks/intel
+
+module Intel;
+
+hook item_expired(indicator: string, indicator_type: Type,
+	metas: set[MetaData]) &priority=-10
+	{
+	break;
+	}
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index 3ea017bb95..1513e33289 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -15,6 +15,7 @@
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
 @load frameworks/intel/do_notice.bro
+@load frameworks/intel/do_expire.bro
 @load frameworks/intel/whitelist.bro
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
new file mode 100644
index 0000000000..e605be1d5e
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
@@ -0,0 +1,22 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-09-19-48-59
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1465501739.703996	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1465501740.704649	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1465501741.705204	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+#close	2016-06-09-19-49-05
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Expired: 1.2.3.4
+Trigger: 1.2.3.4
+Trigger: 1.2.3.4
+Trigger: 1.2.3.4
diff --git a/testing/btest/scripts/base/frameworks/intel/expire-item.bro b/testing/btest/scripts/base/frameworks/intel/expire-item.bro
new file mode 100644
index 0000000000..d56ef504f5
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/expire-item.bro
@@ -0,0 +1,46 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 7
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.4	Intel::ADDR	source1	this host is bad	http://some-data-distributor.com/1
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+@load frameworks/intel/do_expire
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+redef Intel::item_expiration = 3sec;
+redef table_expire_interval = 1sec;
+
+global runs = 0;
+event do_it()
+	{
+	print "Trigger: 1.2.3.4";
+	Intel::seen([$host=1.2.3.4,
+	             $where=SOMEWHERE]);
+
+	++runs;
+	if ( runs < 6 )
+		schedule 1sec { do_it() };
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print fmt("Seen: %s", s$indicator);
+	}
+
+hook Intel::item_expired(indicator: string, indicator_type: Intel::Type,
+	metas: set[Intel::MetaData])
+	{
+	print fmt("Expired: %s", indicator);
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

From 1412de17989c29cc3ca9916f755462dd460ce10a Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 15 Jun 2016 21:56:53 +0200
Subject: [PATCH 08/11] Refactored FAF integration of intel framework.

File Analysis Framework related code has been moved into a separate
script. Using redefinitions of the corresponding records causes the
file-related columns to appear last.
---
 scripts/base/frameworks/intel/__load__.bro    |  3 +
 scripts/base/frameworks/intel/files.bro       | 84 +++++++++++++++++++
 scripts/base/frameworks/intel/main.bro        | 66 ---------------
 .../canonified_loaded_scripts.log             |  5 +-
 .../manager-1.intel.log                       | 10 +--
 .../output                                    | 14 ++--
 .../broproc.intel.log                         | 12 +--
 .../output                                    | 14 ++--
 .../manager-1.intel.log                       | 16 ++--
 .../manager-1.intel.log                       | 10 +--
 .../output                                    | 26 +++---
 .../intel-all.log                             | 30 +++----
 .../intel.log                                 | 48 +++++------
 13 files changed, 180 insertions(+), 158 deletions(-)
 create mode 100644 scripts/base/frameworks/intel/files.bro

diff --git a/scripts/base/frameworks/intel/__load__.bro b/scripts/base/frameworks/intel/__load__.bro
index d8c77b86e3..d1cb61a7e2 100644
--- a/scripts/base/frameworks/intel/__load__.bro
+++ b/scripts/base/frameworks/intel/__load__.bro
@@ -1,5 +1,8 @@
 @load ./main
 
+# File analysis framework integration.
+@load ./files
+
 # The cluster framework must be loaded first.
 @load base/frameworks/cluster
 
diff --git a/scripts/base/frameworks/intel/files.bro b/scripts/base/frameworks/intel/files.bro
new file mode 100644
index 0000000000..89d708cc1e
--- /dev/null
+++ b/scripts/base/frameworks/intel/files.bro
@@ -0,0 +1,84 @@
+##! File analysis framework integration for the intelligence framework. This
+##! script manages file information in intelligence framework datastructures.
+
+@load ./main
+
+module Intel;
+
+export {
+	## Enum type to represent various types of intelligence data.
+	redef enum Type += {
+		## File hash which is non-hash type specific.  It's up to the
+		## user to query for any relevant hash types.
+		FILE_HASH,
+		## File name.  Typically with protocols with definite
+		## indications of a file name.
+		FILE_NAME,
+	};
+
+	## Information about a piece of "seen" data.
+	redef record Seen += {
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:              fa_file  &optional;
+		## If the data was discovered within a file, the file uid should
+		## go here to provide context to the data. If the *f* field is
+		## provided, this will be automatically filled out.
+		fuid:           string   &optional;
+	};
+
+	## Record used for the logging framework representing a positive
+	## hit within the intelligence framework.
+	redef record Info += {
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string   &log &optional;
+		## A mime type if the intelligence hit is related to a file.
+		## If the $f field is provided this will be automatically filled
+		## out.
+		file_mime_type: string   &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically
+		## filled out.
+		file_desc:      string   &log &optional;
+	};
+}
+
+# Add file information to matches if available.
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+	{
+	if ( s?$f )
+		{
+		s$fuid = s$f$id;
+
+		if ( s$f?$conns && |s$f$conns| == 1 )
+			{
+			for ( cid in s$f$conns )
+				s$conn = s$f$conns[cid];
+			}
+
+		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
+			info$file_mime_type = s$f$info$mime_type;
+
+		if ( ! info?$file_desc )
+			info$file_desc = Files::describe(s$f);
+		}
+
+	if ( s?$fuid )
+		info$fuid = s$fuid;
+
+	if ( s?$conn )
+		{
+		s$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	if ( s?$uid )
+		info$uid = s$uid;
+
+	for ( item in items )
+		{
+		add info$sources[item$meta$source];
+		add info$matched[item$indicator_type];
+		}
+	}
diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 6d4e19b222..08dea9bb2f 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -26,12 +26,6 @@ export {
 		DOMAIN,
 		## A user name.
 		USER_NAME,
-		## File hash which is non-hash type specific.  It's up to the
-		## user to query for any relevant hash types.
-		FILE_HASH,
-		## File name.  Typically with protocols with definite
-		## indications of a file name.
-		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
 		## Public key MD5 hash. (SSH server host keys are a good example.)
@@ -100,15 +94,6 @@ export {
 		## If the *conn* field is provided, this will be automatically
 		## filled out.
 		uid:             string        &optional;
-
-		## If the data was discovered within a file, the file record
-		## should go here to provide context to the data.
-		f:               fa_file       &optional;
-
-		## If the data was discovered within a file, the file uid should
-		## go here to provide context to the data. If the *f* field is
-		## provided, this will be automatically filled out.
-		fuid:            string        &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -124,19 +109,6 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
-		## If a file was associated with this intelligence hit,
-		## this is the uid for the file.
-		fuid:           string   &log &optional;
-
-		## A mime type if the intelligence hit is related to a file.
-		## If the $f field is provided this will be automatically filled
-		## out.
-		file_mime_type: string   &log &optional;
-		## Frequently files can be "described" to give a bit more context.
-		## If the $f field is provided this field will be automatically
-		## filled out.
-		file_desc:      string   &log &optional;
-
 		## Where the data was seen.
 		seen:     Seen           &log;
 		## Which indicator types matched.
@@ -391,44 +363,6 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		Log::write(Intel::LOG, info);
 	}
 
-hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
-	{
-	if ( s?$f )
-		{
-		s$fuid = s$f$id;
-
-		if ( s$f?$conns && |s$f$conns| == 1 )
-			{
-			for ( cid in s$f$conns )
-				s$conn = s$f$conns[cid];
-			}
-
-		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
-			info$file_mime_type = s$f$info$mime_type;
-
-		if ( ! info?$file_desc )
-			info$file_desc = Files::describe(s$f);
-		}
-
-	if ( s?$fuid )
-		info$fuid = s$fuid;
-
-	if ( s?$conn )
-		{
-		s$uid = s$conn$uid;
-		info$id  = s$conn$id;
-		}
-
-	if ( s?$uid )
-		info$uid = s$uid;
-
-	for ( item in items )
-		{
-		add info$sources[item$meta$source];
-		add info$matched[item$indicator_type];
-		}
-	}
-
 function insert(item: Item)
 	{
 	# Create and fill out the meta data item.
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index da62a25117..6469b71726 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-06-07-19-22-42
+#open	2016-06-15-19-16-09
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -177,6 +177,7 @@ scripts/base/init-default.bro
     scripts/base/frameworks/communication/main.bro
   scripts/base/frameworks/intel/__load__.bro
     scripts/base/frameworks/intel/main.bro
+    scripts/base/frameworks/intel/files.bro
     scripts/base/frameworks/intel/input.bro
   scripts/base/frameworks/sumstats/__load__.bro
     scripts/base/frameworks/sumstats/main.bro
@@ -310,4 +311,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-06-07-19-22-42
+#close	2016-06-15-19-16-09
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
index 015d2f21bd..48df37a6ec 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-20
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670280.078658	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1
-#close	2016-03-22-18-11-29
+#open	2016-06-15-19-11-27
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017887.060652	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1	-	-	-
+#close	2016-06-15-19-11-36
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
index e605be1d5e..dfa922f88f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-09-19-48-59
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1465501739.703996	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1465501740.704649	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1465501741.705204	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-#close	2016-06-09-19-49-05
+#open	2016-06-15-19-11-06
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017866.348490	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017867.349583	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017868.349656	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2016-06-15-19-11-12
 Trigger: 1.2.3.4
 Seen: 1.2.3.4
 Trigger: 1.2.3.4
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
index 4436253d96..7c29bb659e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-32
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670292.167298	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1
-1458670292.167298	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-#close	2016-03-22-18-11-32
+#open	2016-06-15-19-12-26
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017946.413077	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1	-	-	-
+1466017946.413077	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2016-06-15-19-12-26
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
index 949d428cd1..c20a053bca 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-16-59-39
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1462985979.596867	-	-	-	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1462985979.596867	-	-	-	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1
-1462985979.596867	-	-	-	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1
-#close	2016-05-11-16-59-39
+#open	2016-06-15-19-14-07
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466018047.083068	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466018047.083068	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1466018047.083068	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
+#close	2016-06-15-19-14-07
 
 Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
 Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
index 074e6a0122..12b07e116e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-40
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670300.363597	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1
-1458670300.363597	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1
-1458670301.370555	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1
-1458670301.370555	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1
-#close	2016-03-22-18-11-49
+#open	2016-06-15-19-14-30
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466018070.494693	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1	-	-	-
+1466018070.494693	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1	-	-	-
+1466018071.505800	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1	-	-	-
+1466018071.505800	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1	-	-	-
+#close	2016-06-15-19-14-39
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
index bb3541ba32..b7b7118004 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-30-16-01-31
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1459353691.470304	-	-	-	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end
-#close	2016-03-30-16-01-41
+#open	2016-06-15-19-10-09
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017809.810005	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end	-	-	-
+#close	2016-06-15-19-10-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
index be3604f541..8c8e9d9c0f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-51
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670311.505318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1458670314.509318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
-1458670314.509318	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
-1458670317.513183	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
-1458670317.513183	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
-#close	2016-03-22-18-11-57
+#open	2016-06-15-19-09-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017751.936022	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017754.938975	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
+1466017754.938975	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+1466017757.941783	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
+1466017757.941783	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+#close	2016-06-15-19-09-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2016-03-22-18-11-57
+#open	2016-06-15-19-09-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
-1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2016-03-22-18-11-57
+1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-06-15-19-09-18
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
index abf9490e65..69feed2307 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-16-32-08
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-#close	2016-05-11-16-32-08
+#open	2016-06-15-19-08-03
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
+#close	2016-06-15-19-08-03
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-16-32-08
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-#close	2016-05-11-16-32-08
+#open	2016-06-15-19-08-03
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
+1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
+1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
+#close	2016-06-15-19-08-03
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
index 2aabd3b2e5..f452f65a9e 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
@@ -3,27 +3,27 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-19-38-30
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-#close	2016-05-11-19-38-30
+#open	2016-06-15-19-06-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+#close	2016-06-15-19-06-02

From df5d9adfb4b031722003b5063f5aadf391b755aa Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 22 Jun 2016 21:14:06 +0200
Subject: [PATCH 09/11] Fixed insertion of nested subnets.

When inserting, existance of the given subnet is checked using exact
matching instead of longest prefix matching. Before, inserting a subnet
would have updated the subnet item, which is the longest prefix of the
inserted subnet, if present.
---
 scripts/base/frameworks/intel/main.bro                |  2 +-
 .../scripts.base.frameworks.intel.match-subnet/output | 11 ++++++-----
 .../scripts/base/frameworks/intel/match-subnet.bro    |  3 ++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 08dea9bb2f..027eaaf489 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -393,7 +393,7 @@ function insert(item: Item)
 		local net = to_subnet(item$indicator);
 		if ( have_full_data )
 			{
-			if ( net !in data_store$subnet_data )
+			if ( !check_subnet(net, data_store$subnet_data) )
 				data_store$subnet_data[net] = table();
 			else
 				is_new = F;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
index c20a053bca..aa401ab007 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-14-07
+#open	2016-06-22-19-12-08
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1466018047.083068	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-1466018047.083068	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
-1466018047.083068	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
-#close	2016-06-15-19-14-07
+1466622728.846581	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466622728.846581	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1466622728.846581	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
+#close	2016-06-22-19-12-08
 
 Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
 Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
@@ -18,6 +18,7 @@ Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, wher
 Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
 
 Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
+Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
 Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
 Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
 Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
diff --git a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
index 924fa947b6..1e25868de1 100644
--- a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
+++ b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
@@ -10,6 +10,7 @@
 192.168.2.0/24	Intel::SUBNET	source1	this subnetwork is just plain baaad	http://some-data-distributor.com/2
 192.168.142.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/3
 192.168.142.0/24	Intel::SUBNET	source1	this subnetwork is baaad	http://some-data-distributor.com/4
+192.168.142.0/26	Intel::SUBNET	source1	this subnetwork is inside	http://some-data-distributor.com/4
 192.168.128.0/18	Intel::SUBNET	source1	this subnetwork might be baaad	http://some-data-distributor.com/5
 # @TEST-END-FILE
 
@@ -47,4 +48,4 @@ event Intel::match(s: Intel::Seen, items: set[Intel::Item])
 	print fmt("Seen: %s", s);
 	for ( item in items )
 		print fmt("Item: %s", item);
-	}
\ No newline at end of file
+	}

From a7d3f530fa4b8a9f21364d49be3f3a6cfe6a8c5a Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 29 Jun 2016 20:58:39 +0200
Subject: [PATCH 10/11] Minor documentation cleanups.

---
 scripts/base/frameworks/intel/cluster.bro     | 27 +++---
 scripts/base/frameworks/intel/files.bro       |  4 +-
 scripts/base/frameworks/intel/input.bro       |  9 +-
 scripts/base/frameworks/intel/main.bro        | 87 ++++++++++---------
 scripts/policy/frameworks/intel/do_expire.bro |  4 +
 scripts/policy/frameworks/intel/do_notice.bro |  1 +
 scripts/policy/frameworks/intel/whitelist.bro |  4 +-
 7 files changed, 78 insertions(+), 58 deletions(-)

diff --git a/scripts/base/frameworks/intel/cluster.bro b/scripts/base/frameworks/intel/cluster.bro
index e8fae8327c..0727fb6746 100644
--- a/scripts/base/frameworks/intel/cluster.bro
+++ b/scripts/base/frameworks/intel/cluster.bro
@@ -1,8 +1,8 @@
 ##! Cluster transparency support for the intelligence framework.  This is mostly
 ##! oriented toward distributing intelligence information across clusters.
 
+@load ./main
 @load base/frameworks/cluster
-@load ./input
 
 module Intel;
 
@@ -17,14 +17,17 @@ redef record Item += {
 redef have_full_data = F;
 @endif
 
+# Internal event for cluster data distribution.
 global cluster_new_item: event(item: Item);
 
-# Primary intelligence distribution comes from manager.
+# Primary intelligence management is done by the manager:
+# The manager informs the workers about new items and item removal.
 redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/;
-# If a worker finds intelligence and adds it, it should share it back to the manager.
+# A worker queries the manager to insert, remove or indicate the match of an item.
 redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
+# Handling of new worker nodes.
 event remote_connection_handshake_done(p: event_peer)
 	{
 	# When a worker connects, send it the complete minimal data store.
@@ -35,25 +38,21 @@ event remote_connection_handshake_done(p: event_peer)
 		}
 	}
 
+# Handling of matches triggered by worker nodes.
 event Intel::match_no_items(s: Seen) &priority=5
 	{
 	if ( Intel::find(s) )
 		event Intel::match(s, Intel::get_items(s));
 	}
 
+# Handling of item removal triggered by worker nodes.
 event Intel::remove_item(item: Item, purge_indicator: bool)
 	{
 	remove(item, purge_indicator);
 	}
 @endif
 
-event Intel::cluster_new_item(item: Intel::Item) &priority=5
-	{
-	# Ignore locally generated events to avoid event storms.
-	if ( is_remote_event() )
-		Intel::insert(item);
-	}
-
+# Handling of item insertion.
 event Intel::new_item(item: Intel::Item) &priority=5
 	{
 	# The cluster manager always rebroadcasts intelligence.
@@ -65,3 +64,11 @@ event Intel::new_item(item: Intel::Item) &priority=5
 		event Intel::cluster_new_item(item);
 		}
 	}
+
+# Handling of item insertion by remote node.
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
diff --git a/scripts/base/frameworks/intel/files.bro b/scripts/base/frameworks/intel/files.bro
index 89d708cc1e..b786a6fefb 100644
--- a/scripts/base/frameworks/intel/files.bro
+++ b/scripts/base/frameworks/intel/files.bro
@@ -22,8 +22,8 @@ export {
 		## should go here to provide context to the data.
 		f:              fa_file  &optional;
 		## If the data was discovered within a file, the file uid should
-		## go here to provide context to the data. If the *f* field is
-		## provided, this will be automatically filled out.
+		## go here to provide context to the data. If the file record *f*
+		## is provided, this will be automatically filled out.
 		fuid:           string   &optional;
 	};
 
diff --git a/scripts/base/frameworks/intel/input.bro b/scripts/base/frameworks/intel/input.bro
index 7b494dcd75..9c4d033627 100644
--- a/scripts/base/frameworks/intel/input.bro
+++ b/scripts/base/frameworks/intel/input.bro
@@ -1,11 +1,14 @@
+##! Input handling for the intelligence framework. This script implements the
+##! import of intelligence data from files using the input framework.
+
 @load ./main
 
 module Intel;
 
 export {
-	## Intelligence files that will be read off disk.  The files are
-	## reread every time they are updated so updates must be atomic with
-	## "mv" instead of writing the file in place.
+	## Intelligence files that will be read off disk. The files are
+	## reread every time they are updated so updates must be atomic
+	## with "mv" instead of writing the file in place.
 	const read_files: set[string] = {} &redef;
 }
 
diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 027eaaf489..bc7c9187fe 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -1,7 +1,6 @@
-##! The intelligence framework provides a way to store and query IP addresses,
-##! and strings (with a str_type).  Metadata can
-##! also be associated with the intelligence, like for making more informed
-##! decisions about matching and handling of intelligence.
+##! The intelligence framework provides a way to store and query intelligence data
+##! (e.g. IP addresses, URLs and hashes). The intelligence items can be associated
+##! with metadata to allow informed decisions about matching and handling.
 
 @load base/frameworks/notice
 
@@ -31,15 +30,15 @@ export {
 		## Public key MD5 hash. (SSH server host keys are a good example.)
 		PUBKEY_HASH,
 	};
+
 	## Set of intelligence data types.
 	type TypeSet: set[Type];
 
 	## Data about an :bro:type:`Intel::Item`.
 	type MetaData: record {
-		## An arbitrary string value representing the data source.
-		## Typically, the convention for this field will be the source
-		## name and feed name separated by a hyphen.
-		## For example: "source1-c&c".
+		## An arbitrary string value representing the data source. This
+		## value is used as unique key to identify a metadata record in
+		## the scope of a single intelligence item.
 		source:      string;
 		## A freeform description for the data.
 		desc:        string      &optional;
@@ -55,7 +54,7 @@ export {
 		## The type of data that the indicator field represents.
 		indicator_type: Type;
 		
-		## Metadata for the item.  Typically represents more deeply
+		## Metadata for the item. Typically represents more deeply
 		## descriptive data for a piece of intelligence.
 		meta:           MetaData;
 	};
@@ -117,11 +116,14 @@ export {
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation function.
+	## Function to insert intelligence data. If the indicator is already
+	## present, the associated metadata will be added to the indicator. If
+	## the indicator already contains a metadata record from the same source,
+	## the existing metadata record will be updated.
 	global insert: function(item: Item);
 
 	## Function to remove intelligence data. If purge_indicator is set, the
-	## given meta data is ignored and the indicator is removed completely.
+	## given metadata is ignored and the indicator is removed completely.
 	global remove: function(item: Item, purge_indicator: bool &default = F);
 
 	## Function to declare discovery of a piece of data in order to check
@@ -129,16 +131,17 @@ export {
 	global seen: function(s: Seen);
 
 	## Event to represent a match in the intelligence data from data that
-	## was seen.  On clusters there is no assurance as to where this event
+	## was seen. On clusters there is no assurance as to when this event
 	## will be generated so do not assume that arbitrary global state beyond
 	## the given data will be available.
 	##
-	## This is the primary mechanism where a user will take actions based on
-	## data within the intelligence framework.
+	## This is the primary mechanism where a user may take actions based on
+	## data provided by the intelligence framework.
 	global match: event(s: Seen, items: set[Item]);
 
-	## This hook can be used to extend the intel log by adding data to the
-	## Info record. The default information is added with a priority of 5.
+	## This hook can be used to influence the logging of intelligence hits
+	## (e.g. by adding data to the Info record). The default information is
+	## added with a priority of 5.
 	##
 	## info: The Info record that will be logged.
 	##
@@ -161,7 +164,7 @@ export {
 	##
 	## indicator_type: The indicator type of the expired item.
 	##
-	## metas: The set of meta data describing the expired item.
+	## metas: The set of metadata describing the expired item.
 	##
 	## If all hook handlers are executed, the expiration timeout will be reset.
 	## Otherwise, if one of the handlers terminates using break, the item will
@@ -183,7 +186,7 @@ global purge_item: event(item: Item);
 # if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
-# Table of meta data, indexed by source string.
+# Table of metadata, indexed by source string.
 type MetaDataTable: table[string] of MetaData;
 
 # Expiration handlers.
@@ -215,21 +218,6 @@ event bro_init() &priority=5
 	Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]);
 	}
 
-function find(s: Seen): bool
-	{
-	local ds = have_full_data ? data_store : min_data_store;
-
-	if ( s?$host )
-		{
-		return ((s$host in ds$host_data) ||
-		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
-		}
-	else
-		{
-		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
-		}
-	}
-
 # Function that abstracts expiration of different types.
 function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval
 	{
@@ -275,7 +263,24 @@ function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any
 	return expire_item(indicator, indicator_type, metas);
 	}
 
-# Function to abstract from different data stores for different indicator types.
+# Function to check for intelligence hits.
+function find(s: Seen): bool
+	{
+	local ds = have_full_data ? data_store : min_data_store;
+
+	if ( s?$host )
+		{
+		return ((s$host in ds$host_data) ||
+		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
+		}
+	else
+		{
+		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
+		}
+	}
+
+# Function to retrieve intelligence items while abstracting from different
+# data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
 	local return_data: set[Item];
@@ -365,7 +370,7 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 
 function insert(item: Item)
 	{
-	# Create and fill out the meta data item.
+	# Create and fill out the metadata item.
 	local meta = item$meta;
 	local meta_tbl: table [string] of MetaData;
 	local is_new: bool = T;
@@ -420,7 +425,7 @@ function insert(item: Item)
 
 	if ( have_full_data )
 		{
-		# Insert new meta data or update if already present
+		# Insert new metadata or update if already present
 		meta_tbl[meta$source] = meta;
 		}
 
@@ -430,8 +435,8 @@ function insert(item: Item)
 		event Intel::new_item(item);
 	}
 
-# Function to remove meta data of an item. The function returns T
-# if there is no meta data left for the given indicator.
+# Function to remove metadata of an item. The function returns T
+# if there is no metadata left for the given indicator.
 function remove_meta_data(item: Item): bool
 	{
 	if ( ! have_full_data )
@@ -466,7 +471,7 @@ function remove(item: Item, purge_indicator: bool)
 		return;
 		}
 
-	# Remove meta data from manager's data store
+	# Remove metadata from manager's data store
 	local no_meta_data = remove_meta_data(item);
 	# Remove whole indicator if necessary
 	if ( no_meta_data || purge_indicator )
@@ -485,14 +490,14 @@ function remove(item: Item, purge_indicator: bool)
 				delete data_store$string_data[item$indicator, item$indicator_type];
 				break;
 			}
-		# Trigger deletion in min data stores
+		# Trigger deletion in minimal data stores
 		event Intel::purge_item(item);
 		}
 	}
 
+# Handling of indicator removal in minimal data stores.
 event purge_item(item: Item)
 	{
-	# Remove data from min data store
 	switch ( item$indicator_type )
 		{
 		case ADDR:
diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro
index b5f47c2ade..aabe3630e4 100644
--- a/scripts/policy/frameworks/intel/do_expire.bro
+++ b/scripts/policy/frameworks/intel/do_expire.bro
@@ -1,10 +1,14 @@
+##! This script enables expiration for intelligence items.
 
 @load base/frameworks/intel
 
 module Intel;
 
+redef item_expiration = 10min;
+
 hook item_expired(indicator: string, indicator_type: Type,
 	metas: set[MetaData]) &priority=-10
 	{
+	# Trigger removal of the expired item.
 	break;
 	}
diff --git a/scripts/policy/frameworks/intel/do_notice.bro b/scripts/policy/frameworks/intel/do_notice.bro
index ed859b468b..fc75a8efee 100644
--- a/scripts/policy/frameworks/intel/do_notice.bro
+++ b/scripts/policy/frameworks/intel/do_notice.bro
@@ -1,3 +1,4 @@
+##! This script enables notice generation for intelligence matches.
 
 @load base/frameworks/intel
 @load base/frameworks/notice
diff --git a/scripts/policy/frameworks/intel/whitelist.bro b/scripts/policy/frameworks/intel/whitelist.bro
index 9061ed2a91..527d828881 100644
--- a/scripts/policy/frameworks/intel/whitelist.bro
+++ b/scripts/policy/frameworks/intel/whitelist.bro
@@ -1,12 +1,12 @@
+##! This script enables whitelisting for intelligence items.
 
 @load base/frameworks/intel
-@load base/frameworks/notice
 
 module Intel;
 
 export {
 	redef record Intel::MetaData += {
-		## Add a field to indicate if this is a whitelisted item.
+		## A boolean value to indicate whether the item is whitelisted.
 		whitelist: bool &default=F;
 	};
 }

From 4bb4c54f96e2d93cd9bb383505a045893bdb0b5b Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 5 Aug 2016 09:29:23 -0400
Subject: [PATCH 11/11] Tiny scoping updates and test baseline updates for
 Intel framework.

---
 scripts/base/frameworks/intel/main.bro        |  5 ++-
 scripts/policy/frameworks/intel/do_expire.bro |  2 +-
 .../Baseline/language.expire_subnet/output    |  8 ++--
 .../output                                    | 14 +++---
 .../output                                    | 22 +++++-----
 .../intel-all.log                             | 22 +++++-----
 .../intel.log                                 | 22 +++++-----
 .../intel.log                                 | 44 +++++++++----------
 8 files changed, 70 insertions(+), 69 deletions(-)

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index bc7c9187fe..401b48e2d5 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -154,8 +154,9 @@ export {
 	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
 
 	## The expiration timeout for intelligence items. Once an item expires, the
-	## :bro:id:`item_expired` hook is called. Reinsertion of an item resets the
-	## timeout. A negative value disables expiration of intelligence items.
+	## :bro:id:`Intel::item_expired` hook is called. Reinsertion of an item 
+	## resets the timeout. A negative value disables expiration of intelligence 
+	## items.
 	const item_expiration = -1 min &redef;
 
 	## This hook can be used to handle expiration of intelligence items.
diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro
index aabe3630e4..fedb47b57d 100644
--- a/scripts/policy/frameworks/intel/do_expire.bro
+++ b/scripts/policy/frameworks/intel/do_expire.bro
@@ -4,7 +4,7 @@
 
 module Intel;
 
-redef item_expiration = 10min;
+redef Intel::item_expiration = 10min;
 
 hook item_expired(indicator: string, indicator_type: Type,
 	metas: set[MetaData]) &priority=-10
diff --git a/testing/btest/Baseline/language.expire_subnet/output b/testing/btest/Baseline/language.expire_subnet/output
index 61a6ac6a01..dee030eb0c 100644
--- a/testing/btest/Baseline/language.expire_subnet/output
+++ b/testing/btest/Baseline/language.expire_subnet/output
@@ -15,13 +15,13 @@ Accessed table nums: two; three
 Accessed table nets: two; zero, three
 Time: 7.0 secs 518.0 msecs 828.0 usecs
 
+Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.0 usecs
 Expired Num: 4 --> four at 8.0 secs 835.0 msecs 30.0 usecs
 Expired Num: 1 --> one at 8.0 secs 835.0 msecs 30.0 usecs
 Expired Num: 0 --> zero at 8.0 secs 835.0 msecs 30.0 usecs
-Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.0 usecs
-Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.0 usecs
-Expired Num: 2 --> two at 15.0 secs 150.0 msecs 681.0 usecs
-Expired Num: 3 --> three at 15.0 secs 150.0 msecs 681.0 usecs
 Expired Subnet: 192.168.0.0/16 --> zero at 15.0 secs 150.0 msecs 681.0 usecs
 Expired Subnet: 192.168.3.0/24 --> three at 15.0 secs 150.0 msecs 681.0 usecs
 Expired Subnet: 192.168.2.0/24 --> two at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Num: 2 --> two at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Num: 3 --> three at 15.0 secs 150.0 msecs 681.0 usecs
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
index aa401ab007..d8c2755fe4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-22-19-12-08
+#open	2016-08-05-13-13-14
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1466622728.846581	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-1466622728.846581	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
-1466622728.846581	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
-#close	2016-06-22-19-12-08
+1470402794.307931	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1470402794.307931	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1470402794.307931	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET,Intel::ADDR	source1	-	-	-
+#close	2016-08-05-13-13-14
 
 Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
 Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
@@ -18,7 +18,7 @@ Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, wher
 Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
 
 Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
-Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
-Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
 Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
 Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
+Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
+Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
index 8c8e9d9c0f..5249bb3110 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-09-12
+#open	2016-08-05-13-14-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1466017751.936022	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-1466017754.938975	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
-1466017754.938975	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
-1466017757.941783	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
-1466017757.941783	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
-#close	2016-06-15-19-09-18
+1470402852.531769	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1470402855.546089	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1,source2	-	-	-
+1470402855.546089	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+1470402858.547977	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1,source2	-	-	-
+1470402858.547977	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+#close	2016-08-05-13-14-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2016-06-15-19-09-18
+#open	2016-08-05-13-14-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
-1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2016-06-15-19-09-18
+1470402858.547977	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1470402858.547977	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-08-05-13-14-18
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
index 69feed2307..6bb3e47e60 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-08-03
+#open	2016-08-05-13-22-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
-#close	2016-06-15-19-08-03
+1416942644.593119	CHhAvVGS1DHFjwGM9	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
+#close	2016-08-05-13-22-37
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-08-03
+#open	2016-08-05-13-22-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
-1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
-1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
-#close	2016-06-15-19-08-03
+1170717505.735416	CHhAvVGS1DHFjwGM9	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
+1170717505.934612	CHhAvVGS1DHFjwGM9	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
+1170717508.883051	ClEkJM2Vm5giqnMf4h	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
+1170717509.082241	ClEkJM2Vm5giqnMf4h	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
+1170717511.909717	C4J4Th3PJpwUYZZ6gc	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
+1170717512.108799	C4J4Th3PJpwUYZZ6gc	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
+#close	2016-08-05-13-22-38
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log
index 708b02dd24..c14b4b10c1 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log
@@ -3,14 +3,14 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-07-13-16-17-20
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_RCPT_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashoefer@cern.ch	Intel::EMAIL	SMTP::IN_FROM	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashoefer@gmail.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	addr-spec@example.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	name-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	angle-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-#close	2016-07-13-16-17-20
+#open	2016-08-05-13-22-00
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_RCPT_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashoefer@cern.ch	Intel::EMAIL	SMTP::IN_FROM	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashoefer@gmail.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	addr-spec@example.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	name-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	angle-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+#close	2016-08-05-13-22-00
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
index f452f65a9e..66ba6af8db 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
@@ -3,27 +3,27 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-06-02
+#open	2016-08-05-13-24-29
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-#close	2016-06-15-19-06-02
+1300475168.853899	CmES5u32sYpV7JYN	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.854837	C37jN32gN3y3AZzyf6	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.857956	C0LAHyvtKSQHyJxIl	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.858713	C9rXSW3KSpTYvPrlI1	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.891644	C9mvWx3ezztgzcexV7	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.892414	C7fIlMZDuRiqjpYbb	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.893988	CpmdRlaUoJLN3uIRa	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.894787	CqlVyW1YwZ15RhTBc4	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916018	CwjjYJ2WqgTbAqiHl6	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916183	C3eiCBGOLw3VtHfOj	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.918358	Ck51lg1bScffFj34Ri	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952296	CykQaM33ztNt0csB9a	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952307	CtxTCR2Yer0FR1tIBg	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.954820	CLNN1k2QMum1aexUK7	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.975934	CwjjYJ2WqgTbAqiHl6	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.976436	C3eiCBGOLw3VtHfOj	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.979264	Ck51lg1bScffFj34Ri	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014593	CykQaM33ztNt0csB9a	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014619	CtxTCR2Yer0FR1tIBg	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014927	CLNN1k2QMum1aexUK7	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+#close	2016-08-05-13-24-29