From c5c650b486e5acc2594bdf45e73f7160629df0d0 Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Sat, 19 Mar 2016 17:02:52 +0100
Subject: [PATCH 01/25] Added testcase for intel updates.

By addind debug output to Intel::insert() the testcase reveals that
updating an intel item will cause its metadata to be inserted again,
without the old being deleted.
---
 .../output                                    | 25 ++++++++
 .../base/frameworks/intel/updated-match.bro   | 62 +++++++++++++++++++
 2 files changed, 87 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
 create mode 100644 testing/btest/scripts/base/frameworks/intel/updated-match.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
new file mode 100644
index 0000000000..3e7fa97f40
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -0,0 +1,25 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-03-19-16-01-52
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
+1458403312.669166	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
+1458403315.672095	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
+1458403315.672095	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
+1458403318.675592	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
+1458403318.675592	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
+#close	2016-03-19-16-01-58
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2016-03-19-16-01-58
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
+1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-03-19-16-01-58
diff --git a/testing/btest/scripts/base/frameworks/intel/updated-match.bro b/testing/btest/scripts/base/frameworks/intel/updated-match.bro
new file mode 100644
index 0000000000..75063d4b8f
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/updated-match.bro
@@ -0,0 +1,62 @@
+# @TEST-SERIALIZE: comm
+
+# @TEST-EXEC: cp intel1.dat intel.dat
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp intel2.dat intel.dat
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp intel3.dat intel.dat
+# @TEST-EXEC: btest-bg-wait 6
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/notice.log >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel1.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1234	F
+# @TEST-END-FILE
+
+# @TEST-START-FILE intel2.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source2	this host is just plain baaad	http://some-data-distributor.com/1234	F
+4.3.2.1	Intel::ADDR	source2	this host might also be baaad	http://some-data-distributor.com/4321	F
+# @TEST-END-FILE
+
+# @TEST-START-FILE intel3.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url	meta.do_notice
+1.2.3.4	Intel::ADDR	source2	this host is just plain baaad	http://some-data-distributor.com/1234	T
+4.3.2.1	Intel::ADDR	source2	this host might also be baaad	http://some-data-distributor.com/4321	T
+# @TEST-END-FILE
+
+@load base/frameworks/communication # let network-time run
+@load frameworks/intel/do_notice
+
+redef exit_only_after_terminate = T;
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+global runs = 0;
+event do_it()
+	{
+	Intel::seen([$host=1.2.3.4,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=4.3.2.1,
+	             $where=SOMEWHERE]);
+
+	++runs;
+	if ( runs < 3 )
+		schedule 3sec { do_it() };
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 5 )
+		terminate();
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

From 0146e85c41613190b542d28e549d0c3974e08fa2 Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Sat, 19 Mar 2016 17:12:06 +0100
Subject: [PATCH 02/25] Refactoring of meta data handling for intel.

To simplify meta data handling inside the intel framework and avoid
duplicate insertion of meta data on update, meta data is stored in a
table indexed by meta data source.
---
 scripts/base/frameworks/intel/main.bro | 90 ++++++++++----------------
 1 file changed, 33 insertions(+), 57 deletions(-)

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index eba27ca56a..32b48a089b 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -151,16 +151,18 @@ global match_no_items: event(s: Seen);
 
 # Internal events for cluster data distribution.
 global new_item: event(item: Item);
-global updated_item: event(item: Item);
 
 # Optionally store metadata.  This is used internally depending on
 # if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
+# Table of meta data, indexed by source string.
+type MetaDataTable: table[string] of MetaData;
+
 # The in memory data structure for holding intelligence.
 type DataStore: record {
-	host_data:    table[addr] of set[MetaData];
-	string_data:  table[string, Type] of set[MetaData];
+	host_data:    table[addr] of MetaDataTable;
+	string_data:  table[string, Type] of MetaDataTable;
 };
 global data_store: DataStore &redef;
 
@@ -186,26 +188,23 @@ function find(s: Seen): bool
 		return ((s$host in min_data_store$host_data) || 
 		        (have_full_data && s$host in data_store$host_data));
 		}
-	else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
-	           (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) )
-		{
-		return T;
-		}
 	else
 		{
-		return F;
+		return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
+		        (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data));
 		}
 	}
 
+# Function to abstract from different data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
 	local return_data: set[Item];
+	local mt: MetaDataTable;
 
 	if ( ! have_full_data )
 		{
-		# A reporter warning should be generated here because this function
-		# should never be called from a host that doesn't have the full data.
-		# TODO: do a reporter warning.
+		Reporter::warning(fmt("Intel::get_items was called from a host (%s) that doesn't have the full data.",
+			peer_description));
 		return return_data;
 		}
 
@@ -214,9 +213,10 @@ function get_items(s: Seen): set[Item]
 		# See if the host is known about and it has meta values
 		if ( s$host in data_store$host_data )
 			{
-			for ( m in data_store$host_data[s$host] )
+			mt = data_store$host_data[s$host];
+			for ( m in mt )
 				{
-				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)];
+				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
 				}
 			}
 		}
@@ -226,9 +226,10 @@ function get_items(s: Seen): set[Item]
 		# See if the string is known about and it has meta values
 		if ( [lower_indicator, s$indicator_type] in data_store$string_data )
 			{
-			for ( m in data_store$string_data[lower_indicator, s$indicator_type] )
+			mt = data_store$string_data[lower_indicator, s$indicator_type];
+			for ( m in mt )
 				{
-				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)];
+				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=mt[m])];
 				}
 			}
 		}
@@ -263,20 +264,6 @@ function Intel::seen(s: Seen)
 		}
 	}
 
-
-function has_meta(check: MetaData, metas: set[MetaData]): bool
-	{
-	local check_hash = md5_hash(check);
-	for ( m in metas )
-		{
-		if ( check_hash == md5_hash(m) )
-			return T;
-		}
-
-	# The records must not be equivalent if we made it this far.
-	return F;
-	}
-
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
 	local info = Info($ts=network_time(), $seen=s);
@@ -315,7 +302,8 @@ function insert(item: Item)
 	{
 	# Create and fill out the meta data item.
 	local meta = item$meta;
-	local metas: set[MetaData];
+	local meta_tbl: table [string] of MetaData;
+	local is_new: bool = T;
 
 	# All intelligence is case insensitive at the moment.
 	local lower_indicator = to_lower(item$indicator);
@@ -326,9 +314,11 @@ function insert(item: Item)
 		if ( have_full_data )
 			{
 			if ( host !in data_store$host_data )
-				data_store$host_data[host] = set();
+				data_store$host_data[host] = table();
+			else
+				is_new = F;
 
-			metas = data_store$host_data[host];
+			meta_tbl = data_store$host_data[host];
 			}
 
 		add min_data_store$host_data[host];
@@ -338,39 +328,25 @@ function insert(item: Item)
 		if ( have_full_data )
 			{
 			if ( [lower_indicator, item$indicator_type] !in data_store$string_data )
-				data_store$string_data[lower_indicator, item$indicator_type] = set();
+				data_store$string_data[lower_indicator, item$indicator_type] = table();
+			else
+				is_new = F;
 
-			metas = data_store$string_data[lower_indicator, item$indicator_type];
+			meta_tbl = data_store$string_data[lower_indicator, item$indicator_type];
 			}
 
 		add min_data_store$string_data[lower_indicator, item$indicator_type];
 		}
 
-	local updated = F;
 	if ( have_full_data )
 		{
-		for ( m in metas )
-			{
-			if ( meta$source == m$source )
-				{
-				if ( has_meta(meta, metas) )
-					{
-					# It's the same item being inserted again.
-					return;
-					}
-				else
-					{
-					# Same source, different metadata means updated item.
-					updated = T;
-					}
-				}
-			}
-		add metas[item$meta];
+		# Insert new meta data or update if already present
+		meta_tbl[meta$source] = meta;
 		}
-	
-	if ( updated )
-		event Intel::updated_item(item);
-	else
+
+	if ( is_new )
+		# Trigger insert for cluster in case the item is new
+		# or insert was called on a worker
 		event Intel::new_item(item);
 	}
 	

From cafae5351bbfea8f767bf82fbb97b5317abd74df Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Tue, 22 Mar 2016 19:16:51 +0100
Subject: [PATCH 03/25] Added support for subnets to intel-framework.

The intel-framework now supports the new indicator type Intel::SUBNET.
As subnets are matched against seen addresses, the field matched was
introduced to indicate which indicator types caused the hit. A testcase
for subents was added and the old ones have been updated accordingly.
---
 scripts/base/frameworks/intel/main.bro        | 46 +++++++++++++++--
 .../manager-1.intel.log                       | 10 ++--
 .../broproc.intel.log                         | 12 ++---
 .../output                                    | 23 +++++++++
 .../manager-1.intel.log                       | 16 +++---
 .../output                                    | 26 +++++-----
 .../intel-all.log                             | 24 ++++-----
 .../base/frameworks/intel/match-subnet.bro    | 50 +++++++++++++++++++
 8 files changed, 158 insertions(+), 49 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
 create mode 100644 testing/btest/scripts/base/frameworks/intel/match-subnet.bro

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 32b48a089b..f3bceec25e 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -14,6 +14,8 @@ export {
 	type Type: enum {
 		## An IP address.
 		ADDR,
+		## A subnet in CIDR notation.
+		SUBNET,
 		## A complete URL without the prefix ``"http://"``.
 		URL,
 		## Software name.
@@ -35,7 +37,9 @@ export {
 		## Public key MD5 hash. (SSH server host keys are a good example.)
 		PUBKEY_HASH,
 	};
-	
+	## Set of intelligence data types.
+	type TypeSet: set[Type];
+
 	## Data about an :bro:type:`Intel::Item`.
 	type MetaData: record {
 		## An arbitrary string value representing the data source.
@@ -123,6 +127,8 @@ export {
 
 		## Where the data was seen.
 		seen:     Seen           &log;
+		## Which indicator types matched.
+		matched:  TypeSet        &log;
 		## Sources which supplied data that resulted in this match.
 		sources:  set[string]    &log &default=string_set();
 	};
@@ -162,6 +168,7 @@ type MetaDataTable: table[string] of MetaData;
 # The in memory data structure for holding intelligence.
 type DataStore: record {
 	host_data:    table[addr] of MetaDataTable;
+	subnet_data:  table[subnet] of MetaDataTable;
 	string_data:  table[string, Type] of MetaDataTable;
 };
 global data_store: DataStore &redef;
@@ -171,6 +178,7 @@ global data_store: DataStore &redef;
 # a minimal amount of data for the full match to happen on the manager.
 type MinDataStore: record {
 	host_data:    set[addr];
+	subnet_data:  set[subnet];
 	string_data:  set[string, Type];
 };
 global min_data_store: MinDataStore &redef;
@@ -186,12 +194,11 @@ function find(s: Seen): bool
 	if ( s?$host )
 		{
 		return ((s$host in min_data_store$host_data) || 
-		        (have_full_data && s$host in data_store$host_data));
+		        (|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0));
 		}
 	else
 		{
-		return (([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
-		        (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data));
+		return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data);
 		}
 	}
 
@@ -219,6 +226,17 @@ function get_items(s: Seen): set[Item]
 				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=mt[m])];
 				}
 			}
+		# See if the host is part of a known subnet, which has meta values
+		local nets: table[subnet] of MetaDataTable;
+		nets = filter_subnet_table(addr_to_subnet(s$host), data_store$subnet_data);
+		for ( n in nets )
+			{
+				mt = nets[n];
+				for ( m in mt )
+					{
+					add return_data[Item($indicator=cat(n), $indicator_type=SUBNET, $meta=mt[m])];
+					}
+			}
 		}
 	else
 		{
@@ -266,7 +284,7 @@ function Intel::seen(s: Seen)
 
 event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local info = Info($ts=network_time(), $seen=s);
+	local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
 
 	if ( s?$f )
 		{
@@ -293,7 +311,10 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		}
 
 	for ( item in items )
+		{
 		add info$sources[item$meta$source];
+		add info$matched[item$indicator_type];
+		}
 
 	Log::write(Intel::LOG, info);
 	}
@@ -323,6 +344,21 @@ function insert(item: Item)
 
 		add min_data_store$host_data[host];
 		}
+	else if ( item$indicator_type == SUBNET )
+		{
+		local net = to_subnet(item$indicator);
+		if ( have_full_data )
+			{
+			if ( net !in data_store$subnet_data )
+				data_store$subnet_data[net] = table();
+			else
+				is_new = F;
+
+			meta_tbl = data_store$subnet_data[net];
+			}
+
+		add min_data_store$subnet_data[net];
+		}
 	else
 		{
 		if ( have_full_data )
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
index ba19f4e8d7..015d2f21bd 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-13-39
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1411488819.555114	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	worker-1
-#close	2014-09-23-16-13-49
+#open	2016-03-22-18-11-20
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670280.078658	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1
+#close	2016-03-22-18-11-29
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
index 33c97c0c1e..4436253d96 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-14-49
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1411488889.571819	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	source1
-1411488889.571819	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
-#close	2014-09-23-16-14-49
+#open	2016-03-22-18-11-32
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670292.167298	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1
+1458670292.167298	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+#close	2016-03-22-18-11-32
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
new file mode 100644
index 0000000000..3fbd90949f
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-03-22-18-11-34
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670294.227182	-	-	-	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1458670294.227182	-	-	-	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1
+1458670294.227182	-	-	-	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1
+#close	2016-03-22-18-11-34
+
+Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
+Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
+
+Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
+Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
+
+Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, f=<uninitialized>]
+Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
+Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
+Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
index d8e2d43674..074e6a0122 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2014-09-23-16-15-00
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1411488900.900403	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	source1
-1411488900.900403	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	source1
-1411488901.923543	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	source1
-1411488901.923543	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	source1
-#close	2014-09-23-16-15-09
+#open	2016-03-22-18-11-40
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670300.363597	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1
+1458670300.363597	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1
+1458670301.370555	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1
+1458670301.370555	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1
+#close	2016-03-22-18-11-49
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
index 3e7fa97f40..be3604f541 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-19-16-01-52
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1458403312.669166	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source1
-1458403315.672095	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
-1458403315.672095	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
-1458403318.675592	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	source2,source1
-1458403318.675592	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	source2
-#close	2016-03-19-16-01-58
+#open	2016-03-22-18-11-51
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1458670311.505318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1458670314.509318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
+1458670314.509318	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
+1458670317.513183	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
+1458670317.513183	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
+#close	2016-03-22-18-11-57
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2016-03-19-16-01-58
+#open	2016-03-22-18-11-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
-1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1458403318.675592	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2016-03-19-16-01-58
+1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-03-22-18-11-57
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
index ba1afe4239..ac88efb3d6 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
@@ -3,20 +3,20 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2015-03-14-01-47-46
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	source1
-#close	2015-03-14-01-47-46
+#open	2016-03-22-18-09-35
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	F0txuw2pvrkZOn04a8	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+#close	2016-03-22-18-09-35
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2015-03-14-01-47-46
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	source1
-#close	2015-03-14-01-47-46
+#open	2016-03-22-18-09-35
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	-	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
+#close	2016-03-22-18-09-35
diff --git a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
new file mode 100644
index 0000000000..924fa947b6
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
@@ -0,0 +1,50 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+192.168.1.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/1
+192.168.2.0/24	Intel::SUBNET	source1	this subnetwork is just plain baaad	http://some-data-distributor.com/2
+192.168.142.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/3
+192.168.142.0/24	Intel::SUBNET	source1	this subnetwork is baaad	http://some-data-distributor.com/4
+192.168.128.0/18	Intel::SUBNET	source1	this subnetwork might be baaad	http://some-data-distributor.com/5
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+
+event do_it()
+	{
+	Intel::seen([$host=192.168.1.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=192.168.2.1,
+	             $where=SOMEWHERE]);
+	Intel::seen([$host=192.168.142.1,
+	             $where=SOMEWHERE]);
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}
+
+global log_lines = 0;
+event Intel::log_intel(rec: Intel::Info)
+	{
+	++log_lines;
+	if ( log_lines == 2 )
+		terminate();
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print "";
+	print fmt("Seen: %s", s);
+	for ( item in items )
+		print fmt("Item: %s", item);
+	}
\ No newline at end of file

From 2ebac7078251cee2ef3f4607dcbe28c12e9165ad Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 30 Mar 2016 20:03:07 +0200
Subject: [PATCH 04/25] Added remove function to intel-framework.

---
 scripts/base/frameworks/intel/cluster.bro     | 20 ++--
 scripts/base/frameworks/intel/main.bro        | 95 ++++++++++++++++++-
 .../manager-1..stdout                         |  6 ++
 .../manager-1.intel.log                       | 10 ++
 .../worker-1..stdout                          |  5 +
 .../frameworks/intel/remove-item-cluster.bro  | 88 +++++++++++++++++
 6 files changed, 213 insertions(+), 11 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout
 create mode 100644 testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro

diff --git a/scripts/base/frameworks/intel/cluster.bro b/scripts/base/frameworks/intel/cluster.bro
index 7791c334d5..e8fae8327c 100644
--- a/scripts/base/frameworks/intel/cluster.bro
+++ b/scripts/base/frameworks/intel/cluster.bro
@@ -20,16 +20,11 @@ redef have_full_data = F;
 global cluster_new_item: event(item: Item);
 
 # Primary intelligence distribution comes from manager.
-redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
+redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/;
 # If a worker finds intelligence and adds it, it should share it back to the manager.
-redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
+redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
-event Intel::match_no_items(s: Seen) &priority=5
-	{
-	event Intel::match(s, Intel::get_items(s));
-	}
-
 event remote_connection_handshake_done(p: event_peer)
 	{
 	# When a worker connects, send it the complete minimal data store.
@@ -39,6 +34,17 @@ event remote_connection_handshake_done(p: event_peer)
 		send_id(p, "Intel::min_data_store");
 		}
 	}
+
+event Intel::match_no_items(s: Seen) &priority=5
+	{
+	if ( Intel::find(s) )
+		event Intel::match(s, Intel::get_items(s));
+	}
+
+event Intel::remove_item(item: Item, purge_indicator: bool)
+	{
+	remove(item, purge_indicator);
+	}
 @endif
 
 event Intel::cluster_new_item(item: Intel::Item) &priority=5
diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index f3bceec25e..8e387f71f6 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -136,6 +136,10 @@ export {
 	## Intelligence data manipulation function.
 	global insert: function(item: Item);
 
+	## Function to remove intelligence data. If purge_indicator is set, the
+	## given meta data is ignored and the indicator is removed completely.
+	global remove: function(item: Item, purge_indicator: bool &default = F);
+
 	## Function to declare discovery of a piece of data in order to check
 	## it against known intelligence for matches.
 	global seen: function(s: Seen);
@@ -157,6 +161,8 @@ global match_no_items: event(s: Seen);
 
 # Internal events for cluster data distribution.
 global new_item: event(item: Item);
+global remove_item: event(item: Item, purge_indicator: bool);
+global purge_item: event(item: Item);
 
 # Optionally store metadata.  This is used internally depending on
 # if this is a cluster deployment or not.
@@ -191,14 +197,16 @@ event bro_init() &priority=5
 
 function find(s: Seen): bool
 	{
+	local ds = have_full_data ? data_store : min_data_store;
+
 	if ( s?$host )
 		{
-		return ((s$host in min_data_store$host_data) || 
-		        (|matching_subnets(addr_to_subnet(s$host), min_data_store$subnet_data)| > 0));
+		return ((s$host in ds$host_data) ||
+		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
 		}
 	else
 		{
-		return ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data);
+		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
 		}
 	}
 
@@ -385,4 +393,83 @@ function insert(item: Item)
 		# or insert was called on a worker
 		event Intel::new_item(item);
 	}
-	
+
+# Function to remove meta data of an item. The function returns T
+# if there is no meta data left for the given indicator.
+function remove_meta_data(item: Item): bool
+	{
+	if ( ! have_full_data )
+		{
+		Reporter::warning(fmt("Intel::remove_meta_data was called from a host (%s) that doesn't have the full data.",
+			peer_description));
+		return F;
+		}
+
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			local host = to_addr(item$indicator);
+			delete data_store$host_data[host][item$meta$source];
+			return (|data_store$host_data[host]| == 0);
+		case SUBNET:
+			local net = to_subnet(item$indicator);
+			delete data_store$subnet_data[net][item$meta$source];
+			return (|data_store$subnet_data[net]| == 0);
+		default:
+			delete data_store$string_data[item$indicator, item$indicator_type][item$meta$source];
+			return (|data_store$string_data[item$indicator, item$indicator_type]| == 0);
+		}
+	}
+
+function remove(item: Item, purge_indicator: bool)
+	{
+	# Delegate removal if we are on a worker
+	if ( !have_full_data )
+		{
+		event Intel::remove_item(item, purge_indicator);
+		return;
+		}
+
+	# Remove meta data from manager's data store
+	local no_meta_data = remove_meta_data(item);
+	# Remove whole indicator if necessary
+	if ( no_meta_data || purge_indicator )
+		{
+		switch ( item$indicator_type )
+			{
+			case ADDR:
+				local host = to_addr(item$indicator);
+				delete data_store$host_data[host];
+				break;
+			case SUBNET:
+				local net = to_subnet(item$indicator);
+				delete data_store$subnet_data[net];
+				break;
+			default:
+				delete data_store$string_data[item$indicator, item$indicator_type];
+				break;
+			}
+		# Trigger deletion in min data stores
+		event Intel::purge_item(item);
+		}
+	}
+
+event purge_item(item: Item)
+	{
+	# Remove data from min data store
+	switch ( item$indicator_type )
+		{
+		case ADDR:
+			local host = to_addr(item$indicator);
+			delete min_data_store$host_data[host];
+			break;
+		case SUBNET:
+			local net = to_subnet(item$indicator);
+			delete min_data_store$subnet_data[net];
+			break;
+		default:
+			delete min_data_store$string_data[item$indicator, item$indicator_type];
+			break;
+		}
+	}
+
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout
new file mode 100644
index 0000000000..17862ce14b
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1..stdout
@@ -0,0 +1,6 @@
+Purging 192.168.0.1.
+Purging 192.168.0.2.
+Removing 192.168.1.2 (source: source1).
+Removing 192.168.1.2 (source: source2).
+Purging 192.168.1.2.
+Logging intel hit!
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
new file mode 100644
index 0000000000..bb3541ba32
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-03-30-16-01-31
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1459353691.470304	-	-	-	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end
+#close	2016-03-30-16-01-41
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout
new file mode 100644
index 0000000000..042032cb9d
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/worker-1..stdout
@@ -0,0 +1,5 @@
+Removing 192.168.1.2 (source: source1).
+Removing 192.168.1.2 (source: source2).
+Purging 192.168.0.1.
+Purging 192.168.0.2.
+Purging 192.168.1.2.
diff --git a/testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro b/testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro
new file mode 100644
index 0000000000..d13536a015
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/remove-item-cluster.bro
@@ -0,0 +1,88 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff manager-1/.stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff worker-1/.stdout
+# @TEST-EXEC: btest-diff manager-1/intel.log
+
+# @TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1"],
+};
+# @TEST-END-FILE
+
+@load base/frameworks/control
+
+module Intel;
+
+redef Log::default_rotation_interval=0sec;
+
+event test_manager()
+	{
+	Intel::remove([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::seen([$host=192.168.0.1, $where=Intel::IN_ANYWHERE]);
+	Intel::remove([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]], T);
+	Intel::seen([$host=192.168.0.2, $where=Intel::IN_ANYWHERE]);
+	}
+
+event test_worker()
+	{
+	Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+	Intel::remove([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+	Intel::seen([$host=192.168.1.2, $where=Intel::IN_ANYWHERE]);
+	# Trigger shutdown by matching data that should be present
+	Intel::seen([$host=10.10.10.10, $where=Intel::IN_ANYWHERE]);
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# Insert the data once all workers are connected.
+	if ( Cluster::local_node_type() == Cluster::MANAGER && Cluster::worker_count == 1 )
+		{
+		Intel::insert([$indicator="192.168.0.1", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.0.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+		Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source1"]]);
+		Intel::insert([$indicator="192.168.1.2", $indicator_type=Intel::ADDR, $meta=[$source="source2"]]);
+		Intel::insert([$indicator="10.10.10.10", $indicator_type=Intel::ADDR, $meta=[$source="end"]]);
+
+		event test_manager();
+		}
+	}
+
+global worker_data = 0;
+event Intel::cluster_new_item(item: Intel::Item)
+	{
+	# Run test on worker-1 when all items have been inserted
+	if ( Cluster::node == "worker-1" )
+		{
+		++worker_data;
+		if ( worker_data == 4 )
+			event test_worker();
+		}
+	}
+
+event Intel::remove_item(item: Item, purge_indicator: bool)
+	{
+	print fmt("Removing %s (source: %s).", item$indicator, item$meta$source);
+	}
+
+event purge_item(item: Item)
+	{
+	print fmt("Purging %s.", item$indicator);
+	}
+
+event Intel::log_intel(rec: Intel::Info)
+	{
+	print "Logging intel hit!";
+	event Control::shutdown_request();
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	# Cascading termination
+	terminate_communication();
+	}

From cb330287022b506d870cd970e6f0aa1a810bb08e Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 11 May 2016 23:27:51 +0200
Subject: [PATCH 05/25] Added hook to allow extending the intel log.

The extension mechanism is basically the one that Seth introduced with
his intel extensions. The main difference lies in using a hook instead
of an event. An example policy implements whitelisting.
---
 scripts/base/frameworks/intel/main.bro        | 21 +++++++++-
 scripts/policy/frameworks/intel/whitelist.bro | 30 ++++++++++++++
 scripts/test-all-policy.bro                   |  1 +
 .../intel.log                                 | 29 ++++++++++++++
 .../policy/frameworks/intel/whitelisting.bro  | 39 +++++++++++++++++++
 5 files changed, 118 insertions(+), 2 deletions(-)
 create mode 100644 scripts/policy/frameworks/intel/whitelist.bro
 create mode 100644 testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
 create mode 100644 testing/btest/scripts/policy/frameworks/intel/whitelisting.bro

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index b52b30aff0..55494507a7 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -165,6 +165,19 @@ export {
 	## data within the intelligence framework.
 	global match: event(s: Seen, items: set[Item]);
 
+	## This hook can be used to extend the intel log by adding data to the
+	## Info record. The default information is added with a priority of 5.
+	##
+	## info: The Info record that will be logged.
+	##
+	## s: Information about the data seen.
+	##
+	## items: The intel items that match the seen data.
+	##
+	## In case the hook execution is terminated using break, the match will
+	## not be logged.
+	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -306,6 +319,12 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
 	local info = Info($ts=network_time(), $seen=s, $matched=TypeSet());
 
+	if ( hook extend_match(info, s, items) )
+		Log::write(Intel::LOG, info);
+	}
+
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+	{
 	if ( s?$f )
 		{
 		s$fuid = s$f$id;
@@ -340,8 +359,6 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		add info$sources[item$meta$source];
 		add info$matched[item$indicator_type];
 		}
-
-	Log::write(Intel::LOG, info);
 	}
 
 function insert(item: Item)
diff --git a/scripts/policy/frameworks/intel/whitelist.bro b/scripts/policy/frameworks/intel/whitelist.bro
new file mode 100644
index 0000000000..9061ed2a91
--- /dev/null
+++ b/scripts/policy/frameworks/intel/whitelist.bro
@@ -0,0 +1,30 @@
+
+@load base/frameworks/intel
+@load base/frameworks/notice
+
+module Intel;
+
+export {
+	redef record Intel::MetaData += {
+		## Add a field to indicate if this is a whitelisted item.
+		whitelist: bool &default=F;
+	};
+}
+
+hook Intel::extend_match(info: Info, s: Seen, items: set[Item]) &priority=9
+	{
+	local whitelisted = F;
+	for ( item in items )
+		{
+		if ( item$meta$whitelist )
+			{
+			whitelisted = T;
+			break;
+			}
+		}
+	
+	if ( whitelisted )
+		# Prevent logging
+		break;
+	}
+
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index f85fdb58b0..02602d1dc6 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -15,6 +15,7 @@
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
 @load frameworks/intel/do_notice.bro
+@load frameworks/intel/whitelist.bro
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
 @load frameworks/intel/seen/dns.bro
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
new file mode 100644
index 0000000000..2aabd3b2e5
--- /dev/null
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
@@ -0,0 +1,29 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-05-11-19-38-30
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
+1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
+#close	2016-05-11-19-38-30
diff --git a/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro b/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro
new file mode 100644
index 0000000000..53acd49aa9
--- /dev/null
+++ b/testing/btest/scripts/policy/frameworks/intel/whitelisting.bro
@@ -0,0 +1,39 @@
+# @TEST-EXEC: bro -Cr $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-diff intel.log
+
+#@TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+upload.wikimedia.org	Intel::DOMAIN	source1	somehow bad	http://some-data-distributor.com/1
+meta.wikimedia.org	Intel::DOMAIN	source1	also bad	http://some-data-distributor.com/1
+#@TEST-END-FILE
+
+#@TEST-START-FILE whitelist.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.whitelist	meta.url
+meta.wikimedia.org	Intel::DOMAIN	source2	also bad	T	http://some-data-distributor.com/1
+#@TEST-END-FILE
+
+@load base/frameworks/intel
+@load frameworks/intel/whitelist
+@load frameworks/intel/seen
+
+redef Intel::read_files += {
+	"intel.dat",
+	"whitelist.dat",
+};
+
+global total_files_read = 0;
+
+event bro_init()
+	{
+	suspend_processing();
+	}
+
+event Input::end_of_data(name: string, source: string)
+	{
+	# Wait until both intel files are read.
+	if ( /^intel-/ in name && (++total_files_read == 2) )
+		{
+		continue_processing();
+		}
+	}
+

From a9ad41cdccdfecffc4dda453da3223d203f48658 Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Thu, 19 May 2016 21:03:36 +0200
Subject: [PATCH 06/25] Improved intel notices.

Intel notices are identified by a direction independent 3-tuple
(indicator, originator IP, responder IP). This allows notice
suppression. Additionally service and intel source are added to the
notice mail.
---
 scripts/policy/frameworks/intel/do_notice.bro | 37 ++++++++++++++++---
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/scripts/policy/frameworks/intel/do_notice.bro b/scripts/policy/frameworks/intel/do_notice.bro
index 89910ede32..ed859b468b 100644
--- a/scripts/policy/frameworks/intel/do_notice.bro
+++ b/scripts/policy/frameworks/intel/do_notice.bro
@@ -6,14 +6,14 @@ module Intel;
 
 export {
 	redef enum Notice::Type += {
-		## Intel::Notice is a notice that happens when an intelligence 
+		## Intel::Notice is a notice that happens when an intelligence
 		## indicator is denoted to be notice-worthy.
 		Intel::Notice
 	};
 
 	redef record Intel::MetaData += {
 		## A boolean value to allow the data itself to represent
-		## if the indicator that this metadata is attached to 
+		## if the indicator that this metadata is attached to
 		## is notice worthy.
 		do_notice: bool &default=F;
 
@@ -29,15 +29,42 @@ event Intel::match(s: Seen, items: set[Item])
 	for ( item in items )
 		{
 		if ( item$meta$do_notice &&
-		     (! item$meta?$if_in || s$where == item$meta$if_in) )
+			(! item$meta?$if_in || s$where == item$meta$if_in) )
 			{
 			local n = Notice::Info($note=Intel::Notice,
-			                       $msg=fmt("Intel hit on %s at %s", s$indicator, s$where),
-			                       $sub=s$indicator);
+				$msg = fmt("Intel hit on %s at %s", s$indicator, s$where),
+				$sub = s$indicator);
+			local service_str = "";
 
 			if ( s?$conn )
+				{
 				n$conn = s$conn;
 
+				# Add identifier composed of indicator, originator's and responder's IP,
+				# without considering the direction of the flow.
+				local intel_id = s$indicator;
+				if( s$conn?$id )
+					{
+					if( s$conn$id$orig_h < s$conn$id$resp_h)
+						intel_id = cat(intel_id, s$conn$id$orig_h, s$conn$id$resp_h);
+					else
+						intel_id = cat(intel_id, s$conn$id$resp_h, s$conn$id$orig_h);
+					}
+				n$identifier = intel_id;
+
+				if ( s$conn?$service )
+					{
+					for ( service in s$conn$service )
+						service_str = cat(service_str, service, " ");
+					}
+				}
+
+			# Add additional information to the generated mail
+			local mail_ext = vector(
+				fmt("Service: %s\n", service_str),
+				fmt("Intel source: %s\n", item$meta$source));
+			n$email_body_sections = mail_ext;
+
 			NOTICE(n);
 			}
 		}

From 5d340e669c9d41bc9f7dd3aea0a52624de115cde Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 15 Jun 2016 19:19:13 +0200
Subject: [PATCH 07/25] Added expiration for intelligence items.

Expiration of intelligence items can be configured using
Intel::item_expiration. Expiration can be handled using the
Intel::item_expired hook.
---
 scripts/base/frameworks/intel/main.bro        | 74 ++++++++++++++++++-
 scripts/policy/frameworks/intel/do_expire.bro | 10 +++
 scripts/test-all-policy.bro                   |  1 +
 .../output                                    | 22 ++++++
 .../base/frameworks/intel/expire-item.bro     | 46 ++++++++++++
 5 files changed, 150 insertions(+), 3 deletions(-)
 create mode 100644 scripts/policy/frameworks/intel/do_expire.bro
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
 create mode 100644 testing/btest/scripts/base/frameworks/intel/expire-item.bro

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 55494507a7..6d4e19b222 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -178,6 +178,24 @@ export {
 	## not be logged.
 	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
 
+	## The expiration timeout for intelligence items. Once an item expires, the
+	## :bro:id:`item_expired` hook is called. Reinsertion of an item resets the
+	## timeout. A negative value disables expiration of intelligence items.
+	const item_expiration = -1 min &redef;
+
+	## This hook can be used to handle expiration of intelligence items.
+	##
+	## indicator: The indicator of the expired item.
+	##
+	## indicator_type: The indicator type of the expired item.
+	##
+	## metas: The set of meta data describing the expired item.
+	##
+	## If all hook handlers are executed, the expiration timeout will be reset.
+	## Otherwise, if one of the handlers terminates using break, the item will
+	## be removed.
+	global item_expired: hook(indicator: string, indicator_type: Type, metas: set[MetaData]);
+
 	global log_intel: event(rec: Info);
 }
 
@@ -196,11 +214,16 @@ const have_full_data = T &redef;
 # Table of meta data, indexed by source string.
 type MetaDataTable: table[string] of MetaData;
 
+# Expiration handlers.
+global expire_host_data: function(data: table[addr] of MetaDataTable, idx: addr): interval;
+global expire_subnet_data: function(data: table[subnet] of MetaDataTable, idx: subnet): interval;
+global expire_string_data: function(data: table[string, Type] of MetaDataTable, idx: any): interval;
+
 # The in memory data structure for holding intelligence.
 type DataStore: record {
-	host_data:    table[addr] of MetaDataTable;
-	subnet_data:  table[subnet] of MetaDataTable;
-	string_data:  table[string, Type] of MetaDataTable;
+	host_data:    table[addr] of MetaDataTable &write_expire=item_expiration &expire_func=expire_host_data;
+	subnet_data:  table[subnet] of MetaDataTable &write_expire=item_expiration &expire_func=expire_subnet_data;
+	string_data:  table[string, Type] of MetaDataTable &write_expire=item_expiration &expire_func=expire_string_data;
 };
 global data_store: DataStore &redef;
 
@@ -235,6 +258,51 @@ function find(s: Seen): bool
 		}
 	}
 
+# Function that abstracts expiration of different types.
+function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval
+	{
+	if ( hook item_expired(indicator, indicator_type, metas) )
+		return item_expiration;
+	else
+		remove([$indicator=indicator, $indicator_type=indicator_type, $meta=[$source=""]], T);
+	return 0 sec;
+	}
+
+# Expiration handler definitions.
+function expire_host_data(data: table[addr] of MetaDataTable, idx: addr): interval
+	{
+	local meta_tbl: MetaDataTable = data[idx];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(cat(idx), ADDR, metas);
+	}
+
+function expire_subnet_data(data: table[subnet] of MetaDataTable, idx: subnet): interval
+	{
+	local meta_tbl: MetaDataTable = data[idx];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(cat(idx), ADDR, metas);
+	}
+
+function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any): interval
+	{
+	local indicator: string;
+	local indicator_type: Type;
+	[indicator, indicator_type] = idx;
+
+	local meta_tbl: MetaDataTable = data[indicator, indicator_type];
+	local metas: set[MetaData];
+	for ( src in meta_tbl )
+		add metas[meta_tbl[src]];
+
+	return expire_item(indicator, indicator_type, metas);
+	}
+
 # Function to abstract from different data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro
new file mode 100644
index 0000000000..b5f47c2ade
--- /dev/null
+++ b/scripts/policy/frameworks/intel/do_expire.bro
@@ -0,0 +1,10 @@
+
+@load base/frameworks/intel
+
+module Intel;
+
+hook item_expired(indicator: string, indicator_type: Type,
+	metas: set[MetaData]) &priority=-10
+	{
+	break;
+	}
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index 3ea017bb95..1513e33289 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -15,6 +15,7 @@
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
 @load frameworks/intel/do_notice.bro
+@load frameworks/intel/do_expire.bro
 @load frameworks/intel/whitelist.bro
 @load frameworks/intel/seen/__load__.bro
 @load frameworks/intel/seen/conn-established.bro
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
new file mode 100644
index 0000000000..e605be1d5e
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
@@ -0,0 +1,22 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	intel
+#open	2016-06-09-19-48-59
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
+#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
+1465501739.703996	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1465501740.704649	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+1465501741.705204	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
+#close	2016-06-09-19-49-05
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Trigger: 1.2.3.4
+Seen: 1.2.3.4
+Expired: 1.2.3.4
+Trigger: 1.2.3.4
+Trigger: 1.2.3.4
+Trigger: 1.2.3.4
diff --git a/testing/btest/scripts/base/frameworks/intel/expire-item.bro b/testing/btest/scripts/base/frameworks/intel/expire-item.bro
new file mode 100644
index 0000000000..d56ef504f5
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/intel/expire-item.bro
@@ -0,0 +1,46 @@
+# @TEST-EXEC: btest-bg-run broproc bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 7
+# @TEST-EXEC: cat broproc/intel.log > output
+# @TEST-EXEC: cat broproc/.stdout >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE intel.dat
+#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+1.2.3.4	Intel::ADDR	source1	this host is bad	http://some-data-distributor.com/1
+# @TEST-END-FILE
+
+@load frameworks/communication/listen
+@load frameworks/intel/do_expire
+
+redef Intel::read_files += { "../intel.dat" };
+redef enum Intel::Where += { SOMEWHERE };
+redef Intel::item_expiration = 3sec;
+redef table_expire_interval = 1sec;
+
+global runs = 0;
+event do_it()
+	{
+	print "Trigger: 1.2.3.4";
+	Intel::seen([$host=1.2.3.4,
+	             $where=SOMEWHERE]);
+
+	++runs;
+	if ( runs < 6 )
+		schedule 1sec { do_it() };
+	}
+
+event Intel::match(s: Intel::Seen, items: set[Intel::Item])
+	{
+	print fmt("Seen: %s", s$indicator);
+	}
+
+hook Intel::item_expired(indicator: string, indicator_type: Intel::Type,
+	metas: set[Intel::MetaData])
+	{
+	print fmt("Expired: %s", indicator);
+	}
+
+event bro_init() &priority=-10
+	{
+	schedule 1sec { do_it() };
+	}

From 1412de17989c29cc3ca9916f755462dd460ce10a Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 15 Jun 2016 21:56:53 +0200
Subject: [PATCH 08/25] Refactored FAF integration of intel framework.

File Analysis Framework related code has been moved into a separate
script. Using redefinitions of the corresponding records causes the
file-related columns to appear last.
---
 scripts/base/frameworks/intel/__load__.bro    |  3 +
 scripts/base/frameworks/intel/files.bro       | 84 +++++++++++++++++++
 scripts/base/frameworks/intel/main.bro        | 66 ---------------
 .../canonified_loaded_scripts.log             |  5 +-
 .../manager-1.intel.log                       | 10 +--
 .../output                                    | 14 ++--
 .../broproc.intel.log                         | 12 +--
 .../output                                    | 14 ++--
 .../manager-1.intel.log                       | 16 ++--
 .../manager-1.intel.log                       | 10 +--
 .../output                                    | 26 +++---
 .../intel-all.log                             | 30 +++----
 .../intel.log                                 | 48 +++++------
 13 files changed, 180 insertions(+), 158 deletions(-)
 create mode 100644 scripts/base/frameworks/intel/files.bro

diff --git a/scripts/base/frameworks/intel/__load__.bro b/scripts/base/frameworks/intel/__load__.bro
index d8c77b86e3..d1cb61a7e2 100644
--- a/scripts/base/frameworks/intel/__load__.bro
+++ b/scripts/base/frameworks/intel/__load__.bro
@@ -1,5 +1,8 @@
 @load ./main
 
+# File analysis framework integration.
+@load ./files
+
 # The cluster framework must be loaded first.
 @load base/frameworks/cluster
 
diff --git a/scripts/base/frameworks/intel/files.bro b/scripts/base/frameworks/intel/files.bro
new file mode 100644
index 0000000000..89d708cc1e
--- /dev/null
+++ b/scripts/base/frameworks/intel/files.bro
@@ -0,0 +1,84 @@
+##! File analysis framework integration for the intelligence framework. This
+##! script manages file information in intelligence framework datastructures.
+
+@load ./main
+
+module Intel;
+
+export {
+	## Enum type to represent various types of intelligence data.
+	redef enum Type += {
+		## File hash which is non-hash type specific.  It's up to the
+		## user to query for any relevant hash types.
+		FILE_HASH,
+		## File name.  Typically with protocols with definite
+		## indications of a file name.
+		FILE_NAME,
+	};
+
+	## Information about a piece of "seen" data.
+	redef record Seen += {
+		## If the data was discovered within a file, the file record
+		## should go here to provide context to the data.
+		f:              fa_file  &optional;
+		## If the data was discovered within a file, the file uid should
+		## go here to provide context to the data. If the *f* field is
+		## provided, this will be automatically filled out.
+		fuid:           string   &optional;
+	};
+
+	## Record used for the logging framework representing a positive
+	## hit within the intelligence framework.
+	redef record Info += {
+		## If a file was associated with this intelligence hit,
+		## this is the uid for the file.
+		fuid:           string   &log &optional;
+		## A mime type if the intelligence hit is related to a file.
+		## If the $f field is provided this will be automatically filled
+		## out.
+		file_mime_type: string   &log &optional;
+		## Frequently files can be "described" to give a bit more context.
+		## If the $f field is provided this field will be automatically
+		## filled out.
+		file_desc:      string   &log &optional;
+	};
+}
+
+# Add file information to matches if available.
+hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
+	{
+	if ( s?$f )
+		{
+		s$fuid = s$f$id;
+
+		if ( s$f?$conns && |s$f$conns| == 1 )
+			{
+			for ( cid in s$f$conns )
+				s$conn = s$f$conns[cid];
+			}
+
+		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
+			info$file_mime_type = s$f$info$mime_type;
+
+		if ( ! info?$file_desc )
+			info$file_desc = Files::describe(s$f);
+		}
+
+	if ( s?$fuid )
+		info$fuid = s$fuid;
+
+	if ( s?$conn )
+		{
+		s$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	if ( s?$uid )
+		info$uid = s$uid;
+
+	for ( item in items )
+		{
+		add info$sources[item$meta$source];
+		add info$matched[item$indicator_type];
+		}
+	}
diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 6d4e19b222..08dea9bb2f 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -26,12 +26,6 @@ export {
 		DOMAIN,
 		## A user name.
 		USER_NAME,
-		## File hash which is non-hash type specific.  It's up to the
-		## user to query for any relevant hash types.
-		FILE_HASH,
-		## File name.  Typically with protocols with definite
-		## indications of a file name.
-		FILE_NAME,
 		## Certificate SHA-1 hash.
 		CERT_HASH,
 		## Public key MD5 hash. (SSH server host keys are a good example.)
@@ -100,15 +94,6 @@ export {
 		## If the *conn* field is provided, this will be automatically
 		## filled out.
 		uid:             string        &optional;
-
-		## If the data was discovered within a file, the file record
-		## should go here to provide context to the data.
-		f:               fa_file       &optional;
-
-		## If the data was discovered within a file, the file uid should
-		## go here to provide context to the data. If the *f* field is
-		## provided, this will be automatically filled out.
-		fuid:            string        &optional;
 	};
 
 	## Record used for the logging framework representing a positive
@@ -124,19 +109,6 @@ export {
 		## this is the conn_id for the connection.
 		id:       conn_id        &log &optional;
 
-		## If a file was associated with this intelligence hit,
-		## this is the uid for the file.
-		fuid:           string   &log &optional;
-
-		## A mime type if the intelligence hit is related to a file.
-		## If the $f field is provided this will be automatically filled
-		## out.
-		file_mime_type: string   &log &optional;
-		## Frequently files can be "described" to give a bit more context.
-		## If the $f field is provided this field will be automatically
-		## filled out.
-		file_desc:      string   &log &optional;
-
 		## Where the data was seen.
 		seen:     Seen           &log;
 		## Which indicator types matched.
@@ -391,44 +363,6 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 		Log::write(Intel::LOG, info);
 	}
 
-hook extend_match(info: Info, s: Seen, items: set[Item]) &priority=5
-	{
-	if ( s?$f )
-		{
-		s$fuid = s$f$id;
-
-		if ( s$f?$conns && |s$f$conns| == 1 )
-			{
-			for ( cid in s$f$conns )
-				s$conn = s$f$conns[cid];
-			}
-
-		if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
-			info$file_mime_type = s$f$info$mime_type;
-
-		if ( ! info?$file_desc )
-			info$file_desc = Files::describe(s$f);
-		}
-
-	if ( s?$fuid )
-		info$fuid = s$fuid;
-
-	if ( s?$conn )
-		{
-		s$uid = s$conn$uid;
-		info$id  = s$conn$id;
-		}
-
-	if ( s?$uid )
-		info$uid = s$uid;
-
-	for ( item in items )
-		{
-		add info$sources[item$meta$source];
-		add info$matched[item$indicator_type];
-		}
-	}
-
 function insert(item: Item)
 	{
 	# Create and fill out the meta data item.
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index da62a25117..6469b71726 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-06-07-19-22-42
+#open	2016-06-15-19-16-09
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -177,6 +177,7 @@ scripts/base/init-default.bro
     scripts/base/frameworks/communication/main.bro
   scripts/base/frameworks/intel/__load__.bro
     scripts/base/frameworks/intel/main.bro
+    scripts/base/frameworks/intel/files.bro
     scripts/base/frameworks/intel/input.bro
   scripts/base/frameworks/sumstats/__load__.bro
     scripts/base/frameworks/sumstats/main.bro
@@ -310,4 +311,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-06-07-19-22-42
+#close	2016-06-15-19-16-09
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
index 015d2f21bd..48df37a6ec 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-20
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670280.078658	-	-	-	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1
-#close	2016-03-22-18-11-29
+#open	2016-06-15-19-11-27
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017887.060652	-	-	-	-	-	123.123.123.123	Intel::ADDR	Intel::IN_ANYWHERE	worker-2	Intel::ADDR	worker-1	-	-	-
+#close	2016-06-15-19-11-36
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
index e605be1d5e..dfa922f88f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-09-19-48-59
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1465501739.703996	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1465501740.704649	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1465501741.705204	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-#close	2016-06-09-19-49-05
+#open	2016-06-15-19-11-06
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017866.348490	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017867.349583	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017868.349656	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2016-06-15-19-11-12
 Trigger: 1.2.3.4
 Seen: 1.2.3.4
 Trigger: 1.2.3.4
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
index 4436253d96..7c29bb659e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-32
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670292.167298	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1
-1458670292.167298	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-#close	2016-03-22-18-11-32
+#open	2016-06-15-19-12-26
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017946.413077	-	-	-	-	-	e@mail.com	Intel::EMAIL	SOMEWHERE	bro	Intel::EMAIL	source1	-	-	-
+1466017946.413077	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+#close	2016-06-15-19-12-26
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
index 949d428cd1..c20a053bca 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-16-59-39
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1462985979.596867	-	-	-	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1462985979.596867	-	-	-	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1
-1462985979.596867	-	-	-	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1
-#close	2016-05-11-16-59-39
+#open	2016-06-15-19-14-07
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466018047.083068	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466018047.083068	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1466018047.083068	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
+#close	2016-06-15-19-14-07
 
 Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
 Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
index 074e6a0122..12b07e116e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-40
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670300.363597	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1
-1458670300.363597	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1
-1458670301.370555	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1
-1458670301.370555	-	-	-	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1
-#close	2016-03-22-18-11-49
+#open	2016-06-15-19-14-30
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466018070.494693	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-1	Intel::ADDR	source1	-	-	-
+1466018070.494693	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-1	Intel::EMAIL	source1	-	-	-
+1466018071.505800	-	-	-	-	-	1.2.3.4	Intel::ADDR	Intel::IN_A_TEST	worker-2	Intel::ADDR	source1	-	-	-
+1466018071.505800	-	-	-	-	-	e@mail.com	Intel::EMAIL	Intel::IN_A_TEST	worker-2	Intel::EMAIL	source1	-	-	-
+#close	2016-06-15-19-14-39
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
index bb3541ba32..b7b7118004 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-item-cluster/manager-1.intel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-30-16-01-31
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1459353691.470304	-	-	-	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end
-#close	2016-03-30-16-01-41
+#open	2016-06-15-19-10-09
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017809.810005	-	-	-	-	-	10.10.10.10	Intel::ADDR	Intel::IN_ANYWHERE	worker-1	Intel::ADDR	end	-	-	-
+#close	2016-06-15-19-10-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
index be3604f541..8c8e9d9c0f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-03-22-18-11-51
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1458670311.505318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1
-1458670314.509318	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
-1458670314.509318	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
-1458670317.513183	-	-	-	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1
-1458670317.513183	-	-	-	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2
-#close	2016-03-22-18-11-57
+#open	2016-06-15-19-09-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1466017751.936022	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466017754.938975	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
+1466017754.938975	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+1466017757.941783	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
+1466017757.941783	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+#close	2016-06-15-19-09-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2016-03-22-18-11-57
+#open	2016-06-15-19-09-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
-1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1458670317.513183	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2016-03-22-18-11-57
+1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-06-15-19-09-18
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
index abf9490e65..69feed2307 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-16-32-08
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-#close	2016-05-11-16-32-08
+#open	2016-06-15-19-08-03
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
+#close	2016-06-15-19-08-03
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-16-32-08
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	FeCwNK3rzqPnZ7eBQ5	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	FjkLnG4s34DVZlaBNc	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	FQXAWgI2FB5STbrff	-	-	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1
-#close	2016-05-11-16-32-08
+#open	2016-06-15-19-08-03
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
+1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
+1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
+1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
+1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
+1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
+#close	2016-06-15-19-08-03
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
index 2aabd3b2e5..f452f65a9e 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
@@ -3,27 +3,27 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-05-11-19-38-30
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[enum]	set[string]
-1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	-	-	-	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1
-1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	-	-	-	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1
-#close	2016-05-11-19-38-30
+#open	2016-06-15-19-06-02
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+#close	2016-06-15-19-06-02

From df5d9adfb4b031722003b5063f5aadf391b755aa Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 22 Jun 2016 21:14:06 +0200
Subject: [PATCH 09/25] Fixed insertion of nested subnets.

When inserting, existance of the given subnet is checked using exact
matching instead of longest prefix matching. Before, inserting a subnet
would have updated the subnet item, which is the longest prefix of the
inserted subnet, if present.
---
 scripts/base/frameworks/intel/main.bro                |  2 +-
 .../scripts.base.frameworks.intel.match-subnet/output | 11 ++++++-----
 .../scripts/base/frameworks/intel/match-subnet.bro    |  3 ++-
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 08dea9bb2f..027eaaf489 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -393,7 +393,7 @@ function insert(item: Item)
 		local net = to_subnet(item$indicator);
 		if ( have_full_data )
 			{
-			if ( net !in data_store$subnet_data )
+			if ( !check_subnet(net, data_store$subnet_data) )
 				data_store$subnet_data[net] = table();
 			else
 				is_new = F;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
index c20a053bca..aa401ab007 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-14-07
+#open	2016-06-22-19-12-08
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1466018047.083068	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-1466018047.083068	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
-1466018047.083068	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
-#close	2016-06-15-19-14-07
+1466622728.846581	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1466622728.846581	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1466622728.846581	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
+#close	2016-06-22-19-12-08
 
 Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
 Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
@@ -18,6 +18,7 @@ Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, wher
 Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
 
 Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
+Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
 Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
 Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
 Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
diff --git a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
index 924fa947b6..1e25868de1 100644
--- a/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
+++ b/testing/btest/scripts/base/frameworks/intel/match-subnet.bro
@@ -10,6 +10,7 @@
 192.168.2.0/24	Intel::SUBNET	source1	this subnetwork is just plain baaad	http://some-data-distributor.com/2
 192.168.142.1	Intel::ADDR	source1	this host is just plain baaad	http://some-data-distributor.com/3
 192.168.142.0/24	Intel::SUBNET	source1	this subnetwork is baaad	http://some-data-distributor.com/4
+192.168.142.0/26	Intel::SUBNET	source1	this subnetwork is inside	http://some-data-distributor.com/4
 192.168.128.0/18	Intel::SUBNET	source1	this subnetwork might be baaad	http://some-data-distributor.com/5
 # @TEST-END-FILE
 
@@ -47,4 +48,4 @@ event Intel::match(s: Intel::Seen, items: set[Intel::Item])
 	print fmt("Seen: %s", s);
 	for ( item in items )
 		print fmt("Item: %s", item);
-	}
\ No newline at end of file
+	}

From a7d3f530fa4b8a9f21364d49be3f3a6cfe6a8c5a Mon Sep 17 00:00:00 2001
From: Jan Grashoefer <jan.grashoefer@gmail.com>
Date: Wed, 29 Jun 2016 20:58:39 +0200
Subject: [PATCH 10/25] Minor documentation cleanups.

---
 scripts/base/frameworks/intel/cluster.bro     | 27 +++---
 scripts/base/frameworks/intel/files.bro       |  4 +-
 scripts/base/frameworks/intel/input.bro       |  9 +-
 scripts/base/frameworks/intel/main.bro        | 87 ++++++++++---------
 scripts/policy/frameworks/intel/do_expire.bro |  4 +
 scripts/policy/frameworks/intel/do_notice.bro |  1 +
 scripts/policy/frameworks/intel/whitelist.bro |  4 +-
 7 files changed, 78 insertions(+), 58 deletions(-)

diff --git a/scripts/base/frameworks/intel/cluster.bro b/scripts/base/frameworks/intel/cluster.bro
index e8fae8327c..0727fb6746 100644
--- a/scripts/base/frameworks/intel/cluster.bro
+++ b/scripts/base/frameworks/intel/cluster.bro
@@ -1,8 +1,8 @@
 ##! Cluster transparency support for the intelligence framework.  This is mostly
 ##! oriented toward distributing intelligence information across clusters.
 
+@load ./main
 @load base/frameworks/cluster
-@load ./input
 
 module Intel;
 
@@ -17,14 +17,17 @@ redef record Item += {
 redef have_full_data = F;
 @endif
 
+# Internal event for cluster data distribution.
 global cluster_new_item: event(item: Item);
 
-# Primary intelligence distribution comes from manager.
+# Primary intelligence management is done by the manager:
+# The manager informs the workers about new items and item removal.
 redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/;
-# If a worker finds intelligence and adds it, it should share it back to the manager.
+# A worker queries the manager to insert, remove or indicate the match of an item.
 redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
+# Handling of new worker nodes.
 event remote_connection_handshake_done(p: event_peer)
 	{
 	# When a worker connects, send it the complete minimal data store.
@@ -35,25 +38,21 @@ event remote_connection_handshake_done(p: event_peer)
 		}
 	}
 
+# Handling of matches triggered by worker nodes.
 event Intel::match_no_items(s: Seen) &priority=5
 	{
 	if ( Intel::find(s) )
 		event Intel::match(s, Intel::get_items(s));
 	}
 
+# Handling of item removal triggered by worker nodes.
 event Intel::remove_item(item: Item, purge_indicator: bool)
 	{
 	remove(item, purge_indicator);
 	}
 @endif
 
-event Intel::cluster_new_item(item: Intel::Item) &priority=5
-	{
-	# Ignore locally generated events to avoid event storms.
-	if ( is_remote_event() )
-		Intel::insert(item);
-	}
-
+# Handling of item insertion.
 event Intel::new_item(item: Intel::Item) &priority=5
 	{
 	# The cluster manager always rebroadcasts intelligence.
@@ -65,3 +64,11 @@ event Intel::new_item(item: Intel::Item) &priority=5
 		event Intel::cluster_new_item(item);
 		}
 	}
+
+# Handling of item insertion by remote node.
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
diff --git a/scripts/base/frameworks/intel/files.bro b/scripts/base/frameworks/intel/files.bro
index 89d708cc1e..b786a6fefb 100644
--- a/scripts/base/frameworks/intel/files.bro
+++ b/scripts/base/frameworks/intel/files.bro
@@ -22,8 +22,8 @@ export {
 		## should go here to provide context to the data.
 		f:              fa_file  &optional;
 		## If the data was discovered within a file, the file uid should
-		## go here to provide context to the data. If the *f* field is
-		## provided, this will be automatically filled out.
+		## go here to provide context to the data. If the file record *f*
+		## is provided, this will be automatically filled out.
 		fuid:           string   &optional;
 	};
 
diff --git a/scripts/base/frameworks/intel/input.bro b/scripts/base/frameworks/intel/input.bro
index 7b494dcd75..9c4d033627 100644
--- a/scripts/base/frameworks/intel/input.bro
+++ b/scripts/base/frameworks/intel/input.bro
@@ -1,11 +1,14 @@
+##! Input handling for the intelligence framework. This script implements the
+##! import of intelligence data from files using the input framework.
+
 @load ./main
 
 module Intel;
 
 export {
-	## Intelligence files that will be read off disk.  The files are
-	## reread every time they are updated so updates must be atomic with
-	## "mv" instead of writing the file in place.
+	## Intelligence files that will be read off disk. The files are
+	## reread every time they are updated so updates must be atomic
+	## with "mv" instead of writing the file in place.
 	const read_files: set[string] = {} &redef;
 }
 
diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index 027eaaf489..bc7c9187fe 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -1,7 +1,6 @@
-##! The intelligence framework provides a way to store and query IP addresses,
-##! and strings (with a str_type).  Metadata can
-##! also be associated with the intelligence, like for making more informed
-##! decisions about matching and handling of intelligence.
+##! The intelligence framework provides a way to store and query intelligence data
+##! (e.g. IP addresses, URLs and hashes). The intelligence items can be associated
+##! with metadata to allow informed decisions about matching and handling.
 
 @load base/frameworks/notice
 
@@ -31,15 +30,15 @@ export {
 		## Public key MD5 hash. (SSH server host keys are a good example.)
 		PUBKEY_HASH,
 	};
+
 	## Set of intelligence data types.
 	type TypeSet: set[Type];
 
 	## Data about an :bro:type:`Intel::Item`.
 	type MetaData: record {
-		## An arbitrary string value representing the data source.
-		## Typically, the convention for this field will be the source
-		## name and feed name separated by a hyphen.
-		## For example: "source1-c&c".
+		## An arbitrary string value representing the data source. This
+		## value is used as unique key to identify a metadata record in
+		## the scope of a single intelligence item.
 		source:      string;
 		## A freeform description for the data.
 		desc:        string      &optional;
@@ -55,7 +54,7 @@ export {
 		## The type of data that the indicator field represents.
 		indicator_type: Type;
 		
-		## Metadata for the item.  Typically represents more deeply
+		## Metadata for the item. Typically represents more deeply
 		## descriptive data for a piece of intelligence.
 		meta:           MetaData;
 	};
@@ -117,11 +116,14 @@ export {
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation function.
+	## Function to insert intelligence data. If the indicator is already
+	## present, the associated metadata will be added to the indicator. If
+	## the indicator already contains a metadata record from the same source,
+	## the existing metadata record will be updated.
 	global insert: function(item: Item);
 
 	## Function to remove intelligence data. If purge_indicator is set, the
-	## given meta data is ignored and the indicator is removed completely.
+	## given metadata is ignored and the indicator is removed completely.
 	global remove: function(item: Item, purge_indicator: bool &default = F);
 
 	## Function to declare discovery of a piece of data in order to check
@@ -129,16 +131,17 @@ export {
 	global seen: function(s: Seen);
 
 	## Event to represent a match in the intelligence data from data that
-	## was seen.  On clusters there is no assurance as to where this event
+	## was seen. On clusters there is no assurance as to when this event
 	## will be generated so do not assume that arbitrary global state beyond
 	## the given data will be available.
 	##
-	## This is the primary mechanism where a user will take actions based on
-	## data within the intelligence framework.
+	## This is the primary mechanism where a user may take actions based on
+	## data provided by the intelligence framework.
 	global match: event(s: Seen, items: set[Item]);
 
-	## This hook can be used to extend the intel log by adding data to the
-	## Info record. The default information is added with a priority of 5.
+	## This hook can be used to influence the logging of intelligence hits
+	## (e.g. by adding data to the Info record). The default information is
+	## added with a priority of 5.
 	##
 	## info: The Info record that will be logged.
 	##
@@ -161,7 +164,7 @@ export {
 	##
 	## indicator_type: The indicator type of the expired item.
 	##
-	## metas: The set of meta data describing the expired item.
+	## metas: The set of metadata describing the expired item.
 	##
 	## If all hook handlers are executed, the expiration timeout will be reset.
 	## Otherwise, if one of the handlers terminates using break, the item will
@@ -183,7 +186,7 @@ global purge_item: event(item: Item);
 # if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
-# Table of meta data, indexed by source string.
+# Table of metadata, indexed by source string.
 type MetaDataTable: table[string] of MetaData;
 
 # Expiration handlers.
@@ -215,21 +218,6 @@ event bro_init() &priority=5
 	Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]);
 	}
 
-function find(s: Seen): bool
-	{
-	local ds = have_full_data ? data_store : min_data_store;
-
-	if ( s?$host )
-		{
-		return ((s$host in ds$host_data) ||
-		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
-		}
-	else
-		{
-		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
-		}
-	}
-
 # Function that abstracts expiration of different types.
 function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval
 	{
@@ -275,7 +263,24 @@ function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any
 	return expire_item(indicator, indicator_type, metas);
 	}
 
-# Function to abstract from different data stores for different indicator types.
+# Function to check for intelligence hits.
+function find(s: Seen): bool
+	{
+	local ds = have_full_data ? data_store : min_data_store;
+
+	if ( s?$host )
+		{
+		return ((s$host in ds$host_data) ||
+		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
+		}
+	else
+		{
+		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
+		}
+	}
+
+# Function to retrieve intelligence items while abstracting from different
+# data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
 	local return_data: set[Item];
@@ -365,7 +370,7 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 
 function insert(item: Item)
 	{
-	# Create and fill out the meta data item.
+	# Create and fill out the metadata item.
 	local meta = item$meta;
 	local meta_tbl: table [string] of MetaData;
 	local is_new: bool = T;
@@ -420,7 +425,7 @@ function insert(item: Item)
 
 	if ( have_full_data )
 		{
-		# Insert new meta data or update if already present
+		# Insert new metadata or update if already present
 		meta_tbl[meta$source] = meta;
 		}
 
@@ -430,8 +435,8 @@ function insert(item: Item)
 		event Intel::new_item(item);
 	}
 
-# Function to remove meta data of an item. The function returns T
-# if there is no meta data left for the given indicator.
+# Function to remove metadata of an item. The function returns T
+# if there is no metadata left for the given indicator.
 function remove_meta_data(item: Item): bool
 	{
 	if ( ! have_full_data )
@@ -466,7 +471,7 @@ function remove(item: Item, purge_indicator: bool)
 		return;
 		}
 
-	# Remove meta data from manager's data store
+	# Remove metadata from manager's data store
 	local no_meta_data = remove_meta_data(item);
 	# Remove whole indicator if necessary
 	if ( no_meta_data || purge_indicator )
@@ -485,14 +490,14 @@ function remove(item: Item, purge_indicator: bool)
 				delete data_store$string_data[item$indicator, item$indicator_type];
 				break;
 			}
-		# Trigger deletion in min data stores
+		# Trigger deletion in minimal data stores
 		event Intel::purge_item(item);
 		}
 	}
 
+# Handling of indicator removal in minimal data stores.
 event purge_item(item: Item)
 	{
-	# Remove data from min data store
 	switch ( item$indicator_type )
 		{
 		case ADDR:
diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro
index b5f47c2ade..aabe3630e4 100644
--- a/scripts/policy/frameworks/intel/do_expire.bro
+++ b/scripts/policy/frameworks/intel/do_expire.bro
@@ -1,10 +1,14 @@
+##! This script enables expiration for intelligence items.
 
 @load base/frameworks/intel
 
 module Intel;
 
+redef item_expiration = 10min;
+
 hook item_expired(indicator: string, indicator_type: Type,
 	metas: set[MetaData]) &priority=-10
 	{
+	# Trigger removal of the expired item.
 	break;
 	}
diff --git a/scripts/policy/frameworks/intel/do_notice.bro b/scripts/policy/frameworks/intel/do_notice.bro
index ed859b468b..fc75a8efee 100644
--- a/scripts/policy/frameworks/intel/do_notice.bro
+++ b/scripts/policy/frameworks/intel/do_notice.bro
@@ -1,3 +1,4 @@
+##! This script enables notice generation for intelligence matches.
 
 @load base/frameworks/intel
 @load base/frameworks/notice
diff --git a/scripts/policy/frameworks/intel/whitelist.bro b/scripts/policy/frameworks/intel/whitelist.bro
index 9061ed2a91..527d828881 100644
--- a/scripts/policy/frameworks/intel/whitelist.bro
+++ b/scripts/policy/frameworks/intel/whitelist.bro
@@ -1,12 +1,12 @@
+##! This script enables whitelisting for intelligence items.
 
 @load base/frameworks/intel
-@load base/frameworks/notice
 
 module Intel;
 
 export {
 	redef record Intel::MetaData += {
-		## Add a field to indicate if this is a whitelisted item.
+		## A boolean value to indicate whether the item is whitelisted.
 		whitelist: bool &default=F;
 	};
 }

From 7603567782f5735c815ead75b1ecd6953c555f62 Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@icir.org>
Date: Tue, 26 Jul 2016 15:02:11 -0700
Subject: [PATCH 11/25] Correct endianness of IP addresses in SNMP.

Addresses BIT-1644
---
 src/analyzer/protocol/snmp/snmp-analyzer.pac  |   2 +-
 .../.stdout                                   |  30 ++++++++++++++++++
 .../scripts.base.protocols.snmp.v1/out4       |   2 +-
 testing/btest/Traces/snmp/snmpwalk-short.pcap | Bin 0 -> 43259 bytes
 .../scripts/base/protocols/snmp/snmp-addr.bro |  15 +++++++++
 5 files changed, 47 insertions(+), 2 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.snmp.snmp-addr/.stdout
 create mode 100644 testing/btest/Traces/snmp/snmpwalk-short.pcap
 create mode 100644 testing/btest/scripts/base/protocols/snmp/snmp-addr.bro

diff --git a/src/analyzer/protocol/snmp/snmp-analyzer.pac b/src/analyzer/protocol/snmp/snmp-analyzer.pac
index 44dce4dbf5..0394dbda61 100644
--- a/src/analyzer/protocol/snmp/snmp-analyzer.pac
+++ b/src/analyzer/protocol/snmp/snmp-analyzer.pac
@@ -39,7 +39,7 @@ AddrVal* network_address_to_val(const ASN1Encoding* na)
 
 	const u_char* data = reinterpret_cast<const u_char*>(bs.data());
 	uint32 network_order = extract_uint32(data);
-	return new AddrVal(network_order);
+	return new AddrVal(ntohl(network_order));
 	}
 
 Val* asn1_obj_to_val(const ASN1Encoding* obj)
diff --git a/testing/btest/Baseline/scripts.base.protocols.snmp.snmp-addr/.stdout b/testing/btest/Baseline/scripts.base.protocols.snmp.snmp-addr/.stdout
new file mode 100644
index 0000000000..f21633eb91
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.snmp.snmp-addr/.stdout
@@ -0,0 +1,30 @@
+138.68.0.1
+138.68.14.240
+169.254.169.254
+10.46.0.5
+127.0.0.1
+138.68.10.203
+255.255.0.0
+255.0.0.0
+255.255.240.0
+0.0.0.0
+10.46.0.0
+138.68.0.0
+138.68.0.1
+0.0.0.0
+0.0.0.0
+0.0.0.0
+255.255.0.0
+255.255.240.0
+138.68.0.1
+138.68.14.240
+169.254.169.254
+0.0.0.0
+10.46.0.0
+138.68.0.0
+0.0.0.0
+255.255.0.0
+255.255.240.0
+138.68.0.1
+0.0.0.0
+0.0.0.0
diff --git a/testing/btest/Baseline/scripts.base.protocols.snmp.v1/out4 b/testing/btest/Baseline/scripts.base.protocols.snmp.v1/out4
index 0854c7096c..2111edee6f 100644
--- a/testing/btest/Baseline/scripts.base.protocols.snmp.v1/out4
+++ b/testing/btest/Baseline/scripts.base.protocols.snmp.v1/out4
@@ -3,7 +3,7 @@ snmp_trap
   is_orig: T
   [community=public]
     enterprise:    1.3.6.1.4.1.31337.0
-    agent:         1.0.0.127
+    agent:         127.0.0.1
     generic_trap:  0
     specific_trap: 0
     time_stamp:    0
diff --git a/testing/btest/Traces/snmp/snmpwalk-short.pcap b/testing/btest/Traces/snmp/snmpwalk-short.pcap
new file mode 100644
index 0000000000000000000000000000000000000000..90f2a4766477565ced29752bd8b989a5950ce248
GIT binary patch
literal 43259
zcma)_2Y41$_Qt=KkVfwo2-ro!03i?(I!Grl^dd+J5J^D@3Kk$LMT~tF>?Rh96%;jb
zUF)u0x7gd?-g|dl<$vy(d-L5hbH2myI4ttGyx;Gh_n!C8ojV^dJ-)9l7D+_Rj{^rH
zktqDjq4#f}Id*Aw#D!n0D-j*|&A+4Y^LM{SGa{oRk&zed8X6hedgDXeMz;8N+sLeE
zFMTmm7jg17IY-B$(M0;nHH*s27T0x-B{FAC8WoF1;2$U3$xKh{ksggj(@?=ZB|GvE
z{ORiQ{%lVN74B-NQwr{E^`|S9NF8WgQ}?S=bBQHUUvm!8sCHJRXP`zj8c8gjP*%BS
zZSI;yYbvYP^z7HWPsO61tCv>w>FM;&&Ckovb9(me-E&!KW$CK2#koiHbJq8U-^<VI
z-E%trKXRu|oSZwebV=^GHRZXz`{p{m2Ri)+_U)59ecFiJK2Gm~+_eJ=78K;G;VNH|
z9hnXw>Iw|k=nHn^a5a$O>YKOKInH9KI}A}y)Wg-n$w{JkBX(VRc4Pqvs4FmHQyS`V
z#O@;_Ht3~wPG3u;{%9i7$?0H3Vv+PL7`a6B;f#1JOEDc)Au{bjrU$|>jSe!!A|6xc
z%IwH2;89nAX<Wli$n;mjR8Vx4bArv(6`71kYp11QibqDoBj<mkV6F!pp+L7G%>7|t
z#sp#F8s;xm*^%V{qOJhUoQ9hb<~9OzV%9jP*oNtaFh-=C)78KvB8j#WOLL1WN{5!N
zU0AWQyfm+B)w00~uE$Cd?skL&V_1_lh5N<^;Sw6I<*C_`G5}Lo0B&x>EeN-pz>Us3
z)fr^NIS6M&x;wcBE-jL1$IVkkw*iEO(%pe{`@_(U3(}=&x{Fq2M^*!wx&m~i4YwlQ
z^@MK9I~5L?uBqSE4@5d6a)NWbp-Yb>x=byrt}d%wmRnVsTfMwAcVyM#6}eSQ@z=u_
zu2Y=9!AB4{??ldf!*Grda;9s}KdQ4MGk{TD0nRZEJCU=IaOOXKu5-N2c{p+!k(N$#
z!`TcP{TPLE-Wn0*E=0K}49bKcN;8e}?P($kMwAOZ*9wsQr_Se4jEIGj5g8H1n3{dM
zh;lcg><fc3F^H0(Q7&1V9XS~)qOL$y+y;yf_fCQ`c+)B;->%9rSQR7E+3AS=9R7<;
zh}`r{=<Vt8?07V)>Jwj=9hnJQ>H<BQ?qe_5i%@qFsD63doF0~5T^H1wq@h|lEo?on
zxequ)#KHt|@1R&4R&!WJhGNAuR!jI?3hOq|6R`Fn77U}+2UB2`uvjq<tHW6$7G{Zi
zJH<MOt(K9YSaFS&rK`0K^aQMX5DSL!;8>Gcthk4D;Oy+kNni+d1x#4dfGOfLVbPTv
zoZglx>W@TI7?}=EJIfS_^mH^Kmg<`IBGbLd1Y>w`rYS5_!ejbigUE#0;d5Zo@*0lG
z$WW#<WQs+5EBh_mC^GFwCW!xoGr25Nn#XiYjmU&K;j>@S<t&qtp-kzTDWRB#o+C2d
zhfI6IFimBd(mkf7=ZZ`i7%p8o6)k3&j0|OJrkVOErpS3B)BVT<E184WX&TGa%wrm~
zNo*6C9PUoqCQfC$pOK+$B15-{R<RgPsSiL;XcG@0)~+yE(^;$x53BhW5eu`!y_I6E
zUdc6eBSWz=HCFR@G_I<3H|Pmi4TuH9c<^e?V6ieitnBke2V!=(FnZQ%qNp$HKqEsP
zn57*!Pcdx;F@dQOnP3bL&NP!{%JP^l-j*HN4BcE^fsVem0fWO`Pnec`QRB?9JNiWI
z=tgFUGbq{7<IzN~X|R@^ICgmM(yCRt6H8aGUbw6@ck-&L#qdkD+Fe+=BzI)l>Xi$t
z7cYl@&r|T7E)d}#MEG06z|Z2~<9dIgcWrj$764aQ0DgM|CX73uz@K~B@6I|Kz8v9=
z%zS5_1s_XHQt%ZED}lM9w6Z$4tP)0{VqtX|tnrtWE-kApQ@>n|W24smQzyZO!t~Xp
zt9mY4xEl5qrj{;Vv#P9mUGBuHC8gy!HV?sv7L4&%9GjcNjLmFzZ1f5yaZz^UEEpGc
z1;(bbVZV}bH<PiM^WX;#<a+XMX9SLok?H63wZ<l%I7*ET0#8g1(6CkDg6h)6)oWIv
zYY_Qt_$&hQLx_A+7-aA*?lh6LXk4#u&bv4}vJP<76+m9mfI;SNCCD?6y}^O?bP{<e
zA{&`Lj$<Jw65Y*QsHj@9ro0r(u14UrKDpoy@IvWoW&aMBh`fJ8-uf`Sb6H-!@>y`H
zIQKE++%pL8l7?yA+&40G?x!W^zQv2PK~4N=BPQpTsZ9zsGjK{E{44OOgknF8@O5F}
z=dtj5iPZ1%?8rq>XmtgOy|DqK&^?vFm*-7)7TLv~j>R@Iqn%M!vC|X%nPNA=kDZK-
zZ|n$|@5~^)51&pLghy}?ZVWRBC$WQ|*HlGUhz`Kubk~wWSaR=F&H+Y-I-r^CfaxPy
z_!r?*3Gk01{0(8?=d<v7@%7%7*^!M<eRTyKu&MzQ);*iR&p+ft2Uc3i={_1AU}OrN
z{+0tW635LbD?MFz@?ncr;N^~}s;pjBRqplkrumOuZrXMc`!U3Z{iTB^{sI<TFU~4<
zWJj(9WOW6ww>Dt5x=RUm^+%3#ij92|VjGzfXQG9jnHWNi@1<Ll^ReSks;VwqTDBNx
z{Rz1%%h$kvR90RM+w6J;aO_p$U_6e4ab1|fIGG&`y`-zWIy<rn20~qd!8pAE!_+;E
z494Q8J30_)Ne@iG!7ws|onmV+vJ!c8FtFt1Rm+yC4j34U)o6q-;X?`je*)RB4a2^W
zW!Fo;#I@qSI;N@%DPt{X3wE#P_SKDy+E-Uk2w-$F;W2^U9;*c%0q5_Cb4?f=XuiSp
zPUywlj_X7m3{4ku!^YXk;usl<6Vo^yuNQItfjC!(!CB1W==E384I&Q4pbH6L<J`jH
z7#WHa*Est?N2tz|h;vmKoFy!dURyQZDB@tCxzN*XoSRr2BSUc#8mFR8#CZyFc7(wx
zWpVV<X-B<?gAwJ@X8{FuERK<(IB6Qkxk<!%8gaIV!CA`U=*7_5n?;<*l#C1A)~?PC
zERK<(IO!Va7tj&R_6*`&83tz=i=&q{U+xrfFg#pnpf=8RERK<(IL$Q9wp}95vxsv=
z7@XxSj$WwL?G|w`4_s)VHqJFHj*+1_85*bi9uem`#Q94YoH7<iFF@MeCgNZoxX?gt
zoU2$IBSUdAHO}>*Bbe=Z#JM~S4y=8c+|f&j2W}T}Fb`a4pf=8S7RSg?oGgtq`3@21
z1;n{549*G`N3RMd-6=jY!#wa+Cx6!q+#@q1lYC@`y0~NaJ*Wr|GGAnBlj3o7xrRv|
zhJfGj_O|BSEmj7zz=h^%SLV8>xCM?8Nmd3nRHJI0qi)x8_KKDHC&FA3rZN?Q%Ak&i
zbJRW&2eZJ1=4s<x^CX93WGD_u#T3r1pd-wie<9ArVQ?y09KFEVcaMmJS>QtRv~jNb
z2Zv*1C{8@)Z4T$%E8_ecaV`pjQ^n%w1<uL)vm>+NE<#;_JBjfPm<29$8w;oZF*BW>
z_MJp7-bomlHcl(OnpTyGCSv7PsyuIikWilgAkT$icvf;e=Iqti_hm=s0*kr=JQEu*
z3|ug^#Z&mnH%>2`r#tc(nRZSaizk*ylvXcy6w#9VMWUCGs5T7IsVtE;+}sC5BFqC9
zd~Fkb$PyVDN)$&Te9YGfQbLVhMxqPC5UpZ~wC6r<5T_7Ef_o_;D%gG(H-(H0nL=@J
zT_;tbnnn@l6~x&V24^*gW47b3eo(~0Q1Dr<;EG2%93w+<U>c<>oWj3~IIkkk`C)LX
zSsd-R#Se)%7z;kX6<p5Z7#WHK4GwX#{wCtQhB&ZoeeivwH7t%cTgQj9BXgh`s4LJA
zCNy9yxG+nt=`*1AVJFXS2;HzD7@4+C>tsVnL}PI{S*q%@AC!dI^Ev`;2?KN*2V^!3
zo_<8^*cc2xp#J+F;ySjGAsssri^bC3Qaq)Piac*14?Idac!5r5d9>fE9}{^n7+h#N
zR)PB8&GHx-$`dy{KY@@?pEr?bQy8ALERQza=Z}m18e_rtIHs`tBQAlA4C&Wt(cy{6
zjw?5+3T=BrBzg;p&I>~X&jx+B3ea`Z^cGe9-^GrAvEV`<wJKC_@g}ap85z<M(qhBC
z18Il;L&SL-an22ca|Vl}4R`pHA`Zra3w_kaxsb&%G88ASaV`TL!Ef&%&N*Rl&SY`4
z-)?`3$iPW<XmfDx4NseoZNOOY;q<%g@8rZ=y?5e`m4SoP*xkB6O=SEB>;4aEYQoT*
z#nGf|ujM}@9#+6qaADA_;uLMJ=Z-ZQndD&w6{N95x;H#({_F!Wp+fH>)5b7NXR}P&
za8EreGGQ>dFmyK4xh#{Bp-gejgttK>pA(thL#7R3nAWpQ+I0(`7bg*hgA2{nVk)@c
zF|H>V88V5QnVZzlKu0Li`-lSvJq{j18#tV1+ICqlia3}JE_6~G=X@5&$WR=x9^P|r
z1RVkA1H^%ccL&GW$l_?n?fj>RgTdfJC$(|5uxmIYLvdgYhh{tEUn0(jh;vpLoEjEK
zo2}2kMH~zUUsEX9%HkLqf|Fs)wi9%O>U@MaXNJK!hr`LxX1nV@A`S+FuOAd_W^s%R
z#W7|({3Q|RW5j`j5(hWixh#&!)0agYOa>RasBO0M9^qPnk)b%oY<odRsLm&dvn~wI
zc`S}L+w-r8I2Z~pG*BC7(?$-*$Pk=N^L(-BRT1Y?#DPN$2d~a14kuHat?)Gw2lK#p
zQm)`!7RSg?9P@nfanKQ}^BLlt9tLMKi=)l<&Fdl#=7I0@LqQFTV`M0fdA_*n4H4&a
z#DOR52d~Z+7Dt<{_D%8b6!XA$mTAC~UAencBSY>^v!cV|ZSEPX);d?dB?5hcKx@JP
zZRLQnwB2rdTfF<hMDQKWDZKbq?(WCPkh`C(*sw(8<r`ZnqHgbqL|-CNbr_=aSt9MY
zoc|LYhoRsrQNj8et{)f~;<&8%Fz@bX1?UJ9=qtoo9R_C`i=!QP%e&(3Him+)K>ZgF
z<F*fs47uITjt+|z<L=wY_e7qrk!MvHo(njhY;Cx__eCB|1sD2`)vo&&_TqSq4CRRp
zi%lA;>~<>%3HA8~dEn&T!8<}N%cJe~@CTyZFcy3~1!uAi&d3nEWgELq|4_vF7I9XF
z!MTvd(RN$*k%)t_;5(^QaMlKH_81w0lM~gm=Y7x-s`DM<RE5F0h{MUzZhQA*5eH+z
zg)VBjt>6q6$H-8en8um>iHP$(;#7vgxtPV#W^+CjaWE8IXrVUFS{BF1P#j~nCqPH2
z&JT!F5eDZH7Dt=yt<OXpOavENsExCZeWq<>2u^cjwzZ#&I6opzc^I5aIh^L&Z1cYm
zaWD~F=%6;v=`4<sp*Y5DFMy6vou3eAMHrmRSR8G(=f4zjFc5r4mkZXgI7Wuz7_&|K
zO2qjYaZU+?b2*En&34q+A`a$(3k}q+&S__Joz2J)oEFAxJ3vRM&M$~l76#`p98L>u
zw$0y&IG6`6G*BC-n#D0P6vvqDC(sdaenp(+VQ{WsakSa~_*TThJaD0b+BmCN93w+<
zjM+APC*u5uILpG|T*=~Svu*lb#KAmpp@G^stIy)hW@HFXOJlb94<gQg5eH6LAAFIs
zox^FV&GtRW%z_7U>IyuR8`pq&;6eknaZY7%j10vwW?S~7i1Rz*l!n3C!QyDMZT(5a
z!8~xGf!a7#ERK<(IL2&;|19GC4{?@+!MTdX(PlgH7xAVP^T35EYT*pn-<i89H8SL;
zv{mw^^m<Sdn!z6kv^WgV)f`YOZMQdn6@f4jT$rRb(4R9nAR|M8V#DH{m#3+lghjuJ
zKz}08qA)<$ut3^yCI1x-hne83P(k^bTstr_#Bi;Q;r<CaLVXS(&cZM_*RnX;aG(7y
z-os%g_}=a4Kc$Gfhchzd9<FuMJ?TCFlXz+)$a8WSp6fWC*4l6H{vq;UD7eseEWh<{
zIhx}!GL#4INu62BZp;4^d7{X(APmp-ERVL^dj~|jVJi4|3d+vl>}F($-C7&F&4}a(
zcB_pc4!qcU@btNX#nE<K5Y3U&2UEd?Hfoip;FNV7j*%faZA_TH4mzo4Pi-7=P6~r_
zBZt#QyY1_kh=Zx%LKC%dma{lUhT@not&EE}3B;Kf2B(h2(Po>U5OFXQTxg*-&Qcb~
z$WR<(w);Rws7@N<%ngH6&*EsaJ(?!sU?RBCLT#L7YdNzS8G_T+n5}ENh?9;ubHd=<
z#No8nW;?!_h=YmXLI<^RN?9BuLvf7Rc7u*koo0wLI}FavERHr?U513yh=Jfj1GRA$
zvp7bE;uy2F&lGVo5NB2xoLg8NZMMEyA`a$(@0$S%mYmL+&Bzd(cE)UPfR0d|OvITP
z2Ip1|r=2$2r`aM7=79?h)UM7V7RSg?9Amcf91$lAab|?U*~#K)v(0TT;$R-Q&_HdR
zlUW=iLvf7R-Ul6_I@yRbJq*q+7Dt=y*A^lU=79?h)W%u3hBKRyAvo=$!@UP1=e87a
zau8=)7@XZ4PJ3;(3tNdem<KL2P#b3fi(_OcPE6xOT8lW%5oc-`oINa#Hrs$UA`a$(
z?@LGvPGWJ448@6SoIRi;n5_lkxM6T^V{x?EcC{68Fb`a4pmufU-_4oL$Pk<k(R9yj
zN467jS|ZMrFgUk!I32Xv`nMNxFb`a4pf=7t7RSg?oS4RG1RbF|tq^B&7@Rv;9BsBI
zJBT=#2QD;F8)puSV`L~!T;q)DDB`q6oRTm&cd|IzY&$#UMCQZRl)3`jQ<EDp4_uJ3
zaEi~|<Mgr5j30$(#*Iuzr+s=We8oH-RWD%GWQ~l#|KK62zMXUIZ>OUtD@$tIAk-KF
zRg`xLIr3QNeLLM<8cH1x45)c;V;z4;j_^&pjhGGYM1nTxhaTKF?HV!lO}nirT8kQZ
zc?Q|;0y%3WDcj`Qw#arOVJptNl#FDZH<EX2HZ_t#IS=mILvtbvp=Rm|)NN`bCWSkW
zzzylK!0Bh#?O3dvQ9H!xl-|~cJFaGZ4P2&lhvh_01~GMc$4Wg}JbUZbwnMxT1aENO
z<>VW~>b!5a+^g}_H->S79^fH|=R_8RmbwCf(;G26+|dMZ*tPK8&Nkrj2x#;Ua}G&b
z1rCm(F%#*^`_ZS=7Q+i5c@oH}D}Y#1+a3`|5yTYk+{d}oNP2uvcgcw?0U>n-_+~U>
zn7ANjO}ycIx;q1Gz8=VD#13~3P1?m119#$qSvV=J02g!>E!u&pVG4`h!&$UR(<5A;
zn-f_Ibm|Ha&T7PbafcJaksp^h18u^dNNCi$I)^76qtvuT6X|U>Z*b~HC`PB7=+lnK
zIFuAIg-`G0d}^tBl-;|F6A@#_g=ufqamW#`a}&|%sfpOs(%?iO`gCYw8?;dccn1gy
zF71SXFs#<jQ3{vt=Ui$8J-S^-<U|%h{nQnx;j~6fA$KI98`|PCr@*e^aacnmcBs=i
z=~7P&T&nO|e5>$|I8yXzXQqBBJbEAJ(I!QY@5rOXS%`t;4kLWSYo~Iv(5R_d*wodO
zn)c|>#O^<L|EUNYLGxr7Pjv;_j9YsM5)L8-OySV`Ifq)B9^<jc<Otsu-H5^DP9lu?
zb6(`$7dLvz_r(+HT^fwPTR>$x5UDFbHM;gtq=J5J*)qlZ;ty!5<on{F;-Tz4Rz$%N
z@<n?7>`OQlBVwV%00oafop_vxau}k}Bu|0T5JWNGnLpupvcB`Z%hHHp<Blg)>G$)I
z+`BABM15nP_xC0w;*nvo<@1%<UIrPVEQcct&9oFOjX@UkmHDsq5X*wO;$z95c`;WO
zBVv^$5itjaP7qPLAPV%NgFEcOAd2~d{PLb63dV<TU6ns$8;4><ER-}r!7t=}7gU6*
zbVU@pm`YKVzXnmvx8Z-2C!%0bxMPXK@~2<Op%@VhCEZk|vX_XGizqaXQlLB(L@{4@
zzr_(zFce(4Z?Hl(e_AbvVni&IW(MVdpdwVI8=^oM9Nb`k3!<1Wy#J-Qh=SeUx30~f
zdI5)GL@bmHgEF;`h|(QV=qe~hRUQtam@m8^-&bs5*z$c#*nU5C;aZpxv07NBYGJcg
z3;P$y2xU0}S%!ou%OgRSOr0zZ`63JUeBVN+-?&_k#fVreS-`Rszg6Y%ej>|}$TBz#
z%cDV-EX~rnK(rY4d>>2x<gJ{=jEH5iY-6#FpdyszC`6&lm=qR!EQpe=Q7Zb2DA@B|
zXq5KEFWJJO7!eC4$Dni=Afg<ND8*r_@^}y>N2AOu6j89}`)-HwC+*}=jEIHO9M&#f
z)MR-ZRD`M=gD6E|P@V{)G}kDf4ir(a=lfR8`4cyBC`QCWX<@3ezDPtl7EuO<LHT<S
zrG-XWUo4_v&v&6w+6J3&9*1H?ER>c8<$F*Os&X8n6ox_hM-Zi@MtN_Lh=M)eg+6Jc
zjA#E=7$ah#v@$5m2a71jBg%j<C{G4aT4|K=Lqrs8`7X3c8)e)%Tvd#Sh0@xf{0u5W
zReB&w|1c;|1yNdSl-GueDA@5`Xp=U|*cuMSh*&6X49bRKBFYJfQV<5^=^#oQjq>Ag
z5d}NG3vJRy8MBc?F(MX9TZ2+QLPY6_DE-2qJQGA|t5Ld)BD)Oq?rU-*c6_%lK`DNv
zm~7V8dz-aJL~YhK$L-TZyJ$2kFOoJip?+3Pc@L<~f%{^01@4U})aD^lFM>26??Q4Y
zug?2omS;7RI+RD%yc^`VP87MY{kwfA*B3W%Tt<m<ZGgXAaA-pPthkns7P)#MS6&#d
z=U6VK<8fU+hU}Tpdby351ul%BWxc^^J<0BKy|?>pl+^BX%cgq49$Qw;c(n_a^{gUm
zH8v+Q52~fEK*c82I!M-okPXbc*l}XXZyl~X0{6Y1*JSOTwgEklZQZyW;d{IqF&une
zEbM+J_a33qQtuHqxg(}rs5faM<HdsYMywMERth7&z!eOoJg8GAh)@_4J|h<1aXSZP
zlqi(41`M{URhx+-R3D~RDWG0tp^`4#J&A0+QWusqVqo}OSoGFt^7u;s<w{0LJ-%vk
zVa#)(uGR4+q6PaR*0H2kDJ=L;&VpLZtJd?A#kPU*;r1lctN4uyu5B1C(l+8=J5+x)
zsVdm)DPqC$k?c4^mZD();tGap9@o>Z$c0(rg5RySQCxaG$7Pf#m$Jst1YjZ8_^BdS
zKc-?Sxc<#@C5?FFG_f6GptvxC7T4gL3%Pb^lt?>_n|65Mz_&;?YP#sf0wg<%R4j!T
z|HFAv>v=W1d4||IFjRbAEbOw5>l{XlbPfz<_z=p7V`hp4>yKDR6RZ>kdx<L;N_kLQ
zXNgdlCq5$<?%Kma86^s(tf8S=&K98tFtthn^)d^Ubm5>mVmrh%@wu?*)lpnKG)kl$
zCQLiTE}xhyTCfnYx|3R^u;42iizKoZ^Qtv>p4c`pQQTw5^eTR*j%yo6%W50#qfzJ=
zk)a9ovvT1ZpeC#i2O`-Kge*nDUJVvZt9e{6og{K$q`2UBt8ElNJ&NNpN|X!QJaDO>
z6_-0-<SJq+mV)awmP_e)T*oaCPkLaWxJMGML0K<wCq0akI_c5ETOGm(f)N!MJb^{9
zI*^+S#Zy;caX7KI7{PK$!3y&(B9pn!`{LEtHJF;rJ~@x<g_DUCU2#lo#7OZuu`u@z
zB2^!h(rd%^(U|g}UT-W~D3)swLzSXjZ*b*8DG%!LMIscYiO+_G?Kf~xM#*Z10Hutf
zp(ZaDp#~#VS5m4JP;atONe?bsLi!V})TBnt5}yYL&bpS|G1Pl^3`R-yrzTUV<ysTf
zZ=fX@a0pTzPN-5C@GZ`Oo|aduzDtP|#hTcNY2w2w?82_&ZK);$#ytai2OB_2=z~KU
zsuZ<)o2wN{c~Gw{6QM9mT<Ew~A1rLQo-?3PA_i0oP%t3&ibcyss9^|o7%5c>sCQVX
zqycx7i3Y?Nak~+yfzz+!3}}>y0o59<iR#Ex$N<u_ZWC*VBh?{<Dun_6k29dB<<)BF
z3Nf=VMcgg~t9bkjF0+gj$*g9|fXTx(&w!fX!4V8tigLZnl?&B8uFe%A7lw!nbJ+4=
z@iXi#mr<fz${5Kt#!gTZxJDvZXHu{fT<@`5NgF;_DP}Roh<hmE8g!z|WwB8rS=>x#
zu_~KdWQ?p5oj3}?IufuHPJEwpq9^B7?822q%HIkxOMFf&Y`>H18n#qZE6h+x@Fnt!
zs_Us@xlUxLQk3fhu3RYPL5*G|LSdBnY*?7h-fbBr(h4)YR*26&9|0-BgQF3u11VJs
zs1I4Fqz8|$7F!`EiO+)rORnHrp;02OFhjROq}mHwLKcrfs&<4bg#ka}4CrZjwR&p}
zk@6c5v&4s0*zQ)&fVNbV0W&=V;%Tl0r-`*1%TT4L)yG_|P|Abae7XoVTq(H_B~~9S
z%v{Ss86{%COwWJ_)qbrAH4dTLl2WCB`h<l_8nDwk(SR5uZYKgYZ~|*UqeKjtX)gx%
zf|g*w@krI0P^B>7r<?&jEw5G&o*`xyrij~~U==^Noy%e)MKUW}88Dey^UoB^HGu(3
zQLfLpa-o{Xwf8KM3q!<(Ic#~b_|Y<s%P3JUWenIRJm5vvz_UfJiOAK86f6bT=PXy!
zhV#~oS&T8_wjo@Disy4#Y?MeAXX`9ByM)JV5S=&)!CDZo6i)ntbD}5bRm|B)r2MTA
zv&84b!kl`pYuHjvt+2UrqJ9x)7f1=QRl-oEDA$)<xlqc3+Fv6=VU+l6SQxG5po|h}
zh0T>QG*tdMBGhDrYEDX(0_rOkD(S&x=ZdWmlf>u2fhW=@Y5Km{D3Mm!96hM1egG|@
z6;45_Y(ka7fM0V4^t8NM_1YxvePWilFo~^}QS|e>+}@{=vi3eZz<X1Q3$}DIxb6ip
z;c29cT$zL`#op&PnoEtLe(uyH=7GJnS?q-vCoW843#|CGIb1I^TB;X%?=vK{R`R8(
zd0WUZ(i_Q=+Nns^jF6>xnc-VamV6=1QuEL@Zxx@KV4%2Q4-0MZtygeQO^ld&YSPBr
zCPXy)%G3eSTL6tjU4ebVDYerO4t5o-^Q43FE+g%??r407{GEnV?U&KhmfR-3tAV-V
zW)Qrgo8sKN8b(gNtI_0=m}g1*g{i%uDEM+Z0;Utd6u$hP^QBhw=<d2ed>;(M#RV;^
zc*9n-<=zJ~TFLjpz$vKZdtIuqRy5}frhX~R`2%N8BkA#tx-chF3Pw>En1v?V%tnkE
zH%|CQT;V!})`^4q6VNP1tc!D4(kvjxye#zuhzbs!iG)#7z!VPsk#lHL)MI?*B5?v@
z>bNl5tr`yQb|E(bjhyw^vO_fPIW)PObIQe{KW8BxjHxvNQ~2{I&YxP(W7~9z_#PN$
zkDEr=hV(jvdk@TLCEo)Bf1(z7Q7ZRR(VerI+NE&k&zw7rpmyhF;_Sl&a$|&U*p>g~
zW}gvLv#+U@fmq-*smDQ5@a7!Fx8YAxycFL2h4W@o(j$EHax$EB{*^Rh68XGY^yS-J
zpc^?7=m`_(xM{HbFQPH$V(lV?Erl_E<&3GtJg_xah=GnN<TGY*1-pT2v`C<%76v+!
zeG6(QK?_q?AoEIU=P^}F;mY4QR~j`BEwWwgCzwAzR}Q}JQm&sEF%sw?))eRncMs?Z
zmOKgJekWy1Vafk;mQ3n-c(3jd&s$>t_|_}=9na#<TN*j*yk$po-g4Dd;(5#Y=ta6p
zNpX(<cMWB|xYtoXq1}FUPNWPhp{_u>&27Y>neq$iAEOTxID_nTb1>bENOvbUy_2`}
z>Xh&vsl%TgiQu|9xrs338WO8?fjP5w0ha6sQnC?wJ4me7d9nIGO{ZctrLb4H4cF#G
zPJz#&t^n=4M$DZlKM~q76Nfm1ZQ9;QYebK5x}|qc(yH65nhiA@6fpcqK6B5#j=XbC
zac9+@jJV$s+>v=#k$0}^ymzku(75WI>(t5~Z|U_pk#hJP>I(4AZ^S5?@;%`lxA1po
zsLk6Kd5!8(&JpQ{CVBBDPXX7gRJ@AVJdDq{At$mNen(w_hBK#jArgO0h)>MB#woV6
z>w2N~pPIOv(^b^=m~(Fw9gXoc<wwFi`ghjRMwdDopO-6CzoV}PVQZmCOFXl75dwcp
zfKxd70PAR<ut)n`o#<!`EMJAkOs(J?ZA7V~@tHkwwAn2iTQ53#F;idO(IxP3Mg8MY
zNBfmM-W4~Aj>Z_9@(ro(xL*e|j&4$=j>gphPT{1Z%{JP*pe;Ch2@-!n3d}pYL>&oJ
z#MIG2ZI3zo7BbRw1vIM>)6M7Tk@5AUchqC=Xwp?9ooto(06ytx_+aW$*A`F}>|Bbp
zUlCf~&LuHTOB=LL)kEEXYffYtd=zyB8rJMa%sii+M_swjDYBYIeV%F>O`^F@7pqO;
z2>{E^n)R@qWQ6I|omIP(scZ@_$2BYUvR~GN9kNU8R2YFiFOTjqfyv$yqpCVpvO9ZE
z>+SBWabEyo!N|)H_ft~Z6h=;HTx#T?um}9yZV?zma0*0%b(=XF&YlMs8zn}Uw&nzy
zSpB3I8<0hKf@SI+(aXyb_;UiB!pm@AA7?K0vQOBft+-8W%@}}Fz9h6`#&fN?#E8<?
zjOY0*FDKVlUxD&c7zK3&qIXtp8B^aBc1~yQ>{s@9f4p66&5M-kl+Ot7xOazet=Xv3
z)~rr~LS@Nf0|2pE?YcvB^eIUE5h-vAM>k^~9n|)iufLOww7)fD2>Kj7^7k{j)@*cX
zYfe*u_yZlSx1BrRCE9rf(tbi{Q`k9!wX;vvLp|<pu{C4*`RqKZmTk>Ol(yzH-`ZKT
zJ_oXbm&=*Trtoqm>t(;J2m9Gxu~T9A`Mf;38{3+VDs9ba0WWLZQ}&5Qu0Y)PNoi9U
zIg2%NP}l?Bat|41zmYKkr@-3AYRw~mS;raK=u#u68y|bESs$l~+$$Qn5@|mqv?+|7
z%^KOK>Y;YqFB%!s&sW<~TUa9-QEKG$fRS;J;xUjFj9kT3HieOMSR?yoJ=o{(6OD|?
WH|1ke*U^WuMmDO{$mxuc<NptxMr)-2

literal 0
HcmV?d00001

diff --git a/testing/btest/scripts/base/protocols/snmp/snmp-addr.bro b/testing/btest/scripts/base/protocols/snmp/snmp-addr.bro
new file mode 100644
index 0000000000..5c21cf7be3
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/snmp/snmp-addr.bro
@@ -0,0 +1,15 @@
+# @TEST-EXEC: bro -C -b -r $TRACES/snmp/snmpwalk-short.pcap %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+@load base/protocols/snmp
+
+event snmp_response(c: connection, is_orig: bool, header: SNMP::Header, pdu: SNMP::PDU) {
+
+	for (i in pdu$bindings) {
+		local binding = pdu$bindings[i];
+
+		if (binding$value?$address)
+			print binding$value$address;
+	}
+
+}

From 93db6cd8766fabf295a6372f2d1762d70045abb0 Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@icir.org>
Date: Tue, 26 Jul 2016 15:48:47 -0700
Subject: [PATCH 12/25] Fix behavior of connection_pending event

It is now really only raised when Bro is terminating. Also adds a
test-case that raises the event.
---
 src/analyzer/protocol/tcp/TCP.cc                           | 2 +-
 .../Baseline/scripts.base.protocols.tcp.pending/.stdout    | 1 +
 testing/btest/scripts/base/protocols/tcp/pending.bro       | 7 +++++++
 3 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.tcp.pending/.stdout
 create mode 100644 testing/btest/scripts/base/protocols/tcp/pending.bro

diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc
index 17b99abb39..bd187fc9ac 100644
--- a/src/analyzer/protocol/tcp/TCP.cc
+++ b/src/analyzer/protocol/tcp/TCP.cc
@@ -355,7 +355,7 @@ void TCP_Analyzer::Done()
 	{
 	Analyzer::Done();
 
-	if ( connection_pending && is_active && ! BothClosed() )
+	if ( connection_pending && is_active && ! BothClosed() && terminating )
 		Event(connection_pending);
 
 	LOOP_OVER_GIVEN_CHILDREN(i, packet_children)
diff --git a/testing/btest/Baseline/scripts.base.protocols.tcp.pending/.stdout b/testing/btest/Baseline/scripts.base.protocols.tcp.pending/.stdout
new file mode 100644
index 0000000000..84858cb67b
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.tcp.pending/.stdout
@@ -0,0 +1 @@
+1469573308.013636, Connection pending, [orig_h=192.168.4.149, orig_p=55881/tcp, resp_h=74.125.239.152, resp_p=443/tcp], ShADad
diff --git a/testing/btest/scripts/base/protocols/tcp/pending.bro b/testing/btest/scripts/base/protocols/tcp/pending.bro
new file mode 100644
index 0000000000..1a49f5d19b
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/tcp/pending.bro
@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -C -r $TRACES/tls/chrome-34-google.trace %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event connection_pending(c: connection)
+	{
+	print current_time(), "Connection pending", c$id, c$history;
+	}

From 479ed5b67afbfae84ad34316d1501533aa44e503 Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@icir.org>
Date: Sun, 31 Jul 2016 10:05:50 -0400
Subject: [PATCH 13/25] Also retired remove -J/-K options (set md5/hash key)
 from the manpage.

---
 man/bro.8 | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/man/bro.8 b/man/bro.8
index fb2b44a420..1e9d418e7a 100644
--- a/man/bro.8
+++ b/man/bro.8
@@ -78,12 +78,6 @@ force DNS
 \fB\-I\fR,\ \-\-print\-id <ID name>
 print out given ID
 .TP
-\fB\-J\fR,\ \-\-set\-seed <seed>
-set the random number seed
-.TP
-\fB\-K\fR,\ \-\-md5\-hashkey <hashkey>
-set key for MD5\-keyed hashing
-.TP
 \fB\-N\fR,\ \-\-print\-plugins
 print available plugins and exit (\fB\-NN\fR for verbose)
 .TP

From d04239f4183345aae361b9f3efd9e1b972c777d2 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 07:57:10 -0700
Subject: [PATCH 14/25] Updating submodule(s).

 [nomail]
---
 aux/bro-aux | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/bro-aux b/aux/bro-aux
index dcc64f4ab9..e34845463b 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit dcc64f4ab9f5d80f808aaaf39979525e22817019
+Subproject commit e34845463b9f9040eeb43e4081876964f08680d7

From 1f8eb0bbc3299c7cfe0fa9b98a22088baa7b10e5 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 07:57:41 -0700
Subject: [PATCH 15/25] Updating submodule(s).

 [nomail]
---
 aux/bro-aux | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/bro-aux b/aux/bro-aux
index e34845463b..5d529824d8 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit e34845463b9f9040eeb43e4081876964f08680d7
+Subproject commit 5d529824d8e12a31806a7075373a1ee285e0d350

From 19d66be0aa81d0be6f805b0c8b759f121bf93a7e Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 08:32:41 -0700
Subject: [PATCH 16/25] Removing pkg/make-*-packages scripts.

We aren't using them anymore for the packages we distribute. Because
of that, they haven't been supported in a while, and have problems.

BIT-1509 #closed
---
 CHANGES               |  5 ++++
 NEWS                  |  5 ++++
 VERSION               |  2 +-
 pkg/make-deb-packages | 46 ----------------------------------
 pkg/make-mac-packages | 57 -------------------------------------------
 pkg/make-rpm-packages | 39 -----------------------------
 6 files changed, 11 insertions(+), 143 deletions(-)
 delete mode 100755 pkg/make-deb-packages
 delete mode 100755 pkg/make-mac-packages
 delete mode 100755 pkg/make-rpm-packages

diff --git a/CHANGES b/CHANGES
index 1f72911215..3f9b96ef5f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,9 @@
 
+2.4-732 | 2016-08-01 08:33:00 -0700
+
+  * Removing pkg/make-*-packages scripts. BIT-1509 #closed (Robin
+    Sommer)
+
 2.4-731 | 2016-08-01 08:14:06 -0700
 
   * Correct endianness of IP addresses in SNMP. Addresses BIT-1644.
diff --git a/NEWS b/NEWS
index eff5ec8ad6..13542d73d7 100644
--- a/NEWS
+++ b/NEWS
@@ -191,6 +191,11 @@ Removed Functionality
     - The command line options --set-seed and --md5-hashkey have been
       removed.
 
+    - The packaging scripts pkg/make-*-packages are gone. They aren't
+      used anymore for the binary Bro packages that the projects
+      distributes; haven't been supported in a while; and have
+      problems.
+
 Deprecated Functionality
 ------------------------
 
diff --git a/VERSION b/VERSION
index c7e8fbeda0..8cae8eeb0a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4-731
+2.4-732
diff --git a/pkg/make-deb-packages b/pkg/make-deb-packages
deleted file mode 100755
index 36bd62c19c..0000000000
--- a/pkg/make-deb-packages
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/bin/sh
-
-# This script generates binary DEB packages.
-# They can be found in ../build/ after running.
-
-# The DEB CPack generator depends on `dpkg-shlibdeps` to automatically
-# determine what dependencies to set for the packages
-type dpkg-shlibdeps > /dev/null 2>&1 || {
-    echo "\
-Creating DEB packages requires the "dpkg-shlibs" command, usually provided by
-the 'dpkg-dev' package, please install it first.
-" >&2;
-    exit 1;
-}
-
-prefix=/opt/bro
-localstatedir=/var/opt/bro
-
-# During the packaging process, `dpkg-shlibs` will fail if used on a library
-# that links to other internal/project libraries unless an RPATH is used or
-# we set LD_LIBRARY_PATH such that it can find the internal/project library
-# in the temporary packaging tree.
-export LD_LIBRARY_PATH=./${prefix}/lib
-
-cd ..
-
-# Minimum Bro
-./configure --prefix=${prefix} --disable-broccoli --disable-broctl \
-            --pkg-name-prefix=Bro-minimal --binary-package
-( cd build && make package )
-
-# Full Bro package
-./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package
-( cd build && make package )
-
-# Broccoli
-cd aux/broccoli
-./configure --prefix=${prefix} --binary-package
-( cd build && make package && mv *.deb ../../../build/ )
-cd ../..
-
-# Broctl
-cd aux/broctl
-./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package
-( cd build && make package && mv *.deb ../../../build/ )
-cd ../..
diff --git a/pkg/make-mac-packages b/pkg/make-mac-packages
deleted file mode 100755
index b3d200842f..0000000000
--- a/pkg/make-mac-packages
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/bin/sh
-
-# This script creates binary packages for Mac OS X.
-# They can be found in ../build/ after running.
-
-type sw_vers > /dev/null 2>&1 || {
-    echo "Unable to get Mac OS X version" >&2;
-    exit 1;
-}
-
-# Get the OS X minor version
-# 5 = Leopard, 6 = Snow Leopard, 7 = Lion ...
-osx_ver=`sw_vers | sed -n 's/ProductVersion://p' | cut -d . -f 2`
-
-if [ ${osx_ver} -lt 5 ]; then
-    echo "Packages for OS X < 10.5 are not supported" >&2
-    exit 1
-elif [ ${osx_ver} -eq 5 ]; then
-    # On OS X 10.5, the x86_64 version of libresolv is broken,
-    # so we build for i386 as the easiest solution
-    arch=i386
-else
-    # Currently it's just easiest to build the 10.5 package on
-    # on 10.5, but if it weren't for the libresolv issue, we could
-    # potentially build packages for older OS X version by using the
-    # --osx-sysroot and --osx-min-version options
-    arch=x86_64
-fi
-
-prefix=/opt/bro
-
-cd ..
-
-# Minimum Bro
-CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
-    --disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \
-    --binary-package
-( cd build && make package )
-
-# Full Bro package
-CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
-    --pkg-name-prefix=Bro --binary-package
-( cd build && make package )
-
-# Broccoli
-cd aux/broccoli
-CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
-    --binary-package
-( cd build && make package && mv *.dmg ../../../build/ )
-cd ../..
-
-# Broctl
-cd aux/broctl
-CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
-    --binary-package
-( cd build && make package && mv *.dmg ../../../build/ )
-cd ../..
diff --git a/pkg/make-rpm-packages b/pkg/make-rpm-packages
deleted file mode 100755
index ee09511e44..0000000000
--- a/pkg/make-rpm-packages
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/sh
-
-# This script generates binary RPM packages.
-# They can be found in ../build/ after running.
-
-# The RPM CPack generator depends on `rpmbuild` to create packages
-type rpmbuild > /dev/null 2>&1 || {
-    echo "\
-Creating RPM packages requires the "rpmbuild" command, usually provided by
-the 'rpm-build' package, please install it first.
-" >&2;
-    exit 1;
-}
-
-prefix=/opt/bro
-localstatedir=/var/opt/bro
-
-cd ..
-
-# Minimum Bro
-./configure --prefix=${prefix} --disable-broccoli --disable-broctl \
-            --pkg-name-prefix=Bro-minimal --binary-package
-( cd build && make package )
-
-# Full Bro package
-./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package
-( cd build && make package )
-
-# Broccoli
-cd aux/broccoli
-./configure --prefix=${prefix} --binary-package
-( cd build && make package && mv *.rpm ../../../build/ )
-cd ../..
-
-# Broctl
-cd aux/broctl
-./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package
-( cd build && make package && mv *.rpm ../../../build/ )
-cd ../..

From 420bef58aa031d54ea45e18c27227b63befe949f Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 08:59:51 -0700
Subject: [PATCH 17/25] Fixing a CMake dependency issue for the pcap bifs.

ninja said:

    ninja: warning: multiple rules generate
    scripts/base/bif/const.bif.bro. builds involving this target will
    not be correct; continuing anyway [-w dupbuild=warn]

Looks like there's a larger problem here involving *.bif of the same
name at different locations of the source tree. For now, I'ved fixed
this one by merging src/iosource/pcap/{const,functions}.bif into
pcap.bif.
---
 CHANGES                                       |  4 ++++
 VERSION                                       |  2 +-
 src/iosource/PktSrc.cc                        |  2 +-
 src/iosource/pcap/CMakeLists.txt              |  3 +--
 src/iosource/pcap/Dumper.cc                   |  2 +-
 src/iosource/pcap/Source.cc                   |  2 +-
 src/iosource/pcap/const.bif                   |  4 ----
 src/iosource/pcap/{functions.bif => pcap.bif} |  3 +++
 .../canonified_loaded_scripts.log             |  6 ++---
 .../canonified_loaded_scripts.log             |  6 ++---
 testing/btest/Baseline/plugins.hooks/output   | 22 +++++++++----------
 11 files changed, 29 insertions(+), 27 deletions(-)
 delete mode 100644 src/iosource/pcap/const.bif
 rename src/iosource/pcap/{functions.bif => pcap.bif} (98%)

diff --git a/CHANGES b/CHANGES
index 3f9b96ef5f..99f66268e8 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.4-733 | 2016-08-01 09:09:29 -0700
+
+  * Fixing a CMake dependency issue for the pcap bifs. (Robin Sommer)
+
 2.4-732 | 2016-08-01 08:33:00 -0700
 
   * Removing pkg/make-*-packages scripts. BIT-1509 #closed (Robin
diff --git a/VERSION b/VERSION
index 8cae8eeb0a..34ea98de55 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4-732
+2.4-733
diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc
index a56ba90e86..a9362a0b62 100644
--- a/src/iosource/PktSrc.cc
+++ b/src/iosource/PktSrc.cc
@@ -11,7 +11,7 @@
 #include "Net.h"
 #include "Sessions.h"
 
-#include "pcap/const.bif.h"
+#include "pcap/pcap.bif.h"
 
 using namespace iosource;
 
diff --git a/src/iosource/pcap/CMakeLists.txt b/src/iosource/pcap/CMakeLists.txt
index cf9f577760..fbfffff051 100644
--- a/src/iosource/pcap/CMakeLists.txt
+++ b/src/iosource/pcap/CMakeLists.txt
@@ -5,6 +5,5 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI
 
 bro_plugin_begin(Bro Pcap)
 bro_plugin_cc(Source.cc Dumper.cc Plugin.cc)
-bif_target(functions.bif)
-bif_target(const.bif)
+bif_target(pcap.bif)
 bro_plugin_end()
diff --git a/src/iosource/pcap/Dumper.cc b/src/iosource/pcap/Dumper.cc
index 20e36420c6..e6e400477e 100644
--- a/src/iosource/pcap/Dumper.cc
+++ b/src/iosource/pcap/Dumper.cc
@@ -7,7 +7,7 @@
 #include "../PktSrc.h"
 #include "../../Net.h"
 
-#include "const.bif.h"
+#include "pcap.bif.h"
 
 using namespace iosource::pcap;
 
diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc
index 8158266f1c..0a0633ece0 100644
--- a/src/iosource/pcap/Source.cc
+++ b/src/iosource/pcap/Source.cc
@@ -7,7 +7,7 @@
 #include "Source.h"
 #include "iosource/Packet.h"
 
-#include "const.bif.h"
+#include "pcap.bif.h"
 
 #ifdef HAVE_PCAP_INT_H
 #include <pcap-int.h>
diff --git a/src/iosource/pcap/const.bif b/src/iosource/pcap/const.bif
deleted file mode 100644
index 877dccef74..0000000000
--- a/src/iosource/pcap/const.bif
+++ /dev/null
@@ -1,4 +0,0 @@
-
-
-const Pcap::snaplen: count;
-const Pcap::bufsize: count;
diff --git a/src/iosource/pcap/functions.bif b/src/iosource/pcap/pcap.bif
similarity index 98%
rename from src/iosource/pcap/functions.bif
rename to src/iosource/pcap/pcap.bif
index 0ad057f2ec..b502342d3c 100644
--- a/src/iosource/pcap/functions.bif
+++ b/src/iosource/pcap/pcap.bif
@@ -1,6 +1,9 @@
 
 module Pcap;
 
+const snaplen: count;
+const bufsize: count;
+
 ## Precompiles a PCAP filter and binds it to a given identifier.
 ##
 ## id: The PCAP identifier to reference the filter *s* later on.
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index 1975988224..01400b2393 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-07-12-17-20-37
+#open	2016-08-01-16-08-40
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -52,7 +52,7 @@ scripts/base/init-bare.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/stats.bif.bro
     build/scripts/base/bif/broxygen.bif.bro
-    build/scripts/base/bif/functions.bif.bro
+    build/scripts/base/bif/pcap.bif.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
@@ -135,4 +135,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2016-07-12-17-20-37
+#close	2016-08-01-16-08-40
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 2d5ac86e35..d2e5235d46 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2016-07-12-17-20-44
+#open	2016-08-01-16-08-53
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -52,7 +52,7 @@ scripts/base/init-bare.bro
   build/scripts/base/bif/__load__.bro
     build/scripts/base/bif/stats.bif.bro
     build/scripts/base/bif/broxygen.bif.bro
-    build/scripts/base/bif/functions.bif.bro
+    build/scripts/base/bif/pcap.bif.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
@@ -312,4 +312,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2016-07-12-17-20-44
+#close	2016-08-01-16-08-53
diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output
index 3222c047d7..9dfbfce6f3 100644
--- a/testing/btest/Baseline/plugins.hooks/output
+++ b/testing/btest/Baseline/plugins.hooks/output
@@ -240,7 +240,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1468432721.269887, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1470067740.18502, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Communication::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Conn::LOG)) -> <no result>
@@ -364,7 +364,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1468432721.269887, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1470067740.18502, node=bro, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -397,7 +397,7 @@
 0.000000   MetaHookPost  CallFunction(reading_live_traffic, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(reading_traces, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(set_to_regex, <frame>, ({}, (^\.?|\.)(~~)$)) -> <no result>
-0.000000   MetaHookPost  CallFunction(strftime, <frame>, (%Y, 1468432721.269431)) -> <no result>
+0.000000   MetaHookPost  CallFunction(strftime, <frame>, (%Y, 1470067740.184492)) -> <no result>
 0.000000   MetaHookPost  CallFunction(string_to_pattern, <frame>, ((^\.?|\.)()$, F)) -> <no result>
 0.000000   MetaHookPost  CallFunction(sub, <frame>, ((^\.?|\.)(~~)$, <...>/, )) -> <no result>
 0.000000   MetaHookPost  CallFunction(to_count, <frame>, (2016)) -> <no result>
@@ -506,7 +506,6 @@
 0.000000   MetaHookPost  LoadFile(./exec) -> -1
 0.000000   MetaHookPost  LoadFile(./file_analysis.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./files) -> -1
-0.000000   MetaHookPost  LoadFile(./functions.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./gridftp) -> -1
 0.000000   MetaHookPost  LoadFile(./hll_unique) -> -1
 0.000000   MetaHookPost  LoadFile(./hooks.bif.bro) -> -1
@@ -530,6 +529,7 @@
 0.000000   MetaHookPost  LoadFile(./openflow) -> -1
 0.000000   MetaHookPost  LoadFile(./packetfilter) -> -1
 0.000000   MetaHookPost  LoadFile(./patterns) -> -1
+0.000000   MetaHookPost  LoadFile(./pcap.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./plugin) -> -1
 0.000000   MetaHookPost  LoadFile(./plugins) -> -1
 0.000000   MetaHookPost  LoadFile(./polling) -> -1
@@ -908,7 +908,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1468432721.269887, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1470067740.18502, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Conn::LOG))
@@ -1032,7 +1032,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1468432721.269887, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1470067740.18502, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1065,7 +1065,7 @@
 0.000000   MetaHookPre   CallFunction(reading_live_traffic, <frame>, ())
 0.000000   MetaHookPre   CallFunction(reading_traces, <frame>, ())
 0.000000   MetaHookPre   CallFunction(set_to_regex, <frame>, ({}, (^\.?|\.)(~~)$))
-0.000000   MetaHookPre   CallFunction(strftime, <frame>, (%Y, 1468432721.269431))
+0.000000   MetaHookPre   CallFunction(strftime, <frame>, (%Y, 1470067740.184492))
 0.000000   MetaHookPre   CallFunction(string_to_pattern, <frame>, ((^\.?|\.)()$, F))
 0.000000   MetaHookPre   CallFunction(sub, <frame>, ((^\.?|\.)(~~)$, <...>/, ))
 0.000000   MetaHookPre   CallFunction(to_count, <frame>, (2016))
@@ -1174,7 +1174,6 @@
 0.000000   MetaHookPre   LoadFile(./exec)
 0.000000   MetaHookPre   LoadFile(./file_analysis.bif.bro)
 0.000000   MetaHookPre   LoadFile(./files)
-0.000000   MetaHookPre   LoadFile(./functions.bif.bro)
 0.000000   MetaHookPre   LoadFile(./gridftp)
 0.000000   MetaHookPre   LoadFile(./hll_unique)
 0.000000   MetaHookPre   LoadFile(./hooks.bif.bro)
@@ -1198,6 +1197,7 @@
 0.000000   MetaHookPre   LoadFile(./openflow)
 0.000000   MetaHookPre   LoadFile(./packetfilter)
 0.000000   MetaHookPre   LoadFile(./patterns)
+0.000000   MetaHookPre   LoadFile(./pcap.bif.bro)
 0.000000   MetaHookPre   LoadFile(./plugin)
 0.000000   MetaHookPre   LoadFile(./plugins)
 0.000000   MetaHookPre   LoadFile(./polling)
@@ -1575,7 +1575,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1468432721.269887, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1470067740.18502, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1699,7 +1699,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1468432721.269887, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1470067740.18502, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()
@@ -1732,7 +1732,7 @@
 0.000000 | HookCallFunction reading_live_traffic()
 0.000000 | HookCallFunction reading_traces()
 0.000000 | HookCallFunction set_to_regex({}, (^\.?|\.)(~~)$)
-0.000000 | HookCallFunction strftime(%Y, 1468432721.269431)
+0.000000 | HookCallFunction strftime(%Y, 1470067740.184492)
 0.000000 | HookCallFunction string_to_pattern((^\.?|\.)()$, F)
 0.000000 | HookCallFunction sub((^\.?|\.)(~~)$, <...>/, )
 0.000000 | HookCallFunction to_count(2016)

From c3a43274e457a79ef2f1cda4ac601d5eca66d939 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 08:24:28 -0700
Subject: [PATCH 18/25] Revert "Fix some failing plugin tests on OS X 10.11"

This reverts commit a4e5591e18cc497c16a06eb0d3bf44ba388f8362.
---
 testing/btest/plugins/file-plugin/CMakeLists.txt     | 3 ---
 testing/btest/plugins/protocol-plugin/CMakeLists.txt | 3 ---
 2 files changed, 6 deletions(-)

diff --git a/testing/btest/plugins/file-plugin/CMakeLists.txt b/testing/btest/plugins/file-plugin/CMakeLists.txt
index 1d0941d9da..4823ddb08f 100644
--- a/testing/btest/plugins/file-plugin/CMakeLists.txt
+++ b/testing/btest/plugins/file-plugin/CMakeLists.txt
@@ -9,9 +9,6 @@ endif ()
 
 set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake)
 
-find_package(OpenSSL)
-include_directories(${OPENSSL_INCLUDE_DIR})
-
 include(BroPlugin)
 
 bro_plugin_begin(Demo Foo)
diff --git a/testing/btest/plugins/protocol-plugin/CMakeLists.txt b/testing/btest/plugins/protocol-plugin/CMakeLists.txt
index a10fff1d67..4bc8460c06 100644
--- a/testing/btest/plugins/protocol-plugin/CMakeLists.txt
+++ b/testing/btest/plugins/protocol-plugin/CMakeLists.txt
@@ -9,9 +9,6 @@ endif ()
 
 set(CMAKE_MODULE_PATH ${BRO_DIST}/cmake)
 
-find_package(OpenSSL)
-include_directories(${OPENSSL_INCLUDE_DIR})
-
 include(BroPlugin)
 
 bro_plugin_begin(Demo Foo)

From d7c10ca7c36ff5446bee4a623bcb1020e19482b8 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 08:27:08 -0700
Subject: [PATCH 19/25] Removing OpenSSL dependency for plugins.

Compiling a plugin required having access to OpenSSL headers because
they were pulled in by Bro headers that the plugin had to include.
Removinng then OpenSSL dependency from those Bro headers.

I'm also reverting a4e5591e. This is a different fix for the same
problem, and reverting that commit gives us a test case. :-)
---
 src/ChunkedIO.cc                              |  2 ++
 src/ChunkedIO.h                               |  8 +++++---
 src/File.cc                                   |  4 ++++
 src/File.h                                    | 10 +++++-----
 src/file_analysis/analyzer/x509/X509.cc       |  1 +
 src/file_analysis/analyzer/x509/functions.bif |  2 ++
 src/main.cc                                   |  3 ++-
 7 files changed, 21 insertions(+), 9 deletions(-)

diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc
index 0c402dc2af..d2cdbc6425 100644
--- a/src/ChunkedIO.cc
+++ b/src/ChunkedIO.cc
@@ -5,7 +5,9 @@
 #include <sys/time.h>
 #include <netinet/in.h>
 #include <assert.h>
+
 #include <openssl/ssl.h>
+#include <openssl/err.h>
 
 #include <algorithm>
 
diff --git a/src/ChunkedIO.h b/src/ChunkedIO.h
index 238bea5044..de3e143b66 100644
--- a/src/ChunkedIO.h
+++ b/src/ChunkedIO.h
@@ -14,9 +14,6 @@
 # include <krb5.h>
 #endif
 
-#include <openssl/ssl.h>
-#include <openssl/err.h>
-
 class CompressedChunkedIO;
 
 // #define DEBUG_COMMUNICATION 10
@@ -244,6 +241,11 @@ private:
 	bro::Flare read_flare;
 };
 
+// From OpenSSL. We forward-declare these here to avoid introducing a
+// dependency on OpenSSL headers just for this header file.
+typedef struct ssl_ctx_st SSL_CTX;
+typedef struct ssl_st SSL;
+
 // Chunked I/O using an SSL connection.
 class ChunkedIOSSL : public ChunkedIO {
 public:
diff --git a/src/File.cc b/src/File.cc
index 16d4259fe5..7c4a21d5e8 100644
--- a/src/File.cc
+++ b/src/File.cc
@@ -18,6 +18,10 @@
 #include <errno.h>
 #include <unistd.h>
 
+#include <openssl/evp.h>
+#include <openssl/pem.h>
+#include <openssl/err.h>
+
 #include <algorithm>
 
 #include "File.h"
diff --git a/src/File.h b/src/File.h
index f3fdf2f271..6410a67624 100644
--- a/src/File.h
+++ b/src/File.h
@@ -11,11 +11,11 @@
 # ifdef NEED_KRB5_H
 #  include <krb5.h>
 # endif // NEED_KRB5_H
-extern "C" {
-# include "openssl/evp.h"
-# include "openssl/pem.h"
-# include "openssl/err.h"
-}
+
+// From OpenSSL. We forward-declare these here to avoid introducing a
+// dependency on OpenSSL headers just for this header file.
+typedef struct evp_pkey_st EVP_PKEY;
+typedef struct evp_cipher_ctx_st EVP_CIPHER_CTX;
 
 class BroType;
 class RotateTimer;
diff --git a/src/file_analysis/analyzer/x509/X509.cc b/src/file_analysis/analyzer/x509/X509.cc
index ebf7b1d04f..da3c6635a8 100644
--- a/src/file_analysis/analyzer/x509/X509.cc
+++ b/src/file_analysis/analyzer/x509/X509.cc
@@ -14,6 +14,7 @@
 #include <openssl/x509v3.h>
 #include <openssl/asn1.h>
 #include <openssl/opensslconf.h>
+#include <openssl/err.h>
 
 using namespace file_analysis;
 
diff --git a/src/file_analysis/analyzer/x509/functions.bif b/src/file_analysis/analyzer/x509/functions.bif
index 216f4c69cc..c977c746d4 100644
--- a/src/file_analysis/analyzer/x509/functions.bif
+++ b/src/file_analysis/analyzer/x509/functions.bif
@@ -6,6 +6,8 @@
 #include <openssl/asn1.h>
 #include <openssl/x509_vfy.h>
 #include <openssl/ocsp.h>
+#include <openssl/pem.h>
+#include <openssl/err.h>
 
 // This is the indexed map of X509 certificate stores.
 static map<Val*, X509_STORE*> x509_stores;
diff --git a/src/main.cc b/src/main.cc
index fd462f4996..c2052a03b3 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -18,7 +18,8 @@ extern "C" {
 }
 #endif
 
-#include <openssl/md5.h>
+#include <openssl/ssl.h>
+#include <openssl/err.h>
 
 extern "C" void OPENSSL_add_all_algorithms_conf(void);
 

From 176d9f23be753c21936b28889d507719f3059c78 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 1 Aug 2016 08:42:36 -0700
Subject: [PATCH 20/25] Fixing duplicate SSH authentication failure events.

We now do not raise more than one failure event per connection.

Addresses BIT-1641.
---
 src/analyzer/protocol/ssh/SSH.cc                   | 14 +++++++++-----
 src/analyzer/protocol/ssh/SSH.h                    |  6 ++++--
 .../output                                         | 11 +++++++++++
 .../base/protocols/ssh/one-auth-fail-only.test     |  7 +++++++
 4 files changed, 31 insertions(+), 7 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.ssh.one-auth-fail-only/output
 create mode 100644 testing/btest/scripts/base/protocols/ssh/one-auth-fail-only.test

diff --git a/src/analyzer/protocol/ssh/SSH.cc b/src/analyzer/protocol/ssh/SSH.cc
index f1f8857e03..55f424344b 100644
--- a/src/analyzer/protocol/ssh/SSH.cc
+++ b/src/analyzer/protocol/ssh/SSH.cc
@@ -16,7 +16,7 @@ SSH_Analyzer::SSH_Analyzer(Connection* c)
 	{
 	interp = new binpac::SSH::SSH_Conn(this);
 	had_gap = false;
-	auth_decision_made = false;
+	auth_decision = AUTH_UNKNOWN;
 	skipped_banner = false;
 	service_accept_size = 0;
 	userauth_failure_size = 0;
@@ -60,7 +60,7 @@ void SSH_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 			BifEvent::generate_ssh_encrypted_packet(interp->bro_analyzer(), interp->bro_analyzer()->Conn(),
 				orig, len);
 
-		if ( ! auth_decision_made )
+		if ( auth_decision != AUTH_SUCCESS )
 			ProcessEncrypted(len, orig);
 
 		return;
@@ -105,9 +105,10 @@ void SSH_Analyzer::ProcessEncrypted(int len, bool orig)
 		// -16.
 		if ( ! userauth_failure_size && (len + 16 == service_accept_size) )
 			{
-			auth_decision_made = true;
 			if ( ssh_auth_successful )
 				BifEvent::generate_ssh_auth_successful(interp->bro_analyzer(), interp->bro_analyzer()->Conn(), true);
+
+			auth_decision = AUTH_SUCCESS;
 			return;
 			}
 
@@ -131,17 +132,20 @@ void SSH_Analyzer::ProcessEncrypted(int len, bool orig)
 		// another packet of the same size.
 		if ( len == userauth_failure_size )
 			{
-			if ( ssh_auth_failed )
+			if ( ssh_auth_failed && auth_decision != AUTH_FAILURE )
 				BifEvent::generate_ssh_auth_failed(interp->bro_analyzer(), interp->bro_analyzer()->Conn());
+
+			auth_decision = AUTH_FAILURE;
 			return;
 			}
 
 		// ...or a success packet.
 		if ( len - service_accept_size == -16 )
 			{
-			auth_decision_made = true;
 			if ( ssh_auth_successful )
 				BifEvent::generate_ssh_auth_successful(interp->bro_analyzer(), interp->bro_analyzer()->Conn(), false);
+
+			auth_decision = AUTH_SUCCESS;
 			return;
 			}
 		}
diff --git a/src/analyzer/protocol/ssh/SSH.h b/src/analyzer/protocol/ssh/SSH.h
index dc3a7c5e39..89668f93d1 100644
--- a/src/analyzer/protocol/ssh/SSH.h
+++ b/src/analyzer/protocol/ssh/SSH.h
@@ -35,12 +35,14 @@ namespace analyzer {
 			bool had_gap;
 
 			// Packet analysis stuff
-			bool auth_decision_made;
 			bool skipped_banner;
-
 			int service_accept_size;
 			int userauth_failure_size;
 
+			enum AuthDecision {
+				AUTH_UNKNOWN, AUTH_FAILURE, AUTH_SUCCESS
+			} auth_decision;
+
 			};
 
 		}
diff --git a/testing/btest/Baseline/scripts.base.protocols.ssh.one-auth-fail-only/output b/testing/btest/Baseline/scripts.base.protocols.ssh.one-auth-fail-only/output
new file mode 100644
index 0000000000..f81dd5e219
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.ssh.one-auth-fail-only/output
@@ -0,0 +1,11 @@
+      1 C0LAHyvtKSQHyJxIl
+      1 C37jN32gN3y3AZzyf6
+      1 C3eiCBGOLw3VtHfOj
+      1 C4J4Th3PJpwUYZZ6gc
+      1 CHhAvVGS1DHFjwGM9
+      1 CP5puj4I8PtEU4qzYg
+      1 CUM0KZ3MLUfNB0cl11
+      1 ClEkJM2Vm5giqnMf4h
+      1 CmES5u32sYpV7JYN
+      1 CtPZjS20MLrsMUOJi2
+      1 CwjjYJ2WqgTbAqiHl6
diff --git a/testing/btest/scripts/base/protocols/ssh/one-auth-fail-only.test b/testing/btest/scripts/base/protocols/ssh/one-auth-fail-only.test
new file mode 100644
index 0000000000..0e88556c94
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/ssh/one-auth-fail-only.test
@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -C -r $TRACES/ssh/sshguess.pcap %INPUT | sort | uniq -c >output
+# @TEST-EXEC: btest-diff output
+
+event ssh_auth_failed(c: connection)
+    {
+    print c$uid;
+    }

From b1e47eb71f043e6841bb0396537827b7a056e534 Mon Sep 17 00:00:00 2001
From: Moshe Kaplan <mk.moshe.kaplan@gmail.com>
Date: Tue, 2 Aug 2016 08:45:23 -0400
Subject: [PATCH 21/25] Added String slicing (subscript) examples

---
 doc/script-reference/types.rst | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst
index 847e0f8fab..d9d306b70b 100644
--- a/doc/script-reference/types.rst
+++ b/doc/script-reference/types.rst
@@ -181,11 +181,14 @@ Here is a more detailed description of each type:
     second-to-last character, etc.  Here are a few examples::
 
         local orig = "0123456789";
-        local second_char = orig[1];
-        local last_char = orig[-1];
-        local first_two_chars = orig[:2];
-        local last_two_chars = orig[8:];
-        local no_first_and_last = orig[1:9];
+        local second_char = orig[1];         # "1"
+        local last_char = orig[-1];          # "9"
+        local first_two_chars = orig[:2];    # "01"
+        local last_two_chars = orig[8:];     # "89"
+        local no_first_and_last = orig[1:9]; # "12345678"
+        local no_first = orig[1:];           # "123456789"
+        local no_last = orig[:-1];           # "012345678"
+        local copy_orig = orig[:];           # "0123456789"
 
     Note that the subscript operator cannot be used to modify a string (i.e.,
     it cannot be on the left side of an assignment operator).

From a273143e7d07cbc0451b7ad86d0c06aa9d36cc9a Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 2 Aug 2016 11:08:59 -0700
Subject: [PATCH 22/25] Updating submodule(s).

 [nomail]
---
 aux/binpac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/binpac b/aux/binpac
index 6cffeec0d5..8dbf2470ed 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 6cffeec0d5d2e61be93d4f52ddb7f9b60842ad86
+Subproject commit 8dbf2470eda8f358ea225234e05b406da8e258f1

From 3adad5e19a82ce0af95618a37f6dd6ba6882dc40 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 2 Aug 2016 11:38:07 -0700
Subject: [PATCH 23/25] Fix some Coverity warnings.

---
 CHANGES                |  4 ++++
 VERSION                |  2 +-
 aux/binpac             |  2 +-
 src/Tag.cc             | 14 ++++++++++++++
 src/Tag.h              |  5 +++++
 src/Val.cc             |  3 ++-
 src/logging/Manager.cc |  5 -----
 7 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/CHANGES b/CHANGES
index 2c817c6c2e..9252c9116b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.4-737 | 2016-08-02 11:38:07 -0700
+
+  * Fix some Coverity warnings. (Robin Sommer)
+
 2.4-735 | 2016-08-02 11:05:36 -0700
 
   * Added string slicing examples to documentation. (Moshe Kaplan)
diff --git a/VERSION b/VERSION
index 5b3925f440..45a87b058c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4-735
+2.4-737
diff --git a/aux/binpac b/aux/binpac
index 8dbf2470ed..3664242a21 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 8dbf2470eda8f358ea225234e05b406da8e258f1
+Subproject commit 3664242a218c21100d62917866d6b8cb0d6f0fa1
diff --git a/src/Tag.cc b/src/Tag.cc
index d125e4917b..04eb0c79b0 100644
--- a/src/Tag.cc
+++ b/src/Tag.cc
@@ -65,6 +65,20 @@ Tag& Tag::operator=(const Tag& other)
 	return *this;
 	}
 
+Tag& Tag::operator=(const Tag&& other)
+	{
+	if ( this != &other )
+		{
+		type = other.type;
+		subtype = other.subtype;
+		Unref(val);
+		val = other.val;
+		other.val = nullptr;
+		}
+
+	return *this;
+	}
+
 EnumVal* Tag::AsEnumVal(EnumType* etype) const
 	{
 	if ( ! val )
diff --git a/src/Tag.h b/src/Tag.h
index a3d7197fa0..224fdd40f3 100644
--- a/src/Tag.h
+++ b/src/Tag.h
@@ -77,6 +77,11 @@ protected:
 	 */
 	Tag& operator=(const Tag& other);
 
+	/**
+	 * Move assignment operator.
+	 */
+	Tag& operator=(const Tag&& other);
+
 	/**
 	 * Compares two tags for equality.
 	 */
diff --git a/src/Val.cc b/src/Val.cc
index ca8551f7a7..ca70e1f5df 100644
--- a/src/Val.cc
+++ b/src/Val.cc
@@ -2278,7 +2278,7 @@ double TableVal::GetExpireTime()
 	Unref(timeout);
 
 	if ( interval >= 0 )
-		return timeout->AsInterval();
+		return interval;
 
 	expire_time = 0;
 
@@ -2327,6 +2327,7 @@ double TableVal::CallExpireFunc(Val* idx)
 		if ( vf->Type()->Tag() != TYPE_FUNC )
 			{
 			Unref(vf);
+			delete_vals(vl);
 			vf->Error("not a function");
 			return 0;
 			}
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 9db43518ed..9a9fc3a3f8 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -640,8 +640,6 @@ bool Manager::RemoveFilter(EnumVal* id, string name)
 
 bool Manager::Write(EnumVal* id, RecordVal* columns)
 	{
-	bool error = false;
-
 	Stream* stream = FindStream(id);
 	if ( ! stream )
 		return false;
@@ -850,9 +848,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 	Unref(columns);
 
-	if ( error )
-		RemoveDisabledWriters(stream);
-
 	return true;
 	}
 

From 30b40b214a975a0125361e0c33487ac7e0c169c8 Mon Sep 17 00:00:00 2001
From: Johanna Amann <johanna@icir.org>
Date: Tue, 2 Aug 2016 15:31:39 -0700
Subject: [PATCH 24/25] Updating CHANGES and VERSION.

 [nomail]
---
 CHANGES | 7 +++++++
 VERSION | 2 +-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 9252c9116b..d8da869f8e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,11 @@
 
+2.4-742 | 2016-08-02 15:28:31 -0700
+
+  * Fix duplicate SSH authentication failure events. Addresses BIT-1641.
+    (Robin Sommer)
+
+  * Remove OpenSSL dependency for plugins. (Robin Sommer)
+
 2.4-737 | 2016-08-02 11:38:07 -0700
 
   * Fix some Coverity warnings. (Robin Sommer)
diff --git a/VERSION b/VERSION
index 45a87b058c..43d6e63262 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.4-737
+2.4-742

From 4bb4c54f96e2d93cd9bb383505a045893bdb0b5b Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 5 Aug 2016 09:29:23 -0400
Subject: [PATCH 25/25] Tiny scoping updates and test baseline updates for
 Intel framework.

---
 scripts/base/frameworks/intel/main.bro        |  5 ++-
 scripts/policy/frameworks/intel/do_expire.bro |  2 +-
 .../Baseline/language.expire_subnet/output    |  8 ++--
 .../output                                    | 14 +++---
 .../output                                    | 22 +++++-----
 .../intel-all.log                             | 22 +++++-----
 .../intel.log                                 | 22 +++++-----
 .../intel.log                                 | 44 +++++++++----------
 8 files changed, 70 insertions(+), 69 deletions(-)

diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro
index bc7c9187fe..401b48e2d5 100644
--- a/scripts/base/frameworks/intel/main.bro
+++ b/scripts/base/frameworks/intel/main.bro
@@ -154,8 +154,9 @@ export {
 	global extend_match: hook(info: Info, s: Seen, items: set[Item]);
 
 	## The expiration timeout for intelligence items. Once an item expires, the
-	## :bro:id:`item_expired` hook is called. Reinsertion of an item resets the
-	## timeout. A negative value disables expiration of intelligence items.
+	## :bro:id:`Intel::item_expired` hook is called. Reinsertion of an item 
+	## resets the timeout. A negative value disables expiration of intelligence 
+	## items.
 	const item_expiration = -1 min &redef;
 
 	## This hook can be used to handle expiration of intelligence items.
diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro
index aabe3630e4..fedb47b57d 100644
--- a/scripts/policy/frameworks/intel/do_expire.bro
+++ b/scripts/policy/frameworks/intel/do_expire.bro
@@ -4,7 +4,7 @@
 
 module Intel;
 
-redef item_expiration = 10min;
+redef Intel::item_expiration = 10min;
 
 hook item_expired(indicator: string, indicator_type: Type,
 	metas: set[MetaData]) &priority=-10
diff --git a/testing/btest/Baseline/language.expire_subnet/output b/testing/btest/Baseline/language.expire_subnet/output
index 61a6ac6a01..dee030eb0c 100644
--- a/testing/btest/Baseline/language.expire_subnet/output
+++ b/testing/btest/Baseline/language.expire_subnet/output
@@ -15,13 +15,13 @@ Accessed table nums: two; three
 Accessed table nets: two; zero, three
 Time: 7.0 secs 518.0 msecs 828.0 usecs
 
+Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.0 usecs
+Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.0 usecs
 Expired Num: 4 --> four at 8.0 secs 835.0 msecs 30.0 usecs
 Expired Num: 1 --> one at 8.0 secs 835.0 msecs 30.0 usecs
 Expired Num: 0 --> zero at 8.0 secs 835.0 msecs 30.0 usecs
-Expired Subnet: 192.168.4.0/24 --> four at 8.0 secs 835.0 msecs 30.0 usecs
-Expired Subnet: 192.168.1.0/24 --> one at 8.0 secs 835.0 msecs 30.0 usecs
-Expired Num: 2 --> two at 15.0 secs 150.0 msecs 681.0 usecs
-Expired Num: 3 --> three at 15.0 secs 150.0 msecs 681.0 usecs
 Expired Subnet: 192.168.0.0/16 --> zero at 15.0 secs 150.0 msecs 681.0 usecs
 Expired Subnet: 192.168.3.0/24 --> three at 15.0 secs 150.0 msecs 681.0 usecs
 Expired Subnet: 192.168.2.0/24 --> two at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Num: 2 --> two at 15.0 secs 150.0 msecs 681.0 usecs
+Expired Num: 3 --> three at 15.0 secs 150.0 msecs 681.0 usecs
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
index aa401ab007..d8c2755fe4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.match-subnet/output
@@ -3,13 +3,13 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-22-19-12-08
+#open	2016-08-05-13-13-14
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1466622728.846581	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-1466622728.846581	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
-1466622728.846581	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR,Intel::SUBNET	source1	-	-	-
-#close	2016-06-22-19-12-08
+1470402794.307931	-	-	-	-	-	192.168.1.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1470402794.307931	-	-	-	-	-	192.168.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET	source1	-	-	-
+1470402794.307931	-	-	-	-	-	192.168.142.1	Intel::ADDR	SOMEWHERE	bro	Intel::SUBNET,Intel::ADDR	source1	-	-	-
+#close	2016-08-05-13-13-14
 
 Seen: [indicator=192.168.1.1, indicator_type=Intel::ADDR, host=192.168.1.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
 Item: [indicator=192.168.1.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/1]]
@@ -18,7 +18,7 @@ Seen: [indicator=192.168.2.1, indicator_type=Intel::ADDR, host=192.168.2.1, wher
 Item: [indicator=192.168.2.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is just plain baaad, url=http://some-data-distributor.com/2]]
 
 Seen: [indicator=192.168.142.1, indicator_type=Intel::ADDR, host=192.168.142.1, where=SOMEWHERE, node=bro, conn=<uninitialized>, uid=<uninitialized>, f=<uninitialized>, fuid=<uninitialized>]
-Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
-Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
 Item: [indicator=192.168.142.1, indicator_type=Intel::ADDR, meta=[source=source1, desc=this host is just plain baaad, url=http://some-data-distributor.com/3]]
 Item: [indicator=192.168.128.0/18, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork might be baaad, url=http://some-data-distributor.com/5]]
+Item: [indicator=192.168.142.0/26, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is inside, url=http://some-data-distributor.com/4]]
+Item: [indicator=192.168.142.0/24, indicator_type=Intel::SUBNET, meta=[source=source1, desc=this subnetwork is baaad, url=http://some-data-distributor.com/4]]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
index 8c8e9d9c0f..5249bb3110 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.intel.updated-match/output
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-09-12
+#open	2016-08-05-13-14-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1466017751.936022	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
-1466017754.938975	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
-1466017754.938975	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
-1466017757.941783	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2,source1	-	-	-
-1466017757.941783	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
-#close	2016-06-15-19-09-18
+1470402852.531769	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1	-	-	-
+1470402855.546089	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1,source2	-	-	-
+1470402855.546089	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+1470402858.547977	-	-	-	-	-	1.2.3.4	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source1,source2	-	-	-
+1470402858.547977	-	-	-	-	-	4.3.2.1	Intel::ADDR	SOMEWHERE	bro	Intel::ADDR	source2	-	-	-
+#close	2016-08-05-13-14-18
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2016-06-15-19-09-18
+#open	2016-08-05-13-14-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	set[enum]	interval	bool	string	string	string	double	double
-1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-1466017757.941783	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
-#close	2016-06-15-19-09-18
+1470402858.547977	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 1.2.3.4 at SOMEWHERE	1.2.3.4	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+1470402858.547977	-	-	-	-	-	-	-	-	-	Intel::Notice	Intel hit on 4.3.2.1 at SOMEWHERE	4.3.2.1	-	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2016-08-05-13-14-18
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
index 69feed2307..6bb3e47e60 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.certs/intel-all.log
@@ -3,23 +3,23 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-08-03
+#open	2016-08-05-13-22-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1416942644.593119	CXWv6p3arKYeMETxOg	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
-#close	2016-06-15-19-08-03
+1416942644.593119	CHhAvVGS1DHFjwGM9	192.168.4.149	49422	23.92.19.75	443	www.pantz.org	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	Fi6J8q3lDJpbQWAnvi	application/pkix-cert	23.92.19.75:443/tcp
+#close	2016-08-05-13-22-37
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-08-03
+#open	2016-08-05-13-22-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1170717505.735416	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
-1170717505.934612	CXWv6p3arKYeMETxOg	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
-1170717508.883051	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
-1170717509.082241	CjhGID4nQcgTWjvg4c	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
-1170717511.909717	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
-1170717512.108799	CCvvfg3TEfuqmmG4bh	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
-#close	2016-06-15-19-08-03
+1170717505.735416	CHhAvVGS1DHFjwGM9	192.150.187.164	58868	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FeCwNK3rzqPnZ7eBQ5	application/pkix-cert	194.127.84.106:443/tcp
+1170717505.934612	CHhAvVGS1DHFjwGM9	192.150.187.164	58868	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FeCwNK3rzqPnZ7eBQ5	-	-
+1170717508.883051	ClEkJM2Vm5giqnMf4h	192.150.187.164	58869	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FjkLnG4s34DVZlaBNc	application/pkix-cert	194.127.84.106:443/tcp
+1170717509.082241	ClEkJM2Vm5giqnMf4h	192.150.187.164	58869	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FjkLnG4s34DVZlaBNc	-	-
+1170717511.909717	C4J4Th3PJpwUYZZ6gc	192.150.187.164	58870	194.127.84.106	443	2c322ae2b7fe91391345e070b63668978bb1c9da	Intel::CERT_HASH	X509::IN_CERT	bro	Intel::CERT_HASH	source1	FQXAWgI2FB5STbrff	application/pkix-cert	194.127.84.106:443/tcp
+1170717512.108799	C4J4Th3PJpwUYZZ6gc	192.150.187.164	58870	194.127.84.106	443	www.dresdner-privat.de	Intel::DOMAIN	X509::IN_CERT	bro	Intel::DOMAIN	source1	FQXAWgI2FB5STbrff	-	-
+#close	2016-08-05-13-22-38
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log
index 708b02dd24..c14b4b10c1 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.seen.smtp/intel.log
@@ -3,14 +3,14 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-07-13-16-17-20
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	seen.indicator	seen.indicator_type	seen.where	seen.node	sources
-#types	time	string	addr	port	addr	port	string	string	string	string	enum	enum	string	set[string]
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_RCPT_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashoefer@cern.ch	Intel::EMAIL	SMTP::IN_FROM	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashoefer@gmail.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	addr-spec@example.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	name-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	-	-	-	angle-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	source1
-#close	2016-07-13-16-17-20
+#open	2016-08-05-13-22-00
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
+#types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_RCPT_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashoefer@cern.ch	Intel::EMAIL	SMTP::IN_FROM	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashoefer@gmail.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	jan.grashofer@cern.ch	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	addr-spec@example.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	name-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+1449610263.071201	CHhAvVGS1DHFjwGM9	188.184.129.157	35119	188.184.36.24	25	angle-addr@example.com	Intel::EMAIL	SMTP::IN_TO	bro	Intel::EMAIL	source1	-	-	-
+#close	2016-08-05-13-22-00
diff --git a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
index f452f65a9e..66ba6af8db 100644
--- a/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
+++ b/testing/btest/Baseline/scripts.policy.frameworks.intel.whitelisting/intel.log
@@ -3,27 +3,27 @@
 #empty_field	(empty)
 #unset_field	-
 #path	intel
-#open	2016-06-15-19-06-02
+#open	2016-08-05-13-24-29
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	seen.indicator	seen.indicator_type	seen.where	seen.node	matched	sources	fuid	file_mime_type	file_desc
 #types	time	string	addr	port	addr	port	string	enum	enum	string	set[enum]	set[string]	string	string	string
-1300475168.853899	CPbrpk1qSsw6ESzHV4	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.854837	CIPOse170MGiRM1Qf4	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.857956	CMXxB5GvmoxJFXdTa	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.858713	Che1bq3i2rO3KD1Syg	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.891644	CEle3f3zno26fFZkrh	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.892414	CfTOmO0HKorjr8Zp7	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.893988	Cab0vO1xNYSS2hJkle	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.894787	Cx3C534wEyF3OvvcQe	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.916018	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.916183	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.918358	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.952296	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.952307	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.954820	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.975934	CJ3xTn1c4Zw9TmAE05	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.976436	C7XEbhP654jzLoe3a	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475168.979264	C3SfNE4BWaU4aSuwkc	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475169.014593	CzA03V1VcgagLjnO92	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475169.014619	CyAhVIzHqb7t7kv28	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-1300475169.014927	CkDsfG2YIeWJmXWNWj	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
-#close	2016-06-15-19-06-02
+1300475168.853899	CmES5u32sYpV7JYN	141.142.220.118	43927	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.854837	C37jN32gN3y3AZzyf6	141.142.220.118	40526	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.857956	C0LAHyvtKSQHyJxIl	141.142.220.118	32902	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.858713	C9rXSW3KSpTYvPrlI1	141.142.220.118	59714	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.891644	C9mvWx3ezztgzcexV7	141.142.220.118	58206	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.892414	C7fIlMZDuRiqjpYbb	141.142.220.118	59746	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.893988	CpmdRlaUoJLN3uIRa	141.142.220.118	45000	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.894787	CqlVyW1YwZ15RhTBc4	141.142.220.118	48128	141.142.2.2	53	upload.wikimedia.org	Intel::DOMAIN	DNS::IN_REQUEST	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916018	CwjjYJ2WqgTbAqiHl6	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.916183	C3eiCBGOLw3VtHfOj	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.918358	Ck51lg1bScffFj34Ri	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952296	CykQaM33ztNt0csB9a	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.952307	CtxTCR2Yer0FR1tIBg	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.954820	CLNN1k2QMum1aexUK7	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.975934	CwjjYJ2WqgTbAqiHl6	141.142.220.118	49997	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.976436	C3eiCBGOLw3VtHfOj	141.142.220.118	49996	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475168.979264	Ck51lg1bScffFj34Ri	141.142.220.118	49998	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014593	CykQaM33ztNt0csB9a	141.142.220.118	49999	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014619	CtxTCR2Yer0FR1tIBg	141.142.220.118	50000	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+1300475169.014927	CLNN1k2QMum1aexUK7	141.142.220.118	50001	208.80.152.3	80	upload.wikimedia.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	bro	Intel::DOMAIN	source1	-	-	-
+#close	2016-08-05-13-24-29