From bec965b66f7df28f023ff34600ed26a4f1735c20 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 21 May 2013 15:52:59 -0400
Subject: [PATCH 01/25] Large update for the SumStats framework.

 - On-demand access to sumstats results through "return from"
   functions named SumStats::request and Sumstats::request_key.
   Both functions are tested in standalone and clustered modes.

 - $name field has returned to SumStats which simplifies cluster
   code and makes the on-demand access stuff possible.

 - Clustered results can only be collected for 1 minute from their
   time of creation now instead of time of last read.

 - Thresholds use doubles instead of counts everywhere now.

 - Calculation dependency resolution occurs at start up time now
   instead of doing it at observation time which provide a minor
   cpu performance improvement.  A new plugin registration mechanism
   was created to support this change.

 - AppStats now has a minimal doc string and is broken into hook-based
   plugins.

 - AppStats and traceroute detection added to local.bro
---
 scripts/base/frameworks/sumstats/cluster.bro  | 200 +++++++++------
 scripts/base/frameworks/sumstats/main.bro     | 241 +++++++++++++-----
 .../base/frameworks/sumstats/non-cluster.bro  |  29 ++-
 .../frameworks/sumstats/plugins/average.bro   |   9 +-
 .../base/frameworks/sumstats/plugins/max.bro  |   8 +-
 .../base/frameworks/sumstats/plugins/min.bro  |   9 +-
 .../frameworks/sumstats/plugins/sample.bro    |  13 +-
 .../frameworks/sumstats/plugins/std-dev.bro   |  15 +-
 .../base/frameworks/sumstats/plugins/sum.bro  |  30 ++-
 .../frameworks/sumstats/plugins/unique.bro    |   8 +-
 .../frameworks/sumstats/plugins/variance.bro  |  10 +-
 scripts/policy/misc/app-stats/__load__.bro    |   2 +
 .../{app-metrics.bro => app-stats/main.bro}   |  48 +---
 .../misc/app-stats/plugins/__load__.bro       |   6 +
 .../misc/app-stats/plugins/facebook.bro       |  12 +
 .../policy/misc/app-stats/plugins/gmail.bro   |  12 +
 .../policy/misc/app-stats/plugins/google.bro  |  12 +
 .../policy/misc/app-stats/plugins/netflix.bro |  12 +
 .../policy/misc/app-stats/plugins/pandora.bro |  12 +
 .../policy/misc/app-stats/plugins/youtube.bro |  12 +
 .../policy/misc/detect-traceroute/main.bro    |   9 +-
 scripts/policy/misc/scan.bro                  |  18 +-
 .../protocols/ftp/detect-bruteforcing.bro     |   7 +-
 scripts/policy/protocols/http/detect-sqli.bro |  12 +-
 .../protocols/ssh/detect-bruteforcing.bro     |   7 +-
 scripts/site/local.bro                        |   7 +
 .../manager-1..stdout                         |   7 +
 .../.stdout                                   |   5 +
 .../frameworks/sumstats/basic-cluster.bro     |   3 +-
 .../base/frameworks/sumstats/basic.bro        |  21 +-
 .../sumstats/cluster-intermediate-update.bro  |   7 +-
 .../frameworks/sumstats/on-demand-cluster.bro |  93 +++++++
 .../base/frameworks/sumstats/on-demand.bro    |  45 ++++
 .../base/frameworks/sumstats/thresholding.bro |  23 +-
 34 files changed, 687 insertions(+), 277 deletions(-)
 create mode 100644 scripts/policy/misc/app-stats/__load__.bro
 rename scripts/policy/misc/{app-metrics.bro => app-stats/main.bro} (58%)
 create mode 100644 scripts/policy/misc/app-stats/plugins/__load__.bro
 create mode 100644 scripts/policy/misc/app-stats/plugins/facebook.bro
 create mode 100644 scripts/policy/misc/app-stats/plugins/gmail.bro
 create mode 100644 scripts/policy/misc/app-stats/plugins/google.bro
 create mode 100644 scripts/policy/misc/app-stats/plugins/netflix.bro
 create mode 100644 scripts/policy/misc/app-stats/plugins/pandora.bro
 create mode 100644 scripts/policy/misc/app-stats/plugins/youtube.bro
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout
 create mode 100644 testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro
 create mode 100644 testing/btest/scripts/base/frameworks/sumstats/on-demand.bro

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index 9ee63a674e..23e20ae9c0 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -27,39 +27,34 @@ export {
 	## performed.  In practice this should hopefully have a minimal effect.
 	const max_outstanding_global_views = 10 &redef;
 
-	## Intermediate updates can cause overload situations on very large clusters. This
-	## option may help reduce load and correct intermittent problems. The goal for this
-	## option is also meant to be temporary.
-	const enable_intermediate_updates = T &redef;
-
 	## Event sent by the manager in a cluster to initiate the collection of values for
 	## a sumstat.
-	global cluster_ss_request: event(uid: string, ssid: string);
+	global cluster_ss_request: event(uid: string, ss_name: string, cleanup: bool);
 
 	## Event sent by nodes that are collecting sumstats after receiving a request for
 	## the sumstat from the manager.
-	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
+	global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool);
 
 	## This event is sent by the manager in a cluster to initiate the collection of
 	## a single key value from a sumstat.  It's typically used to get intermediate
 	## updates before the break interval triggers to speed detection of a value
 	## crossing a threshold.
-	global cluster_key_request: event(uid: string, ssid: string, key: Key);
+	global cluster_key_request: event(uid: string, ss_name: string, key: Key, cleanup: bool);
 
 	## This event is sent by nodes in response to a
 	## :bro:id:`SumStats::cluster_key_request` event.
-	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
+	global cluster_key_response: event(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool);
 
 	## This is sent by workers to indicate that they crossed the percent
 	## of the current threshold by the percentage defined globally in
 	## :bro:id:`SumStats::cluster_request_global_view_percent`
-	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
+	global cluster_key_intermediate_response: event(ss_name: string, key: SumStats::Key);
 
 	## This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, ssid: string, data: ResultTable);
+	global send_data: event(uid: string, ss_name: string, data: ResultTable, cleanup: bool);
 
 	## This event is generated when a threshold is crossed.
-	global cluster_threshold_crossed: event(ssid: string, key: SumStats::Key, thold: Thresholding);
+	global cluster_threshold_crossed: event(ss_name: string, key: SumStats::Key, thold: Thresholding);
 }
 
 # Add events to the cluster framework to make this work.
@@ -74,44 +69,38 @@ redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_resp
 # an intermediate result has been received.
 global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
 
-event bro_init() &priority=-100
-	{
-	# The manager is the only host allowed to track these.
-	stats_store = table();
-	reducer_store = table();
-	}
-
 # This is done on all non-manager node types in the event that a sumstat is
 # being collected somewhere other than a worker.
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	# If an intermediate update for this value was sent recently, don't send
 	# it again.
-	if ( [ss$id, key] in recent_global_view_keys )
+	if ( [ss$name, key] in recent_global_view_keys )
 		return;
 
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an
 	# intermediate update.
-	if ( enable_intermediate_updates &&
-	     check_thresholds(ss, key, result, cluster_request_global_view_percent) )
+	if ( check_thresholds(ss, key, result, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event SumStats::cluster_key_intermediate_response(ss$id, key);
-		++recent_global_view_keys[ss$id, key];
+		event SumStats::cluster_key_intermediate_response(ss$name, key);
+		++recent_global_view_keys[ss$name, key];
 		}
 	}
 
-event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
+event SumStats::send_data(uid: string, ss_name: string, data: ResultTable, cleanup: bool)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 
 	local local_data: ResultTable = table();
+	local incoming_data: ResultTable = cleanup ? data : copy(data);
+
 	local num_added = 0;
-	for ( key in data )
+	for ( key in incoming_data )
 		{
-		local_data[key] = data[key];
-		delete data[key];
+		local_data[key] = incoming_data[key];
+		delete incoming_data[key];
 
 		# Only send cluster_send_in_groups_of at a time.  Queue another
 		# event to send the next group.
@@ -121,56 +110,56 @@ event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
 
 	local done = F;
 	# If data is empty, this sumstat is done.
-	if ( |data| == 0 )
+	if ( |incoming_data| == 0 )
 		done = T;
 
-	event SumStats::cluster_ss_response(uid, ssid, local_data, done);
+	event SumStats::cluster_ss_response(uid, ss_name, local_data, done);
 	if ( ! done )
-		schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
+		schedule 0.01 sec { SumStats::send_data(uid, ss_name, incoming_data, T) };
 	}
 
-event SumStats::cluster_ss_request(uid: string, ssid: string)
+event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
 	{
 	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
 
 	# Initiate sending all of the data for the requested stats.
-	if ( ssid in result_store )
-		event SumStats::send_data(uid, ssid, result_store[ssid]);
+	if ( ss_name in result_store )
+		event SumStats::send_data(uid, ss_name, result_store[ss_name], cleanup);
 	else
-		event SumStats::send_data(uid, ssid, table());
+		event SumStats::send_data(uid, ss_name, table(), cleanup);
 
 	# Lookup the actual sumstats and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
-	if ( ssid in stats_store )
-		reset(stats_store[ssid]);
+	if ( ss_name in stats_store && cleanup )
+		reset(stats_store[ss_name]);
 	}
 
-event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
+event SumStats::cluster_key_request(uid: string, ss_name: string, key: Key, cleanup: bool)
 	{
-	if ( ssid in result_store && key in result_store[ssid] )
+	if ( ss_name in result_store && key in result_store[ss_name] )
 		{
 		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
-		event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]);
+		event SumStats::cluster_key_response(uid, ss_name, key, result_store[ss_name][key], cleanup);
 		}
 	else
 		{
 		# We need to send an empty response if we don't have the data so that the manager
 		# can know that it heard back from all of the workers.
-		event SumStats::cluster_key_response(uid, ssid, key, table());
+		event SumStats::cluster_key_response(uid, ss_name, key, table(), cleanup);
 		}
 	}
 
-event SumStats::cluster_threshold_crossed(ssid: string, key: SumStats::Key, thold: Thresholding)
+event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, thold: Thresholding)
 	{
-	if ( ssid !in threshold_tracker )
-		threshold_tracker[ssid] = table();
+	if ( ss_name !in threshold_tracker )
+		threshold_tracker[ss_name] = table();
 
-	threshold_tracker[ssid][key] = thold;
+	threshold_tracker[ss_name][key] = thold;
 	}
 
-event SumStats::thresholds_reset(ssid: string)
+event SumStats::thresholds_reset(ss_name: string)
 	{
-	threshold_tracker[ssid] = table();
+	threshold_tracker[ss_name] = table();
 	}
 
 @endif
@@ -181,7 +170,7 @@ event SumStats::thresholds_reset(ssid: string)
 # This variable is maintained by manager nodes as they collect and aggregate
 # results.
 # Index on a uid.
-global stats_results: table[string] of ResultTable &read_expire=1min;
+global stats_results: table[string] of ResultTable &create_expire=1min &default=table();
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid
@@ -189,18 +178,18 @@ global stats_results: table[string] of ResultTable &read_expire=1min;
 # result is written out and deleted from here.
 # Indexed on a uid.
 # TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &read_expire=1min &default=0;
+global done_with: table[string] of count &create_expire=1min &default=0;
 
 # This variable is maintained by managers to track intermediate responses as
 # they are getting a global view for a certain key.
 # Indexed on a uid.
-global key_requests: table[string] of Result &read_expire=1min;
+global key_requests: table[string] of Result &create_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
 # to too many intermediate updates.  Each sumstat is tracked separately so that
 # one won't overwhelm and degrade other quieter sumstats.
 # Indexed on a sumstat id.
-global outstanding_global_views: table[string] of count &default=0;
+global outstanding_global_views: table[string] of count &create_expire=1min &default=0;
 
 const zero_time = double_to_time(0.0);
 # Managers handle logging.
@@ -216,7 +205,7 @@ event SumStats::finish_epoch(ss: SumStat)
 		stats_results[uid] = table();
 
 		# Request data from peers.
-		event SumStats::cluster_ss_request(uid, ss$id);
+		event SumStats::cluster_ss_request(uid, ss$name, T);
 		}
 
 	# Schedule the next finish_epoch event.
@@ -230,20 +219,20 @@ function data_added(ss: SumStat, key: Key, result: Result)
 	if ( check_thresholds(ss, key, result, 1.0) )
 		{
 		threshold_crossed(ss, key, result);
-		event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+		event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
 		}
 	}
 
-event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
+event SumStats::cluster_key_response(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool)
 	{
 	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
 
 	# We only want to try and do a value merge if there are actually measured datapoints
 	# in the Result.
-	if ( uid in key_requests )
-		key_requests[uid] = compose_results(key_requests[uid], result);
-	else
+	if ( uid !in key_requests || |key_requests[uid]| == 0 )
 		key_requests[uid] = result;
+	else
+		key_requests[uid] = compose_results(key_requests[uid], result);
 
 	# Mark that a worker is done.
 	++done_with[uid];
@@ -251,30 +240,39 @@ event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		local ss = stats_store[ssid];
+		local ss = stats_store[ss_name];
 		local ir = key_requests[uid];
 		if ( check_thresholds(ss, key, ir, 1.0) )
 			{
 			threshold_crossed(ss, key, ir);
-			event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+			event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
 			}
 
-		delete done_with[uid];
-		delete key_requests[uid];
-		# Check that there is an outstanding view before subtracting.
-		if ( outstanding_global_views[ssid] > 0 )
-			--outstanding_global_views[ssid];
+		if ( cleanup )
+			{
+			# We only want to delete the data if this is a non dynamic 
+			# request because the dynamic requests use when statements
+			# and the data needs to remain available.
+			delete key_requests[uid];
+			delete done_with[uid];
+
+			# Check that there is an outstanding view before subtracting.
+			# Global views only apply to non-dynamic requests. Dynamic
+			# requests must be serviced.
+			if ( outstanding_global_views[ss_name] > 0 )
+				--outstanding_global_views[ss_name];
+			}
 		}
 	}
 
 # Managers handle intermediate updates here.
-event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
+event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 	{
 	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting key data for %s", key2str(key));
 
-	if ( ssid in outstanding_global_views &&
-	     |outstanding_global_views[ssid]| > max_outstanding_global_views )
+	if ( ss_name in outstanding_global_views &&
+	     |outstanding_global_views[ss_name]| > max_outstanding_global_views )
 		{
 		# Don't do this intermediate update.  Perhaps at some point in the future
 		# we will queue and randomly select from these ignored intermediate
@@ -282,13 +280,14 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
 		return;
 		}
 
-	++outstanding_global_views[ssid];
+	++outstanding_global_views[ss_name];
 
 	local uid = unique_id("");
-	event SumStats::cluster_key_request(uid, ssid, key);
+	done_with[uid] = 0;
+	event SumStats::cluster_key_request(uid, ss_name, key, T);
 	}
 
-event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
+event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 
@@ -297,7 +296,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
 		++done_with[uid];
 
 	local local_data = stats_results[uid];
-	local ss = stats_store[ssid];
+	local ss = stats_store[ss_name];
 
 	for ( key in data )
 		{
@@ -314,13 +313,14 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
 			if ( check_thresholds(ss, key, local_data[key], 1.0) )
 				{
 				threshold_crossed(ss, key, local_data[key]);
-				event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+				event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
 				}
 			}
 		}
 
 	# If the data has been collected from all peers, we are done and ready to finish.
-	if ( Cluster::worker_count == done_with[uid] )
+	if ( Cluster::worker_count == done_with[uid] &&
+	     /^dyn-/ !in uid )
 		{
 		if ( ss?$epoch_finished )
 			ss$epoch_finished(local_data);
@@ -328,14 +328,60 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
 		# Clean up
 		delete stats_results[uid];
 		delete done_with[uid];
-		# Not sure I need to reset the sumstat on the manager.
 		reset(ss);
 		}
 	}
 
-event remote_connection_handshake_done(p: event_peer) &priority=5
+function request(ss_name: string): ResultTable
 	{
-	send_id(p, "SumStats::stats_store");
-	send_id(p, "SumStats::reducer_store");
+	# This only needs to be implemented this way for cluster compatibility.
+	local uid = unique_id("dyn-");
+	stats_results[uid] = table();
+	done_with[uid] = 0;
+	event SumStats::cluster_ss_request(uid, ss_name, F);
+
+	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+		{
+		if ( uid in stats_results )
+			{
+			local ss_result = stats_results[uid];
+			# Clean up
+			delete stats_results[uid];
+			delete done_with[uid];
+			reset(stats_store[ss_name]);
+			return ss_result;
+			}
+		else
+			return table();
+		}
+	timeout 1.1min
+		{
+		Reporter::warning(fmt("Dynamic SumStat request for %s took longer than 1 minute and was automatically cancelled.", ss_name));
+		return table();
+		}
 	}
+
+function request_key(ss_name: string, key: Key): Result
+	{
+	local uid = unique_id("dyn-");
+	done_with[uid] = 0;
+	key_requests[uid] = table();
+
+	event SumStats::cluster_key_request(uid, ss_name, key, F);
+	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+		{
+		local result = key_requests[uid];
+		# Clean up
+		delete key_requests[uid];
+		delete done_with[uid];
+
+		return result;
+		}
+	timeout 1.1min
+		{
+		Reporter::warning(fmt("Dynamic SumStat key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
+		return table();
+		}
+	}
+
 @endif
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index 6864966766..b1e5761483 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -87,6 +87,10 @@ export {
 	## is no assurance provided as to where the callbacks
 	## will be executed on clusters.
 	type SumStat: record {
+		## An arbitrary name for the sumstat so that it can 
+		## be referred to later.
+		name:               string;
+		
 		## The interval at which this filter should be "broken"
 		## and the '$epoch_finished' callback called.  The
 		## results are also reset at this time so any threshold
@@ -102,22 +106,22 @@ export {
 		## :bro:see:`Result` structure which will be used
 		## for thresholding.
 		## This is required if a $threshold value is given.
-		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): count &optional;
+		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): double &optional;
 
 		## The threshold value for calling the
 		## $threshold_crossed callback.
-		threshold:          count             &optional;
+		threshold:          double            &optional;
 
 		## A series of thresholds for calling the
 		## $threshold_crossed callback.
-		threshold_series:   vector of count   &optional;
+		threshold_series:   vector of double  &optional;
 
 		## A callback that is called when a threshold is crossed.
 		threshold_crossed:  function(key: SumStats::Key, result: SumStats::Result) &optional;
 
 		## A callback with the full collection of Results for
 		## this SumStat.
-		epoch_finished:    function(rt: SumStats::ResultTable) &optional;
+		epoch_finished:     function(rt: SumStats::ResultTable) &optional;
 	};
 
 	## Create a summary statistic.
@@ -134,19 +138,37 @@ export {
 	## obs: The data point to send into the stream.
 	global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
 
-	## This record is primarily used for internal threshold tracking.
-	type Thresholding: record {
-		# Internal use only.  Indicates if a simple threshold was already crossed.
-		is_threshold_crossed: bool &default=F;
+	## Dynamically request a sumstat.  This function should be
+	## used sparingly and not as a replacement for the callbacks 
+	## from the :bro:see:`SumStat` record.  The function is only
+	## available for use within "when" statements as an asynchronous
+	## function.
+	##
+	## ss_name: SumState name.
+	##
+	## Returns: The result table for the requested sumstat.
+	global request: function(ss_name: string): ResultTable;
 
-		# Internal use only.  Current key for threshold series.
-		threshold_series_index: count &default=0;
-	};
+	## Dynamically request a sumstat key.  This function should be
+	## used sparingly and not as a replacement for the callbacks 
+	## from the :bro:see:`SumStat` record.  The function is only
+	## available for use within "when" statements as an asynchronous
+	## function.
+	##
+	## ss_name: SumStat name.
+	##
+	## key: The SumStat key being requested.
+	##
+	## Returns: The result for the requested sumstat key.
+	global request_key: function(ss_name: string, key: Key): Result;
+
+	## This record is primarily used for internal threshold tracking.
+	type Thresholding: record {};
 
 	## This event is generated when thresholds are reset for a SumStat.
 	##
-	## ssid: SumStats ID that thresholds were reset for.
-	global thresholds_reset: event(ssid: string);
+	## name: SumStats name that thresholds were reset for.
+	global thresholds_reset: event(name: string);
 
 	## Helper function to represent a :bro:type:`SumStats::Key` value as
 	## a simple string.
@@ -157,19 +179,43 @@ export {
 	global key2str: function(key: SumStats::Key): string;
 }
 
+# The function prototype for plugins to do calculations.
+type ObserveFunc: function(r: Reducer, val: double, data: Observation, rv: ResultVal);
+
 redef record Reducer += {
-	# Internal use only.  Provides a reference back to the related SumStats by it's ID.
-	sid: string &optional;
+	# Internal use only.  Provides a reference back to the related SumStats by its name.
+	ssname: string &optional;
+
+	calc_funcs: vector of Calculation &optional;
 };
 
+redef record Thresholding += {
+	# Internal use only.  Indicates if a simple threshold was already crossed.
+	is_threshold_crossed: bool &default=F;
+
+	# Internal use only.  Current key for threshold series.
+	threshold_series_index: count &default=0;
+};
+
+
 # Internal use only.  For tracking thresholds per sumstat and key.
 global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
 
 redef record SumStat += {
+	# Internal use only.
+	ssname: string &optional;
+
 	# Internal use only (mostly for cluster coherency).
 	id: string &optional;
 };
 
+# Prototype the hook point for plugins to initialize any result values.
+global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
+
+# Prototype the hook point for plugins to merge Results.
+global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
+
+
 # Store of sumstats indexed on the sumstat id.
 global stats_store: table[string] of SumStat = table();
 
@@ -182,20 +228,20 @@ global result_store: table[string] of ResultTable = table();
 # Store of threshold information.
 global thresholds_store: table[string, Key] of bool = table();
 
+# Store the calculations.
+global calc_store: table[Calculation] of ObserveFunc = table();
+
+# Store the dependencies for Calculations.
+global calc_deps: table[Calculation] of vector of Calculation = table();
+
+# Hook for registering observation calculation plugins.
+global register_observe_plugins: hook();
+
 # This is called whenever key values are updated and the new val is given as the
 # `val` argument. It's only prototyped here because cluster and non-cluster have
 # separate  implementations.
 global data_added: function(ss: SumStat, key: Key, result: Result);
 
-# Prototype the hook point for plugins to do calculations.
-global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
-
-# Prototype the hook point for plugins to initialize any result values.
-global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
-
-# Prototype the hook point for plugins to merge Results.
-global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
-
 # Event that is used to "finish" measurements and adapt the measurement
 # framework for clustered or non-clustered usage.
 global finish_epoch: event(ss: SumStat);
@@ -210,6 +256,24 @@ function key2str(key: Key): string
 	return fmt("sumstats_key(%s)", out);
 	}
 
+function register_observe_plugin(calc: Calculation, func: ObserveFunc)
+	{
+	calc_store[calc] = func;
+	}
+
+function add_observe_plugin_dependency(calc: Calculation, depends_on: Calculation)
+	{
+	if ( calc !in calc_deps )
+		calc_deps[calc] = vector();
+	calc_deps[calc][|calc_deps[calc]|] = depends_on;
+	}
+
+event bro_init() &priority=100000
+	{
+	# Call all of the plugin registration hooks
+	hook register_observe_plugins();
+	}
+
 function init_resultval(r: Reducer): ResultVal
 	{
 	local rv: ResultVal = [$begin=network_time(), $end=network_time()];
@@ -234,25 +298,17 @@ function compose_results(r1: Result, r2: Result): Result
 	{
 	local result: Result = table();
 
-	if ( |r1| > |r2| )
+	for ( id in r1 )
 		{
-		for ( data_id in r1 )
-			{
-			if ( data_id in r2 )
-				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
-			else
-				result[data_id] = r1[data_id];
-			}
+		result[id] = r1[id];
 		}
-	else
+
+	for ( id in r2 )
 		{
-		for ( data_id in r2 )
-			{
-			if ( data_id in r1 )
-				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
-			else
-				result[data_id] = r2[data_id];
-			}
+		if ( id in r1 )
+			result[id] = compose_resultvals(r1[id], r2[id]);
+		else
+			result[id] = r2[id];
 		}
 
 	return result;
@@ -261,18 +317,42 @@ function compose_results(r1: Result, r2: Result): Result
 
 function reset(ss: SumStat)
 	{
-	if ( ss$id in result_store )
-		delete result_store[ss$id];
+	if ( ss$name in result_store )
+		delete result_store[ss$name];
 
-	result_store[ss$id] = table();
+	result_store[ss$name] = table();
 
 	if ( ss?$threshold || ss?$threshold_series )
 		{
-		threshold_tracker[ss$id] = table();
-		event SumStats::thresholds_reset(ss$id);
+		threshold_tracker[ss$name] = table();
+		event SumStats::thresholds_reset(ss$name);
 		}
 	}
 
+# This could potentially recurse forever, but plugin authors 
+# should be making sure they aren't causing reflexive dependencies.
+function add_calc_deps(calcs: vector of Calculation, c: Calculation)
+	{
+	#print fmt("Checking for deps for %s", c);
+	for ( i in calc_deps[c] )
+		{
+		local skip_calc=F;
+		for ( j in calcs )
+			{
+			if ( calcs[j] == calc_deps[c][i] )
+				skip_calc=T;
+			}
+		if ( ! skip_calc )
+			{
+			if ( calc_deps[c][i] in calc_deps )
+				add_calc_deps(calcs, calc_deps[c][i]);
+			calcs[|c|] = calc_deps[c][i];
+			#print fmt("add dep for %s [%s] ", c, calc_deps[c][i]);
+			}
+		}
+
+	}
+
 function create(ss: SumStat)
 	{
 	if ( (ss?$threshold || ss?$threshold_series) && ! ss?$threshold_val )
@@ -280,14 +360,32 @@ function create(ss: SumStat)
 		Reporter::error("SumStats given a threshold with no $threshold_val function");
 		}
 
-	if ( ! ss?$id )
-		ss$id=unique_id("");
-	threshold_tracker[ss$id] = table();
-	stats_store[ss$id] = ss;
+	threshold_tracker[ss$name] = table();
+	stats_store[ss$name] = ss;
 
 	for ( reducer in ss$reducers )
 		{
-		reducer$sid = ss$id;
+		reducer$ssname = ss$name;
+		reducer$calc_funcs = vector();
+		for ( calc in reducer$apply )
+			{
+			# Add in dependencies recursively.
+			if ( calc in calc_deps )
+				add_calc_deps(reducer$calc_funcs, calc);
+
+			# Don't add this calculation to the vector if 
+			# it was already added by something else as a 
+			# dependency.
+			local skip_calc=F;
+			for ( j in reducer$calc_funcs )
+				{
+				if ( calc == reducer$calc_funcs[j] )
+					skip_calc=T;
+				}
+			if ( ! skip_calc )
+				reducer$calc_funcs[|reducer$calc_funcs|] = calc;
+			}
+
 		if ( reducer$stream !in reducer_store )
 			reducer_store[reducer$stream] = set();
 		add reducer_store[reducer$stream][reducer];
@@ -313,7 +411,7 @@ function observe(id: string, key: Key, obs: Observation)
 		if ( r?$pred && ! r$pred(key, obs) )
 			next;
 
-		local ss = stats_store[r$sid];
+		local ss = stats_store[r$ssname];
 
 		# If there is a threshold and no epoch_finished callback
 		# we don't need to continue counting since the data will
@@ -324,17 +422,21 @@ function observe(id: string, key: Key, obs: Observation)
 		#       future if on demand access is provided to the
 		#       SumStats results.
 		if ( ! ss?$epoch_finished &&
-		     r$sid in threshold_tracker &&
-		     key in threshold_tracker[r$sid] &&
 		     ( ss?$threshold &&
-		       threshold_tracker[r$sid][key]$is_threshold_crossed ) ||
+		       r$ssname in threshold_tracker &&
+		       key in threshold_tracker[r$ssname] &&
+		       threshold_tracker[r$ssname][key]$is_threshold_crossed ) ||
 		     ( ss?$threshold_series &&
-		       threshold_tracker[r$sid][key]$threshold_series_index+1 == |ss$threshold_series| ) )
+		       r$ssname in threshold_tracker &&
+		       key in threshold_tracker[r$ssname] &&
+		       threshold_tracker[r$ssname][key]$threshold_series_index == |ss$threshold_series| ) )
+			{
 			next;
+			}
 
-		if ( r$sid !in result_store )
-			result_store[ss$id] = table();
-		local results = result_store[r$sid];
+		if ( r$ssname !in result_store )
+			result_store[r$ssname] = table();
+		local results = result_store[r$ssname];
 
 		if ( key !in results )
 			results[key] = table();
@@ -350,10 +452,13 @@ function observe(id: string, key: Key, obs: Observation)
 
 		# If a string was given, fall back to 1.0 as the value.
 		local val = 1.0;
-		if ( obs?$num || obs?$dbl )
-			val = obs?$dbl ? obs$dbl : obs$num;
+		if ( obs?$num )
+			val = obs$num;
+		else if ( obs?$dbl )
+			val = obs$dbl;
 
-		hook observe_hook(r, val, obs, result_val);
+		for ( i in r$calc_funcs )
+			calc_store[r$calc_funcs[i]](r, val, obs, result_val);
 		data_added(ss, key, result);
 		}
 	}
@@ -366,6 +471,8 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
 		return F;
 
 	# Add in the extra ResultVals to make threshold_vals easier to write.
+	# This length comparison should work because we just need to make 
+	# sure that we have the same number of reducers and results.
 	if ( |ss$reducers| != |result| )
 		{
 		for ( reducer in ss$reducers )
@@ -378,11 +485,11 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
 	local watch = ss$threshold_val(key, result);
 
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
-		watch = double_to_count(floor(watch/modify_pct));
+		watch = watch/modify_pct;
 
-	if ( ss$id !in threshold_tracker )
-		threshold_tracker[ss$id] = table();
-	local t_tracker = threshold_tracker[ss$id];
+	if ( ss$name !in threshold_tracker )
+		threshold_tracker[ss$name] = table();
+	local t_tracker = threshold_tracker[ss$name];
 
 	if ( key !in t_tracker )
 		{
@@ -398,7 +505,7 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
 		}
 
 	if ( ss?$threshold_series &&
-	     |ss$threshold_series| >= tt$threshold_series_index &&
+	     |ss$threshold_series| > tt$threshold_series_index &&
 	     watch >= ss$threshold_series[tt$threshold_series_index] )
 		{
 		# A threshold series was given and the value crossed the next
@@ -426,7 +533,7 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
 		}
 
 	ss$threshold_crossed(key, result);
-	local tt = threshold_tracker[ss$id][key];
+	local tt = threshold_tracker[ss$name][key];
 	tt$is_threshold_crossed = T;
 
 	# Bump up to the next threshold series index if a threshold series is being used.
diff --git a/scripts/base/frameworks/sumstats/non-cluster.bro b/scripts/base/frameworks/sumstats/non-cluster.bro
index f27d4b5cfb..265261f1bd 100644
--- a/scripts/base/frameworks/sumstats/non-cluster.bro
+++ b/scripts/base/frameworks/sumstats/non-cluster.bro
@@ -4,9 +4,9 @@ module SumStats;
 
 event SumStats::finish_epoch(ss: SumStat)
 	{
-	if ( ss$id in result_store )
+	if ( ss$name in result_store )
 		{
-		local data = result_store[ss$id];
+		local data = result_store[ss$name];
 		if ( ss?$epoch_finished )
 			ss$epoch_finished(data);
 
@@ -16,9 +16,32 @@ event SumStats::finish_epoch(ss: SumStat)
 	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 
-
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	if ( check_thresholds(ss, key, result, 1.0) )
 		threshold_crossed(ss, key, result);
 	}
+
+function request(ss_name: string): ResultTable
+	{
+	# This only needs to be implemented this way for cluster compatibility.
+	return when ( T )
+		{
+		if ( ss_name in result_store )
+			return result_store[ss_name];
+		else
+			return table();
+		}
+	}
+
+function request_key(ss_name: string, key: Key): Result
+	{
+	# This only needs to be implemented this way for cluster compatibility.
+	return when ( T )
+		{
+		if ( ss_name in result_store && key in result_store[ss_name] )
+			return result_store[ss_name][key];
+		else
+			return table();
+		}
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro
index a409bb9408..8f7f7b568f 100644
--- a/scripts/base/frameworks/sumstats/plugins/average.bro
+++ b/scripts/base/frameworks/sumstats/plugins/average.bro
@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,17 +14,18 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( AVERAGE in r$apply )
+	register_observe_plugin(AVERAGE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$average )
 			rv$average = val;
 		else
 			rv$average += (val - rv$average) / rv$num;
-		}
+		});
 	}
 
+
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$average && rv2?$average )
diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro
index 6167d31f10..d43ad9dc38 100644
--- a/scripts/base/frameworks/sumstats/plugins/max.bro
+++ b/scripts/base/frameworks/sumstats/plugins/max.bro
@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,15 +14,15 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( MAX in r$apply )
+	register_observe_plugin(MAX, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$max )
 			rv$max = val;
 		else if ( val > rv$max )
 			rv$max = val;
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro
index a15ed0e733..014755cf32 100644
--- a/scripts/base/frameworks/sumstats/plugins/min.bro
+++ b/scripts/base/frameworks/sumstats/plugins/min.bro
@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,17 +14,18 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( MIN in r$apply )
+	register_observe_plugin(MIN, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$min )
 			rv$min = val;
 		else if ( val < rv$min )
 			rv$min = val;
-		}
+		});
 	}
 
+
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$min && rv2?$min )
diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro
index d0587bde08..92d17e503d 100644
--- a/scripts/base/frameworks/sumstats/plugins/sample.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sample.bro
@@ -1,9 +1,14 @@
-@load base/frameworks/sumstats
 @load base/utils/queue
+@load ../main
 
 module SumStats;
 
 export {
+	redef enum Calculation += {
+		## Collect a sample of the last few observations.
+		SAMPLE
+	};
+
 	redef record Reducer += {
 		## A number of sample Observations to collect.
 		samples: count &default=0;
@@ -27,14 +32,14 @@ function get_samples(rv: ResultVal): vector of Observation
 	return s;
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( r$samples > 0 )
+	register_observe_plugin(SAMPLE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$samples )
 			rv$samples = Queue::init([$max_len=r$samples]);
 		Queue::put(rv$samples, obs);
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
index 6411fe4bce..2e5b95b212 100644
--- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro
+++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
@@ -1,5 +1,5 @@
 @load ./variance
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -21,11 +21,18 @@ function calc_std_dev(rv: ResultVal)
 		rv$std_dev = sqrt(rv$variance);
 	}
 
-# This depends on the variance plugin which uses priority -5
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
+hook std_dev_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
-	if ( STD_DEV in r$apply )
+	calc_std_dev(rv);
+	}
+
+hook register_observe_plugins() &priority=-10
+	{
+	register_observe_plugin(STD_DEV, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
 		calc_std_dev(rv);
+		});
+	add_observe_plugin_dependency(STD_DEV, VARIANCE);
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro
index 3e5b28e2be..074b4b72f3 100644
--- a/scripts/base/frameworks/sumstats/plugins/sum.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sum.bro
@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,19 +14,19 @@ export {
 		sum: double &default=0.0;
 	};
 
-	type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
-	global sum_threshold: function(data_id: string): threshold_function;
+	#type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
+	#global sum_threshold: function(data_id: string): threshold_function;
 }
 
-function sum_threshold(data_id: string): threshold_function
-	{
-	return function(key: SumStats::Key, result: SumStats::Result): count
-		{
-		print fmt("data_id: %s", data_id);
-		print result;
-		return double_to_count(result[data_id]$sum);
-		};
-	}
+#function sum_threshold(data_id: string): threshold_function
+#	{
+#	return function(key: SumStats::Key, result: SumStats::Result): count
+#		{
+#		print fmt("data_id: %s", data_id);
+#		print result;
+#		return double_to_count(result[data_id]$sum);
+#		};
+#	}
 
 hook init_resultval_hook(r: Reducer, rv: ResultVal)
 	{
@@ -34,10 +34,12 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( SUM in r$apply )
+	register_observe_plugin(SUM, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
 		rv$sum += val;
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro
index a407a487a2..15fc28adb6 100644
--- a/scripts/base/frameworks/sumstats/plugins/unique.bro
+++ b/scripts/base/frameworks/sumstats/plugins/unique.bro
@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -23,15 +23,15 @@ redef record ResultVal += {
 	unique_vals: set[Observation] &optional;
 };
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( UNIQUE in r$apply )
+	register_observe_plugin(UNIQUE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$unique_vals )
 			rv$unique_vals=set();
 		add rv$unique_vals[obs];
 		rv$unique = |rv$unique_vals|;
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro
index 1e7a00ea97..12d30cc4fe 100644
--- a/scripts/base/frameworks/sumstats/plugins/variance.bro
+++ b/scripts/base/frameworks/sumstats/plugins/variance.bro
@@ -1,5 +1,5 @@
 @load ./average
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -28,17 +28,17 @@ function calc_variance(rv: ResultVal)
 	rv$variance = (rv$num > 1) ? rv$var_s/(rv$num-1) : 0.0;
 	}
 
-# Reduced priority since this depends on the average
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
+hook register_observe_plugins() &priority=-5
 	{
-	if ( VARIANCE in r$apply )
+	register_observe_plugin(VARIANCE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( rv$num > 1 )
 			rv$var_s += ((val - rv$prev_avg) * (val - rv$average));
 
 		calc_variance(rv);
 		rv$prev_avg = rv$average;
-		}
+		});
+	add_observe_plugin_dependency(VARIANCE, AVERAGE);
 	}
 
 # Reduced priority since this depends on the average
diff --git a/scripts/policy/misc/app-stats/__load__.bro b/scripts/policy/misc/app-stats/__load__.bro
new file mode 100644
index 0000000000..c468d055ee
--- /dev/null
+++ b/scripts/policy/misc/app-stats/__load__.bro
@@ -0,0 +1,2 @@
+@load ./main
+@load ./plugins
\ No newline at end of file
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-stats/main.bro
similarity index 58%
rename from scripts/policy/misc/app-metrics.bro
rename to scripts/policy/misc/app-stats/main.bro
index 3df38ad8ad..e4a38c6893 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-stats/main.bro
@@ -1,3 +1,6 @@
+#! AppStats collects information about web applications in use
+#! on the network.  
+
 @load base/protocols/http
 @load base/protocols/ssl
 @load base/frameworks/sumstats
@@ -30,13 +33,17 @@ redef record connection += {
 	resp_hostname: string &optional;
 };
 
+global add_sumstats: hook(id: conn_id, hostname: string, size: count);
+
+
 event bro_init() &priority=3
 	{
 	Log::create_stream(AppStats::LOG, [$columns=Info]);
 
 	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
 	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=break_interval,
+	SumStats::create([$name="app-metrics",
+	                  $epoch=break_interval,
 	                  $reducers=set(r1, r2),
 	                  $epoch_finished(data: SumStats::ResultTable) =
 	                  	{
@@ -55,41 +62,6 @@ event bro_init() &priority=3
 	                  	}]);
 	}
 
-function add_sumstats(id: conn_id, hostname: string, size: count)
-	{
-	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.google\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
-		{
-		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
-		}
-	else if ( /\.gmail\.com$/ in hostname && size > 20 )
-		{
-		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
-		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
-		}
-}
-
-
 event ssl_established(c: connection)
 	{
 	if ( c?$ssl && c$ssl?$server_name )
@@ -99,11 +71,11 @@ event ssl_established(c: connection)
 event connection_finished(c: connection)
 	{
 	if ( c?$resp_hostname )
-		add_sumstats(c$id, c$resp_hostname, c$resp$size);
+		hook add_sumstats(c$id, c$resp_hostname, c$resp$size);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if( rec?$host )
-		add_sumstats(rec$id, rec$host, rec$response_body_len);
+		hook add_sumstats(rec$id, rec$host, rec$response_body_len);
 	}
diff --git a/scripts/policy/misc/app-stats/plugins/__load__.bro b/scripts/policy/misc/app-stats/plugins/__load__.bro
new file mode 100644
index 0000000000..7a3ea2da81
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/__load__.bro
@@ -0,0 +1,6 @@
+@load ./facebook
+@load ./gmail
+@load ./google
+@load ./netflix
+@load ./pandora
+@load ./youtube
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/facebook.bro b/scripts/policy/misc/app-stats/plugins/facebook.bro
new file mode 100644
index 0000000000..edcb02b72a
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/facebook.bro
@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.(facebook\.com|fbcdn\.net)$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/gmail.bro b/scripts/policy/misc/app-stats/plugins/gmail.bro
new file mode 100644
index 0000000000..1642fb7651
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/gmail.bro
@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.gmail\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/google.bro b/scripts/policy/misc/app-stats/plugins/google.bro
new file mode 100644
index 0000000000..e1da3a9068
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/google.bro
@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.google\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/netflix.bro b/scripts/policy/misc/app-stats/plugins/netflix.bro
new file mode 100644
index 0000000000..5d429f0caf
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/netflix.bro
@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/pandora.bro b/scripts/policy/misc/app-stats/plugins/pandora.bro
new file mode 100644
index 0000000000..6cfbfab72d
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/pandora.bro
@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/misc/app-stats/plugins/youtube.bro b/scripts/policy/misc/app-stats/plugins/youtube.bro
new file mode 100644
index 0000000000..af872cfdac
--- /dev/null
+++ b/scripts/policy/misc/app-stats/plugins/youtube.bro
@@ -0,0 +1,12 @@
+@load ../main
+
+module AppStats;
+
+hook add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index c194d03e13..0cb7082919 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -29,7 +29,7 @@ export {
 	## Defines the threshold for ICMP Time Exceeded messages for a src-dst pair.
 	## This threshold only comes into play after a host is found to be
 	## sending low ttl packets.
-	const icmp_time_exceeded_threshold = 3 &redef;
+	const icmp_time_exceeded_threshold: double = 3 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`ICMPTimeExceeded::icmp_time_exceeded_threshold` variable to be crossed.
@@ -57,16 +57,17 @@ event bro_init() &priority=5
 
 	local r1: SumStats::Reducer = [$stream="traceroute.time_exceeded", $apply=set(SumStats::UNIQUE)];
 	local r2: SumStats::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=icmp_time_exceeded_interval,
+	SumStats::create([$name="traceroute-detection",
+	                  $epoch=icmp_time_exceeded_interval,
 	                  $reducers=set(r1, r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	# Give a threshold value of zero depending on if the host
 	                  	# sends a low ttl packet.
 	                  	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
-	                  		return 0;
+	                  		return 0.0;
 	                  	else
-	                  		return result["traceroute.time_exceeded"]$unique;
+	                  		return result["traceroute.time_exceeded"]$unique+0;
 	                  	},
 	                  $threshold=icmp_time_exceeded_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index f3dcaf2291..f9c47a0311 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -39,15 +39,11 @@ export {
 
 	## The threshold of a unique number of hosts a scanning host has to have failed
 	## connections with on a single port.
-	const addr_scan_threshold = 25 &redef;
+	const addr_scan_threshold = 25.0 &redef;
 
 	## The threshold of a number of unique ports a scanning host has to have failed
 	## connections with on a single victim host.
-	const port_scan_threshold = 15 &redef;
-
-	## Custom thresholds based on service for address scan.  This is primarily
-	## useful for setting reduced thresholds for specific ports.
-	const addr_scan_custom_thresholds: table[port] of count &redef;
+	const port_scan_threshold = 15.0 &redef;
 
 	global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
 	global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
@@ -56,11 +52,12 @@ export {
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="scan.addr.fail", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=addr_scan_interval,
+	SumStats::create([$name="addr-scan",
+	                  $epoch=addr_scan_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["scan.addr.fail"]$unique);
+	                  	return result["scan.addr.fail"]$unique+0.0;
 	                  	},
 	                  #$threshold_func=check_addr_scan_threshold,
 	                  $threshold=addr_scan_threshold,
@@ -80,11 +77,12 @@ event bro_init() &priority=5
 
 	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
 	local r2: SumStats::Reducer = [$stream="scan.port.fail", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=port_scan_interval,
+	SumStats::create([$name="port-scan",
+	                  $epoch=port_scan_interval,
 	                  $reducers=set(r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["scan.port.fail"]$unique);
+	                  	return result["scan.port.fail"]$unique+0.0;
 	                  	},
 	                  $threshold=port_scan_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 21c9c403c7..36dfafb53a 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -17,7 +17,7 @@ export {
 
 	## How many rejected usernames or passwords are required before being
 	## considered to be bruteforcing.
-	const bruteforce_threshold = 20 &redef;
+	const bruteforce_threshold: double = 20 &redef;
 
 	## The time period in which the threshold needs to be crossed before
 	## being reset.
@@ -28,11 +28,12 @@ export {
 event bro_init()
 	{
 	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE)];
-	SumStats::create([$epoch=bruteforce_measurement_interval,
+	SumStats::create([$name="ftp-detect-bruteforcing",
+	                  $epoch=bruteforce_measurement_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return result["ftp.failed_auth"]$num;
+	                  	return result["ftp.failed_auth"]$num+0.0;
 	                  	},
 	                  $threshold=bruteforce_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 11dba0dc46..1d8d9d0904 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -28,7 +28,7 @@ export {
 	## Defines the threshold that determines if an SQL injection attack
 	## is ongoing based on the number of requests that appear to be SQL
 	## injection attacks.
-	const sqli_requests_threshold = 50 &redef;
+	const sqli_requests_threshold: double = 50.0 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`HTTP::sqli_requests_threshold` variable to be crossed.
@@ -64,11 +64,12 @@ event bro_init() &priority=3
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
 	local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples];
-	SumStats::create([$epoch=sqli_requests_interval,
+	SumStats::create([$name="detect-sqli-attackers",
+	                  $epoch=sqli_requests_interval,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["http.sqli.attacker"]$sum);
+	                  	return result["http.sqli.attacker"]$sum;
 	                  	},
 	                  $threshold=sqli_requests_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
@@ -82,11 +83,12 @@ event bro_init() &priority=3
 	                  	}]);
 
 	local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples];
-	SumStats::create([$epoch=sqli_requests_interval,
+	SumStats::create([$name="detect-sqli-victims",
+	                  $epoch=sqli_requests_interval,
 	                  $reducers=set(r2),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["http.sqli.victim"]$sum);
+	                  	return result["http.sqli.victim"]$sum;
 	                  	},
 	                  $threshold=sqli_requests_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index 309905e939..161a314b8c 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -27,7 +27,7 @@ export {
 
 	## The number of failed SSH connections before a host is designated as
 	## guessing passwords.
-	const password_guesses_limit = 30 &redef;
+	const password_guesses_limit: double = 30 &redef;
 
 	## The amount of time to remember presumed non-successful logins to build
 	## model of a password guesser.
@@ -43,11 +43,12 @@ export {
 event bro_init()
 	{
 	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=guessing_timeout,
+	SumStats::create([$name="detect-ssh-bruteforcing",
+	                  $epoch=guessing_timeout,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["ssh.login.failure"]$sum);
+	                  	return result["ssh.login.failure"]$sum;
 	                  	},
 	                  $threshold=password_guesses_limit,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
diff --git a/scripts/site/local.bro b/scripts/site/local.bro
index dfebd9923a..90832b4a07 100644
--- a/scripts/site/local.bro
+++ b/scripts/site/local.bro
@@ -11,6 +11,13 @@
 # Load the scan detection script.
 @load misc/scan
 
+# Log some information about web applications being used by users 
+# on your network.
+@load misc/app-metrics
+
+# Detect traceroute being run on the network.  
+@load misc/detect-traceroute
+
 # Generate notices when vulnerable versions of software are discovered.
 # The default is to only monitor software found in the address space defined
 # as "local".  Refer to the software framework's documentation for more 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout
new file mode 100644
index 0000000000..ed14a1c753
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout
@@ -0,0 +1,7 @@
+Complete SumStat request
+    Host: 6.5.4.3 -> 1
+    Host: 10.10.10.10 -> 5
+    Host: 1.2.3.4 -> 169
+    Host: 7.2.1.5 -> 145
+SumStat key request
+    Host: 7.2.1.5 -> 145
diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout
new file mode 100644
index 0000000000..876c368eb3
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout
@@ -0,0 +1,5 @@
+Complete SumStat request
+    Host: 1.2.3.4 -> 42
+    Host: 4.3.2.1 -> 7
+Key request for 1.2.3.4
+    Host: 1.2.3.4 -> 42
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
index 1b7903ca1a..956c43a57b 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
@@ -23,7 +23,8 @@ global n = 0;
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
-	SumStats::create([$epoch=5secs,
+	SumStats::create([$name="test",
+	                  $epoch=5secs,
 	                  $reducers=set(r1),
 	                  $epoch_finished(rt: SumStats::ResultTable) =
 	                  	{
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic.bro b/testing/btest/scripts/base/frameworks/sumstats/basic.bro
index 0b2851bf10..54160cbf54 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/basic.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic.bro
@@ -11,16 +11,17 @@ event bro_init() &priority=5
 	                                          SumStats::MIN, 
 	                                          SumStats::STD_DEV,
 	                                          SumStats::UNIQUE)];
-	SumStats::create([$epoch=3secs,
-	                     $reducers=set(r1),
-	                     $epoch_finished(data: SumStats::ResultTable) = 
-	                     	{
-	                     	for ( key in data )
-	                     		{
-	                     		local r = data[key]["test.metric"];
-	                     		print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
-	                     		}
-	                     	}
+	SumStats::create([$name="test",
+	                  $epoch=3secs,
+	                  $reducers=set(r1),
+	                  $epoch_finished(data: SumStats::ResultTable) = 
+	                  	{
+	                  	for ( key in data )
+	                  		{
+	                  		local r = data[key]["test.metric"];
+	                  		print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
+	                  		}
+	                  	}
 		 ]);
 
 	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=5]);
diff --git a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
index 303a0dc852..d54e432030 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
@@ -20,13 +20,14 @@ redef Log::default_rotation_interval = 0secs;
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=1hr,
+	SumStats::create([$name="test",
+	                  $epoch=1hr,
 	                  $reducers=set(r1),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	return double_to_count(result["test.metric"]$sum);
+	                  	return result["test.metric"]$sum;
 	                  	},
-	                  $threshold=100,
+	                  $threshold=100.0,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);
diff --git a/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro
new file mode 100644
index 0000000000..9ee0511b47
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro
@@ -0,0 +1,93 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait 15
+
+# @TEST-EXEC: btest-diff manager-1/.stdout
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+redef Log::default_rotation_interval = 0secs;
+
+global n = 0;
+
+event bro_init() &priority=5
+	{
+	local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
+	SumStats::create([$name="test sumstat",
+	                  $epoch=1hr,
+	                  $reducers=set(r1)]);
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready_for_data: event();
+redef Cluster::manager2worker_events += /^ready_for_data$/;
+
+event ready_for_data()
+	{
+	if ( Cluster::node == "worker-1" )
+		{
+		SumStats::observe("test", [$host=1.2.3.4], [$num=34]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test", [$host=6.5.4.3], [$num=1]);
+		SumStats::observe("test", [$host=7.2.1.5], [$num=54]);
+		}
+	if ( Cluster::node == "worker-2" )
+		{
+		SumStats::observe("test", [$host=1.2.3.4], [$num=75]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test", [$host=7.2.1.5], [$num=91]);
+		SumStats::observe("test", [$host=10.10.10.10], [$num=5]);
+		}
+	}
+
+
+event on_demand2()
+	{
+	local host = 7.2.1.5;
+	when ( local result = SumStats::request_key("test sumstat", [$host=host]) )
+		{
+		print "SumStat key request";
+		print fmt("    Host: %s -> %.0f", host, result["test"]$sum);
+		terminate();
+		}
+	}
+
+event on_demand()
+	{
+	when ( local results = SumStats::request("test sumstat") )
+		{
+		print "Complete SumStat request";
+		for ( key in results )
+			print fmt("    Host: %s -> %.0f", key$host, results[key]["test"]$sum);
+
+		event on_demand2();
+		}
+	}
+
+global peer_count = 0;
+event remote_connection_handshake_done(p: event_peer) &priority=-5
+	{
+	++peer_count;
+	if ( peer_count == 2 )
+		{
+		if ( Cluster::local_node_type() == Cluster::MANAGER )
+			event ready_for_data();
+		
+		schedule 1sec { on_demand() };
+		}
+	}
+
diff --git a/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro b/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro
new file mode 100644
index 0000000000..f93e1a72dc
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro
@@ -0,0 +1,45 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+redef exit_only_after_terminate=T;
+
+
+event on_demand()
+	{
+	when ( local results = SumStats::request("test") )
+		{
+		print "Complete SumStat request";
+		for ( key in results )
+			{
+			print fmt("    Host: %s -> %.0f", key$host, results[key]["test.reducer"]$sum);
+			}
+		}
+	}
+
+event on_demand_key()
+	{
+	local host = 1.2.3.4;
+	when ( local result = SumStats::request_key("test", [$host=host]) )
+		{
+		print fmt("Key request for %s", host);
+		print fmt("    Host: %s -> %.0f", host, result["test.reducer"]$sum);
+		terminate();
+		}
+	}
+
+event bro_init() &priority=5
+	{
+	local r1: SumStats::Reducer = [$stream="test.reducer", 
+	                               $apply=set(SumStats::SUM)];
+	SumStats::create([$name="test",
+	                  $epoch=1hr,
+	                  $reducers=set(r1)]);
+
+	# Seed some data but notice there are no callbacks defined in the sumstat!
+	SumStats::observe("test.reducer", [$host=1.2.3.4], [$num=42]);
+	SumStats::observe("test.reducer", [$host=4.3.2.1], [$num=7]);
+
+	schedule 0.1 secs { on_demand() };
+	schedule 1 secs { on_demand_key() };
+	}
+
diff --git a/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro b/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro
index ddc053bd23..b00b30a375 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro
@@ -8,14 +8,15 @@ redef enum Notice::Type += {
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=3secs,
+	SumStats::create([$name="test1",
+	                  $epoch=3secs,
 	                  $reducers=set(r1),
 	                  #$threshold_val = SumStats::sum_threshold("test.metric"),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{ 
-	                  	return double_to_count(result["test.metric"]$sum);
+	                  	return result["test.metric"]$sum;
 	                  	},
-	                  $threshold=5,
+	                  $threshold=5.0,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
 	                  	{
 	                  	local r = result["test.metric"];
@@ -24,14 +25,15 @@ event bro_init() &priority=5
 	                  ]);
 
 	local r2: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=3secs,
+	SumStats::create([$name="test2",
+	                  $epoch=3secs,
 	                  $reducers=set(r2),
 	                  #$threshold_val = SumStats::sum_threshold("test.metric"),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{ 
-	                  	return double_to_count(result["test.metric"]$sum); 
+	                  	return result["test.metric"]$sum;
 	                  	},
-	                  $threshold_series=vector(3,6,800),
+	                  $threshold_series=vector(3.0,6.0,800.0),
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
 	                  	{
 	                  	local r = result["test.metric"];
@@ -41,19 +43,20 @@ event bro_init() &priority=5
 
 	local r3: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
 	local r4: SumStats::Reducer = [$stream="test.metric2", $apply=set(SumStats::SUM)];
-	SumStats::create([$epoch=3secs,
+	SumStats::create([$name="test3",
+	                  $epoch=3secs,
 	                  $reducers=set(r3, r4),
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
 	                  	{ 
 	                  	# Calculate a ratio between sums of two reducers.
 	                  	if ( "test.metric2" in result && "test.metric" in result &&
 	                  	     result["test.metric"]$sum > 0 )
-	                  		return double_to_count(result["test.metric2"]$sum / result["test.metric"]$sum);
+	                  		return result["test.metric2"]$sum / result["test.metric"]$sum;
 	                  	else
-	                  		return 0;
+	                  		return 0.0;
 	                  	},
 	                  # Looking for metric2 sum to be 5 times the sum of metric
-	                  $threshold=5, 
+	                  $threshold=5.0, 
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	local thold = result["test.metric2"]$sum / result["test.metric"]$sum;

From c4a1f30a87df3a8f04e4dfb7af08c26aead7e841 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 22 May 2013 14:59:31 -0400
Subject: [PATCH 02/25] Hopefully fixing a strange error.

---
 scripts/base/frameworks/sumstats/cluster.bro | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index 59cb4a8b47..e747c190fa 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -33,7 +33,7 @@ export {
 
 	## Event sent by nodes that are collecting sumstats after receiving a request for
 	## the sumstat from the manager.
-	global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool);
+	global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool);
 
 	## This event is sent by the manager in a cluster to initiate the collection of
 	## a single key value from a sumstat.  It's typically used to get intermediate
@@ -115,7 +115,7 @@ event SumStats::send_data(uid: string, ss_name: string, data: ResultTable, clean
 
 	# Note: copy is needed to compensate serialization caching issue. This should be
 	# changed to something else later. 
-	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done);
+	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done, cleanup);
 	if ( ! done )
 		schedule 0.01 sec { SumStats::send_data(uid, ss_name, incoming_data, T) };
 	}
@@ -175,7 +175,7 @@ event SumStats::thresholds_reset(ss_name: string)
 # This variable is maintained by manager nodes as they collect and aggregate
 # results.
 # Index on a uid.
-global stats_results: table[string] of ResultTable &create_expire=1min &default=table();
+global stats_results: table[string] of ResultTable &create_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid
@@ -292,7 +292,7 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 	event SumStats::cluster_key_request(uid, ss_name, key, T);
 	}
 
-event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool)
+event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 
@@ -300,6 +300,13 @@ event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTa
 	if ( done )
 		++done_with[uid];
 
+	# We had better only be getting requests for stuff that exists.
+	if ( ss_name !in stats_store )
+		return;
+
+	if ( uid !in stats_results )
+		stats_results[uid] = table();
+
 	local local_data = stats_results[uid];
 	local ss = stats_store[ss_name];
 
@@ -324,8 +331,7 @@ event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTa
 		}
 
 	# If the data has been collected from all peers, we are done and ready to finish.
-	if ( Cluster::worker_count == done_with[uid] &&
-	     /^dyn-/ !in uid )
+	if ( cleanup && Cluster::worker_count == done_with[uid] )
 		{
 		if ( ss?$epoch_finished )
 			ss$epoch_finished(local_data);

From 6bd9ab3bd671783e969783ca2ddfcc5ac96afcc4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 22 May 2013 16:41:46 -0400
Subject: [PATCH 03/25] More adjustments to try and correct SumStats memory
 use.

---
 scripts/base/frameworks/sumstats/cluster.bro |  2 +-
 scripts/base/frameworks/sumstats/main.bro    | 51 +++++++++++---------
 2 files changed, 30 insertions(+), 23 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index e747c190fa..01ec237948 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -164,7 +164,7 @@ event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, t
 
 event SumStats::thresholds_reset(ss_name: string)
 	{
-	threshold_tracker[ss_name] = table();
+	delete threshold_tracker[ss_name];
 	}
 
 @endif
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index b1e5761483..804b611319 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -204,9 +204,6 @@ global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
 redef record SumStat += {
 	# Internal use only.
 	ssname: string &optional;
-
-	# Internal use only (mostly for cluster coherency).
-	id: string &optional;
 };
 
 # Prototype the hook point for plugins to initialize any result values.
@@ -215,7 +212,6 @@ global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
 # Prototype the hook point for plugins to merge Results.
 global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
 
-
 # Store of sumstats indexed on the sumstat id.
 global stats_store: table[string] of SumStat = table();
 
@@ -322,8 +318,10 @@ function reset(ss: SumStat)
 
 	result_store[ss$name] = table();
 
-	if ( ss?$threshold || ss?$threshold_series )
+	if ( (ss?$threshold || ss?$threshold_series) && 
+	     ss$name in threshold_tracker )
 		{
+		delete threshold_tracker[ss$name];
 		threshold_tracker[ss$name] = table();
 		event SumStats::thresholds_reset(ss$name);
 		}
@@ -360,7 +358,6 @@ function create(ss: SumStat)
 		Reporter::error("SumStats given a threshold with no $threshold_val function");
 		}
 
-	threshold_tracker[ss$name] = table();
 	stats_store[ss$name] = ss;
 
 	for ( reducer in ss$reducers )
@@ -491,28 +488,38 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
 		threshold_tracker[ss$name] = table();
 	local t_tracker = threshold_tracker[ss$name];
 
-	if ( key !in t_tracker )
+	if ( ss?$threshold )
 		{
-		local ttmp: Thresholding;
-		t_tracker[key] = ttmp;
-		}
-	local tt = t_tracker[key];
+		local tt: Thresholding;
+		if ( key in t_tracker )
+			tt = t_tracker[key];
+		
+		if ( ! tt$is_threshold_crossed && 
+		     watch >= ss$threshold )
+			{
+			t_tracker[key] = tt;
 
-	if ( ss?$threshold && ! tt$is_threshold_crossed && watch >= ss$threshold )
-		{
-		# Value crossed the threshold.
-		return T;
+			# Value crossed the threshold.
+			return T;
+			}
 		}
 
-	if ( ss?$threshold_series &&
-	     |ss$threshold_series| > tt$threshold_series_index &&
-	     watch >= ss$threshold_series[tt$threshold_series_index] )
+	if ( ss?$threshold_series )
 		{
-		# A threshold series was given and the value crossed the next
-		# value in the series.
-		return T;
-		}
+		local tt2: Thresholding;
+		if ( key in t_tracker )
+			tt2 = t_tracker[key];
 
+		if ( |ss$threshold_series| > tt2$threshold_series_index &&
+		     watch >= ss$threshold_series[tt2$threshold_series_index] )
+			{
+			t_tracker[key] = tt2;
+
+			# A threshold series was given and the value crossed the next
+			# value in the series.
+			return T;
+			}
+		}
 	return F;
 	}
 

From 4f4ef99a6b12d4d44a7aa2600b5317d2e3806376 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 23 May 2013 10:12:17 -0400
Subject: [PATCH 04/25] SumStats changes to how thresholding works to simplify
 and reduce memory use.

---
 scripts/base/frameworks/sumstats/cluster.bro |   6 +-
 scripts/base/frameworks/sumstats/main.bro    | 101 ++++++++-----------
 2 files changed, 47 insertions(+), 60 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index 01ec237948..a69dd1ab54 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -54,7 +54,7 @@ export {
 	global send_data: event(uid: string, ss_name: string, data: ResultTable, cleanup: bool);
 
 	## This event is generated when a threshold is crossed.
-	global cluster_threshold_crossed: event(ss_name: string, key: SumStats::Key, thold: Thresholding);
+	global cluster_threshold_crossed: event(ss_name: string, key: SumStats::Key, thold_index: count);
 }
 
 # Add events to the cluster framework to make this work.
@@ -154,12 +154,12 @@ event SumStats::cluster_key_request(uid: string, ss_name: string, key: Key, clea
 		}
 	}
 
-event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, thold: Thresholding)
+event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, thold_index: count)
 	{
 	if ( ss_name !in threshold_tracker )
 		threshold_tracker[ss_name] = table();
 
-	threshold_tracker[ss_name][key] = thold;
+	threshold_tracker[ss_name][key] = thold_index;
 	}
 
 event SumStats::thresholds_reset(ss_name: string)
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index 804b611319..5c822e9983 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -189,22 +189,30 @@ redef record Reducer += {
 	calc_funcs: vector of Calculation &optional;
 };
 
-redef record Thresholding += {
-	# Internal use only.  Indicates if a simple threshold was already crossed.
-	is_threshold_crossed: bool &default=F;
-
-	# Internal use only.  Current key for threshold series.
-	threshold_series_index: count &default=0;
-};
-
-
 # Internal use only.  For tracking thresholds per sumstat and key.
-global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
+# In the case of a single threshold, 0 means the threshold isn't crossed.
+# In the case of a threshold series, the number tracks the threshold offset.
+global threshold_tracker: table[string] of table[Key] of count;
 
-redef record SumStat += {
-	# Internal use only.
-	ssname: string &optional;
-};
+function increment_threshold_tracker(ss_name: string, key: Key)
+	{
+	if ( ss_name !in threshold_tracker )
+		threshold_tracker[ss_name] = table();
+	if ( key !in threshold_tracker[ss_name] )
+		threshold_tracker[ss_name][key] = 0;
+
+	++threshold_tracker[ss_name][key];
+	}
+
+function get_threshold_index(ss_name: string, key: Key): count
+	{
+	if ( ss_name !in threshold_tracker )
+		return 0;
+	if ( key !in threshold_tracker[ss_name] )
+		return 0;
+
+	return threshold_tracker[ss_name][key];
+	}
 
 # Prototype the hook point for plugins to initialize any result values.
 global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
@@ -318,8 +326,7 @@ function reset(ss: SumStat)
 
 	result_store[ss$name] = table();
 
-	if ( (ss?$threshold || ss?$threshold_series) && 
-	     ss$name in threshold_tracker )
+	if ( ss$name in threshold_tracker )
 		{
 		delete threshold_tracker[ss$name];
 		threshold_tracker[ss$name] = table();
@@ -360,6 +367,9 @@ function create(ss: SumStat)
 
 	stats_store[ss$name] = ss;
 
+	if ( ss?$threshold || ss?$threshold_series )
+		threshold_tracker[ss$name] = table();
+
 	for ( reducer in ss$reducers )
 		{
 		reducer$ssname = ss$name;
@@ -420,13 +430,11 @@ function observe(id: string, key: Key, obs: Observation)
 		#       SumStats results.
 		if ( ! ss?$epoch_finished &&
 		     ( ss?$threshold &&
-		       r$ssname in threshold_tracker &&
 		       key in threshold_tracker[r$ssname] &&
-		       threshold_tracker[r$ssname][key]$is_threshold_crossed ) ||
+		       threshold_tracker[r$ssname][key] != 0 ) ||
 		     ( ss?$threshold_series &&
-		       r$ssname in threshold_tracker &&
 		       key in threshold_tracker[r$ssname] &&
-		       threshold_tracker[r$ssname][key]$threshold_series_index == |ss$threshold_series| ) )
+		       threshold_tracker[r$ssname][key] == |ss$threshold_series| ) )
 			{
 			next;
 			}
@@ -464,7 +472,7 @@ function observe(id: string, key: Key, obs: Observation)
 # mid-break-interval threshold crossing detection for cluster deployments.
 function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: double): bool
 	{
-	if ( ! (ss?$threshold || ss?$threshold_series) )
+	if ( ! (ss?$threshold || ss?$threshold_series || ss?$threshold_crossed) )
 		return F;
 
 	# Add in the extra ResultVals to make threshold_vals easier to write.
@@ -484,42 +492,25 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
 		watch = watch/modify_pct;
 
-	if ( ss$name !in threshold_tracker )
-		threshold_tracker[ss$name] = table();
-	local t_tracker = threshold_tracker[ss$name];
+	local t_index = get_threshold_index(ss$name, key);
 
-	if ( ss?$threshold )
+	if ( ss?$threshold &&
+	     t_index == 0 && # Check that the threshold hasn't already been crossed.
+	     watch >= ss$threshold )
 		{
-		local tt: Thresholding;
-		if ( key in t_tracker )
-			tt = t_tracker[key];
-		
-		if ( ! tt$is_threshold_crossed && 
-		     watch >= ss$threshold )
-			{
-			t_tracker[key] = tt;
-
-			# Value crossed the threshold.
-			return T;
-			}
+		# Value crossed the threshold.
+		return T;
 		}
 
-	if ( ss?$threshold_series )
+	if ( ss?$threshold_series &&
+	     |ss$threshold_series| > t_index && # Check if there are more thresholds.
+	     watch >= ss$threshold_series[t_index] )
 		{
-		local tt2: Thresholding;
-		if ( key in t_tracker )
-			tt2 = t_tracker[key];
-
-		if ( |ss$threshold_series| > tt2$threshold_series_index &&
-		     watch >= ss$threshold_series[tt2$threshold_series_index] )
-			{
-			t_tracker[key] = tt2;
-
-			# A threshold series was given and the value crossed the next
-			# value in the series.
-			return T;
-			}
+		# A threshold series was given and the value crossed the next
+		# value in the series.
+		return T;
 		}
+
 	return F;
 	}
 
@@ -529,6 +520,8 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
 	if ( ! ss?$threshold_crossed )
 		return;
 
+	increment_threshold_tracker(ss$name,key);
+
 	# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
 	if ( |ss$reducers| != |result| )
 		{
@@ -540,11 +533,5 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
 		}
 
 	ss$threshold_crossed(key, result);
-	local tt = threshold_tracker[ss$name][key];
-	tt$is_threshold_crossed = T;
-
-	# Bump up to the next threshold series index if a threshold series is being used.
-	if ( ss?$threshold_series )
-		++tt$threshold_series_index;
 	}
 

From 26f8bd7ad7d07a0cb03197bce6076ae0833f8b33 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 20 Jul 2013 01:21:01 -0400
Subject: [PATCH 05/25] Fix a reporter message in sumstats.

---
 scripts/base/frameworks/sumstats/main.bro | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index 5c822e9983..3afa507c7a 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -122,6 +122,12 @@ export {
 		## A callback with the full collection of Results for
 		## this SumStat.
 		epoch_finished:     function(rt: SumStats::ResultTable) &optional;
+		#epoch_finished:     function(num_keys: count) &optional;
+
+		## A callback that receives each of the results at the
+		## end of the analysis epoch.  The function will be 
+		## called once for each key.
+		#epoch_finished_result:     function(key::SumStats::Key, result: SumStats::Result) &optional;
 	};
 
 	## Create a summary statistic.
@@ -162,9 +168,6 @@ export {
 	## Returns: The result for the requested sumstat key.
 	global request_key: function(ss_name: string, key: Key): Result;
 
-	## This record is primarily used for internal threshold tracking.
-	type Thresholding: record {};
-
 	## This event is generated when thresholds are reset for a SumStat.
 	##
 	## name: SumStats name that thresholds were reset for.
@@ -429,6 +432,7 @@ function observe(id: string, key: Key, obs: Observation)
 		#       future if on demand access is provided to the
 		#       SumStats results.
 		if ( ! ss?$epoch_finished &&
+			 r$ssname in threshold_tracker &&
 		     ( ss?$threshold &&
 		       key in threshold_tracker[r$ssname] &&
 		       threshold_tracker[r$ssname][key] != 0 ) ||

From 58f59b9bc3f00adeb46a0a1259ec3ab8ef05d355 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 20 Jul 2013 02:08:52 -0400
Subject: [PATCH 06/25] Add server samples to SSH bruteforce detection.

---
 scripts/policy/protocols/ssh/detect-bruteforcing.bro | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index 161a314b8c..f8d41db2b8 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -42,7 +42,7 @@ export {
 
 event bro_init()
 	{
-	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM)];
+	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=5];
 	SumStats::create([$name="detect-ssh-bruteforcing",
 	                  $epoch=guessing_timeout,
 	                  $reducers=set(r1),
@@ -54,9 +54,15 @@ event bro_init()
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	local r = result["ssh.login.failure"];
+	                  	local sub_msg = fmt("Sampled servers: ");
+	                  	for ( i in r$samples )
+	                  		{
+	                  		sub_msg = fmt("%s%s %s", sub_msg, i==0 ? "":",", r$samples[i]$str);
+	                  		}
 	                  	# Generate the notice.
 	                  	NOTICE([$note=Password_Guessing,
 	                  	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
+	                  	        $sub=sub_msg,
 	                  	        $src=key$host,
 	                  	        $identifier=cat(key$host)]);
 	                  	# Insert the guesser into the intel framework.
@@ -83,5 +89,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		SumStats::observe("ssh.login.failure", [$host=id$orig_h], [$num=1]);
+		SumStats::observe("ssh.login.failure", [$host=id$orig_h], [$str=cat(id$resp_h)]);
 	}

From deeb5ec38e2f0c3dadcb7138d672cf1ff905d577 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Thu, 18 Jul 2013 21:17:06 -0400
Subject: [PATCH 07/25] Rework the DHCP analyzer to make it compatible again.

---
 scripts/base/init-default.bro                 |   1 +
 scripts/base/protocols/dhcp/__load__.bro      |   2 +
 scripts/base/protocols/dhcp/consts.bro        |  20 ++
 scripts/base/protocols/dhcp/main.bro          |  74 +++++
 scripts/base/protocols/dhcp/utils.bro         |  21 ++
 .../protocols/dhcp/handle_extra_msg_types.bro | 256 +++++++++++++++++
 src/analyzer/protocol/dhcp/DHCP.cc            |   1 -
 src/analyzer/protocol/dhcp/DHCP.h             |   3 +
 src/analyzer/protocol/dhcp/dhcp-analyzer.pac  | 269 ++++++++++--------
 src/analyzer/protocol/dhcp/dhcp-protocol.pac  |  34 +--
 src/analyzer/protocol/dhcp/dhcp.pac           |   1 +
 src/analyzer/protocol/dhcp/events.bif         | 195 ++++---------
 src/util.cc                                   |  20 ++
 src/util.h                                    |   1 +
 14 files changed, 623 insertions(+), 275 deletions(-)
 create mode 100644 scripts/base/protocols/dhcp/__load__.bro
 create mode 100644 scripts/base/protocols/dhcp/consts.bro
 create mode 100644 scripts/base/protocols/dhcp/main.bro
 create mode 100644 scripts/base/protocols/dhcp/utils.bro
 create mode 100644 scripts/policy/protocols/dhcp/handle_extra_msg_types.bro

diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro
index 72ba0bf115..6aaf7446b9 100644
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@@ -36,6 +36,7 @@
 @load base/frameworks/tunnels
 
 @load base/protocols/conn
+@load base/protocols/dhcp
 @load base/protocols/dns
 @load base/protocols/ftp
 @load base/protocols/http
diff --git a/scripts/base/protocols/dhcp/__load__.bro b/scripts/base/protocols/dhcp/__load__.bro
new file mode 100644
index 0000000000..0098b81a7a
--- /dev/null
+++ b/scripts/base/protocols/dhcp/__load__.bro
@@ -0,0 +1,2 @@
+@load ./consts
+@load ./main
\ No newline at end of file
diff --git a/scripts/base/protocols/dhcp/consts.bro b/scripts/base/protocols/dhcp/consts.bro
new file mode 100644
index 0000000000..81914a98c3
--- /dev/null
+++ b/scripts/base/protocols/dhcp/consts.bro
@@ -0,0 +1,20 @@
+##! Types, errors, and fields for analyzing DHCP data.  A helper file
+##! for DHCP analysis scripts.
+
+module DHCP;
+
+export {
+       
+	## Types of DHCP messages. See RFC 1533.
+	const message_types = {
+       		[1] = "DHCP_DISCOVER",
+		[2] = "DHCP_OFFER",
+		[3] = "DHCP_REQUEST",
+		[4] = "DHCP_DECLINE",
+		[5] = "DHCP_ACK",
+		[6] = "DHCP_NAK",
+		[7] = "DHCP_RELEASE",
+		[8] = "DHCP_INFORM",
+	} &default = function(n: count): string { return fmt("unknown-message-type-%d", n); };
+
+}
\ No newline at end of file
diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro
new file mode 100644
index 0000000000..9fc63df152
--- /dev/null
+++ b/scripts/base/protocols/dhcp/main.bro
@@ -0,0 +1,74 @@
+##! Analyzes DHCP traffic in order to log DHCP leases given to clients.
+##! This script ignores large swaths of the protocol, since it is rather
+##! noisy on most networks, and focuses on the end-result: assigned leases.
+##! 
+##! To enable further analysis and log output for DHCP, see the optional
+##! scripts in the policy/protocols/dhcp directory.
+
+@load ./utils.bro
+
+module DHCP;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## The record type which contains the column fields of the DHCP log.
+	type Info: record {
+		## The earliest time at which a DHCP message over the
+		## associated connection is observed.
+		ts:		time		&log;
+		## A unique identifier of the connection over which DHCP is
+		## occuring.
+		uid:		string		&log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:		conn_id		&log;
+		## Client's hardware address.
+		mac:		string		&log &optional;
+		## Client's actual assigned IP address.
+		assigned_ip:	addr		&log &optional;
+		## IP address lease interval.
+		lease_time:	interval	&log &optional;
+		## A random number choosen by the client for this transaction.
+		trans_id:	count		&log;
+	};
+
+        ## Event that can be handled to access the DHCP
+        ## record as it is sent on to the logging framework.
+        global log_dhcp: event(rec: Info);
+}
+
+# Add the dhcp info to the connection record
+redef record connection += {
+	dhcp: Info &optional;
+};
+
+const ports = { 67/udp, 68/udp };
+redef likely_server_ports += { 67/udp };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
+	Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
+	}
+
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=5
+	{
+	local info: Info;
+	info$ts          = network_time();
+	info$id          = c$id;
+	info$uid         = c$uid;
+	info$assigned_ip = reverse_ip(msg$yiaddr);
+	info$lease_time  = lease;
+	info$trans_id    = msg$xid;
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+	
+	c$dhcp = info;
+	}
+
+# We let policy scripts add stuff too, so we run this at a lower priority
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=1
+	{
+	Log::write(DHCP::LOG, c$dhcp);
+	}
diff --git a/scripts/base/protocols/dhcp/utils.bro b/scripts/base/protocols/dhcp/utils.bro
new file mode 100644
index 0000000000..cb06450088
--- /dev/null
+++ b/scripts/base/protocols/dhcp/utils.bro
@@ -0,0 +1,21 @@
+##! Utilities specific for DHCP processing.
+
+@load ./main
+
+module DHCP;
+
+export {
+	## Reverse the octets of an IPv4 IP.
+	##
+	## ip: An :bro:type:`addr` IPv4 address.
+	##
+	## Returns: A reversed addr.
+	global reverse_ip: function(ip: addr): addr;
+}
+
+function reverse_ip(ip: addr): addr
+	{
+	local octets = split(cat(ip), /\./);
+	return to_addr(cat(octets[4], ".", octets[3], ".", octets[2], ".", octets[1]));
+	}
+
diff --git a/scripts/policy/protocols/dhcp/handle_extra_msg_types.bro b/scripts/policy/protocols/dhcp/handle_extra_msg_types.bro
new file mode 100644
index 0000000000..e5fa713da4
--- /dev/null
+++ b/scripts/policy/protocols/dhcp/handle_extra_msg_types.bro
@@ -0,0 +1,256 @@
+##! Handlers for DHCP message types other than DHCPACK, which is handled in base/protocols/dhcp.
+##! For networks that wish to get more details from their DHCP logs, at the expense
+##! of a significantly higher log rate.
+
+@load base/protocols/dhcp
+
+module DHCP;
+
+export {
+	redef record Info += {
+		## The value of the host name option, if seen
+		host_name:	string		&log &optional;
+		## The IP requested by the client, if any
+		requested_ip:	addr		&log &optional;
+		## The type of the DHCP message (DHCPOFFER, DHCPRELEASE, etc.)
+		msg_type:	string		&log &optional;
+	};
+
+	#### Enabled by default
+	
+	## A boolean value to determine if DHCPREQUEST messages are logged.
+	## Often useful to see client activity, and because host_name is often available.
+	const log_dhcprequest = T &redef;
+
+	## A boolean value to determine if DHCPDECLINE messages are logged.
+	## A client declines a lease if it detects that the IP is already in use (usually via ARP).
+	const log_dhcpdecline = T &redef;
+
+	## A boolean value to determine if DHCPNAK messages are logged.
+	## A server issues a DHCPNAK if a client DHCPREQUEST is invalid.
+	const log_dhcpnak = T &redef;
+
+	## A boolean value to determine if DHCPRELEASE messages are logged.
+	## A client issues a DHCPRELEASE when it no longer needs the lease (e.g. it's shutting down).
+	const log_dhcprelease = T &redef;
+
+	#### Not enabled by default
+
+	## A boolean value to determine if DHCPOFFER messages are logged.
+	## Used to profile server -> client communication.
+	const log_dhcpoffer = F &redef;
+	
+	## A boolean value to determine if DHCPDISCOVER messages are logged.
+	## Used to profile broadcast client discovery requests.
+	const log_dhcpdiscover = F &redef;
+	
+	## A boolean value to determine if DHCPINFORM messages are logged.
+	## Used to profile clients attempting to request/renew specific IPs.
+	const log_dhcpinform = F &redef;
+
+}
+
+event dhcp_offer(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=5
+	{
+	if ( ! log_dhcpoffer )
+		return;
+	
+	local info: Info;
+	info$ts          = network_time();
+	info$id          = c$id;
+	info$uid         = c$uid;
+	info$assigned_ip = reverse_ip(msg$yiaddr);
+	info$lease_time  = lease;
+	info$trans_id    = msg$xid;
+	info$msg_type    = "DHCPOFFER";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_discover(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &priority=5
+	{
+	if ( ! log_dhcpdiscover )
+		return;
+	
+	local info: Info;
+	info$ts           = network_time();
+	info$id           = c$id;
+	info$uid          = c$uid;
+	info$requested_ip = req_addr;
+	info$trans_id     = msg$xid;
+	info$msg_type     = "DHCPDISCOVER";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &priority=5
+	{
+	if ( ! log_dhcprequest )
+		return;
+	
+	local info: Info;
+	info$ts           = network_time();
+	info$id           = c$id;
+	info$uid          = c$uid;
+	info$requested_ip = req_addr;
+	info$trans_id     = msg$xid;
+	info$msg_type     = "DHCPREQUEST";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_decline(c: connection, msg: dhcp_msg, host_name: string) &priority=5
+	{
+	if ( ! log_dhcpdecline )
+		return;
+	
+	local info: Info;
+	info$ts           = network_time();
+	info$id           = c$id;
+	info$uid          = c$uid;
+	info$trans_id     = msg$xid;
+	info$msg_type     = "DHCPDECLINE";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_nak(c: connection, msg: dhcp_msg, host_name: string) &priority=5
+	{
+	if ( ! log_dhcpnak )
+		return;
+	
+	local info: Info;
+	info$ts           = network_time();
+	info$id           = c$id;
+	info$uid          = c$uid;
+	info$trans_id     = msg$xid;
+	info$msg_type     = "DHCPNAK";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_release(c: connection, msg: dhcp_msg, host_name: string) &priority=5
+	{
+	if ( ! log_dhcprelease )
+		return;
+	
+	local info: Info;
+	info$ts           = network_time();
+	info$id           = c$id;
+	info$uid          = c$uid;
+	info$trans_id     = msg$xid;
+	info$msg_type     = "DHCPRELEASE";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_inform(c: connection, msg: dhcp_msg, host_name: string) &priority=5
+	{
+	if ( ! log_dhcpinform )
+		return;
+	
+	local info: Info;
+	info$ts           = network_time();
+	info$id           = c$id;
+	info$uid          = c$uid;
+	info$trans_id     = msg$xid;
+	info$msg_type     = "DHCPINFORM";
+
+	if ( msg$h_addr != "" )
+		info$mac = msg$h_addr;
+
+	if ( host_name != "" )
+		info$host_name = host_name;
+
+	c$dhcp = info;
+	}
+
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=4
+	{
+	## For the sake of consistency, let's add msg_type to DHCPACK as well.
+	c$dhcp$msg_type = "DHCPACK";
+	## host_name is generally not in ACKs, but let's check anyway.
+	if ( host_name != "" )
+		c$dhcp$host_name = host_name;
+	}
+
+#### We log stuff at a lower priority, in case any other scripts would like to extend the Info record first.
+
+event dhcp_offer(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=1
+	{
+	if ( log_dhcpoffer )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
+event dhcp_discover(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &priority=1
+	{
+	if ( log_dhcpdiscover )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
+event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &priority=1
+	{
+	if ( log_dhcprequest )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
+event dhcp_decline(c: connection, msg: dhcp_msg, host_name: string) &priority=1
+	{
+	if ( log_dhcpdecline )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
+event dhcp_nak(c: connection, msg: dhcp_msg, host_name: string) &priority=1
+	{
+	if ( log_dhcpnak )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
+event dhcp_release(c: connection, msg: dhcp_msg, host_name: string) &priority=1
+	{
+	if ( log_dhcprelease )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
+event dhcp_inform(c: connection, msg: dhcp_msg, host_name: string) &priority=1
+	{
+	if ( log_dhcpinform )
+		Log::write(DHCP::LOG, c$dhcp);
+	}
+
diff --git a/src/analyzer/protocol/dhcp/DHCP.cc b/src/analyzer/protocol/dhcp/DHCP.cc
index 8d05aef37d..1fa8759fbf 100644
--- a/src/analyzer/protocol/dhcp/DHCP.cc
+++ b/src/analyzer/protocol/dhcp/DHCP.cc
@@ -1,4 +1,3 @@
-
 #include "DHCP.h"
 
 #include "events.bif.h"
diff --git a/src/analyzer/protocol/dhcp/DHCP.h b/src/analyzer/protocol/dhcp/DHCP.h
index a1c06e8b85..b9784d34f1 100644
--- a/src/analyzer/protocol/dhcp/DHCP.h
+++ b/src/analyzer/protocol/dhcp/DHCP.h
@@ -19,6 +19,9 @@ public:
 	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new DHCP_Analyzer(conn); }
 
+	static bool Available()
+		{ return dhcp_discover || dhcp_offer || dhcp_request || dhcp_decline || dhcp_ack || dhcp_nak || dhcp_release || dhcp_inform; }
+
 protected:
 	binpac::DHCP::DHCP_Conn* interp;
 };
diff --git a/src/analyzer/protocol/dhcp/dhcp-analyzer.pac b/src/analyzer/protocol/dhcp/dhcp-analyzer.pac
index 5267075445..df928b4be4 100644
--- a/src/analyzer/protocol/dhcp/dhcp-analyzer.pac
+++ b/src/analyzer/protocol/dhcp/dhcp-analyzer.pac
@@ -8,12 +8,10 @@ flow DHCP_Flow(is_orig: bool) {
 
 	%member{
 		BroVal dhcp_msg_val_;
-		BroAnalyzer interp;
 	%}
 
 	%init{
 		dhcp_msg_val_ = 0;
-		interp = connection->bro_analyzer();
 	%}
 
 	%cleanup{
@@ -45,7 +43,7 @@ flow DHCP_Flow(is_orig: bool) {
 			}
 
 		if ( type == 0 )
-			interp->Weird("DHCP_no_type_option");
+			connection()->bro_analyzer()->ProtocolViolation("no DHCP message type option");
 
 		return type;
 		%}
@@ -56,54 +54,63 @@ flow DHCP_Flow(is_orig: bool) {
 
 		// Requested IP address to the server.
 		::uint32 req_addr = 0, serv_addr = 0;
+		StringVal* host_name = 0;
 
-		for ( ptr = options->begin();
-		       ptr != options->end() && ! (*ptr)->last(); ++ptr )
+		for ( ptr = options->begin();  ptr != options->end() && ! (*ptr)->last(); ++ptr )
 			{
-			switch ( (*ptr)->code() ) {
-			case REQ_IP_OPTION:
-				req_addr = htonl((*ptr)->info()->req_addr());
-				break;
+			switch ( (*ptr)->code() ) 
+				{
+				case REQ_IP_OPTION:
+					req_addr = htonl((*ptr)->info()->req_addr());
+					break;
 
-			case SERV_ID_OPTION:
-				serv_addr = htonl((*ptr)->info()->serv_addr());
-				break;
-			}
+				case SERV_ID_OPTION:
+					serv_addr = htonl((*ptr)->info()->serv_addr());
+					break;
+
+				case HOST_NAME_OPTION:
+					host_name = new StringVal((*ptr)->info()->host_name().length(),
+								  (const char*) (*ptr)->info()->host_name().begin());
+					break;
+				}
 			}
 
+		if ( host_name == 0 )
+			host_name = new StringVal("");
+
 		switch ( type )
-		{
-		case DHCPDISCOVER:
-			BifEvent::generate_dhcp_discover(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref(), new AddrVal(req_addr));
-			break;
+			{
+			case DHCPDISCOVER:
+				BifEvent::generate_dhcp_discover(connection()->bro_analyzer(),
+								 connection()->bro_analyzer()->Conn(),
+								 dhcp_msg_val_->Ref(), new AddrVal(req_addr), host_name);
+				break;
 
-		case DHCPREQUEST:
-			BifEvent::generate_dhcp_request(connection()->bro_analyzer(),
-				connection()->bro_analyzer()->Conn(),
-				dhcp_msg_val_->Ref(), new AddrVal(req_addr),
-				new AddrVal(serv_addr));
-			break;
+			case DHCPREQUEST:
+				BifEvent::generate_dhcp_request(connection()->bro_analyzer(),
+								connection()->bro_analyzer()->Conn(),
+								dhcp_msg_val_->Ref(), new AddrVal(req_addr),
+								new AddrVal(serv_addr), host_name);
+				break;
 
-		case DHCPDECLINE:
-			BifEvent::generate_dhcp_decline(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
-			break;
+			case DHCPDECLINE:
+				BifEvent::generate_dhcp_decline(connection()->bro_analyzer(),
+								connection()->bro_analyzer()->Conn(),
+								dhcp_msg_val_->Ref(), host_name);
+				break;
 
-		case DHCPRELEASE:
-			BifEvent::generate_dhcp_release(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
-			break;
+			case DHCPRELEASE:
+				BifEvent::generate_dhcp_release(connection()->bro_analyzer(),
+								connection()->bro_analyzer()->Conn(),
+								dhcp_msg_val_->Ref(), host_name);
+				break;
 
-		case DHCPINFORM:
-			BifEvent::generate_dhcp_inform(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
-			break;
-		}
+			case DHCPINFORM:
+				BifEvent::generate_dhcp_inform(connection()->bro_analyzer(),
+							       connection()->bro_analyzer()->Conn(),
+							       dhcp_msg_val_->Ref(), host_name);
+				break;
+			}
 
 		return true;
 		%}
@@ -118,72 +125,83 @@ flow DHCP_Flow(is_orig: bool) {
 		::uint32 subnet_mask = 0, serv_addr = 0;
 
 		uint32 lease = 0;
+		StringVal* host_name = 0;
 
 		for ( ptr = options->begin();
 		      ptr != options->end() && ! (*ptr)->last(); ++ptr )
 			{
-			switch ( (*ptr)->code() ) {
-			case  SUBNET_OPTION:
-				subnet_mask = htonl((*ptr)->info()->mask());
-				break;
-
-			case  ROUTER_OPTION:
-				// Let's hope there aren't multiple
-				// such options.
-				Unref(router_list);
-				router_list = new TableVal(dhcp_router_list);
-
+			switch ( (*ptr)->code() ) 
 				{
-				int num_routers =
-					(*ptr)->info()->router_list()->size();
+				case SUBNET_OPTION:
+					subnet_mask = htonl((*ptr)->info()->mask());
+					break;
 
-				for ( int i = 0; i < num_routers; ++i )
-					{
-					vector<uint32>* rlist =
-						(*ptr)->info()->router_list();
-					uint32 raddr = (*rlist)[i];
-					::uint32 tmp_addr;
-					tmp_addr = htonl(raddr);
-					// index starting from 1
-					Val* index = new Val(i + 1, TYPE_COUNT);
-					router_list->Assign(index, new AddrVal(tmp_addr));
-					Unref(index);
-					}
+				case ROUTER_OPTION:
+					// Let's hope there aren't multiple
+					// such options.
+					Unref(router_list);
+					router_list = new TableVal(dhcp_router_list);
+
+						{
+						int num_routers = (*ptr)->info()->router_list()->size();
+
+						for ( int i = 0; i < num_routers; ++i )
+							{
+							vector<uint32>* rlist = (*ptr)->info()->router_list();
+						
+							uint32 raddr = (*rlist)[i];
+							::uint32 tmp_addr;
+							tmp_addr = htonl(raddr);
+					
+							// index starting from 1
+							Val* index = new Val(i + 1, TYPE_COUNT);
+							router_list->Assign(index, new AddrVal(tmp_addr));
+							Unref(index);
+							}
+						}
+					break;
+
+				case LEASE_OPTION:
+					lease = (*ptr)->info()->lease();
+					break;
+
+				case SERV_ID_OPTION:
+					serv_addr = htonl((*ptr)->info()->serv_addr());
+					break;
+				
+				case HOST_NAME_OPTION:
+					host_name = new StringVal((*ptr)->info()->host_name().length(),
+								  (const char*) (*ptr)->info()->host_name().begin());
+					break;
 				}
-				break;
-
-			case  LEASE_OPTION:
-				lease = (*ptr)->info()->lease();
-				break;
-
-			case  SERV_ID_OPTION:
-				serv_addr = htonl((*ptr)->info()->serv_addr());
-				break;
-			}
 			}
 
-		switch ( type ) {
-		case DHCPOFFER:
-			BifEvent::generate_dhcp_offer(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
-					router_list, lease, new AddrVal(serv_addr));
-			break;
+			if ( host_name == 0 )
+				host_name = new StringVal("");
 
-		case DHCPACK:
-			BifEvent::generate_dhcp_ack(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
-					router_list, lease, new AddrVal(serv_addr));
-			break;
+		switch ( type ) 
+			{
+			case DHCPOFFER:
+				BifEvent::generate_dhcp_offer(connection()->bro_analyzer(),
+							      connection()->bro_analyzer()->Conn(),
+							      dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
+							      router_list, lease, new AddrVal(serv_addr), host_name);
+				break;
 
-		case DHCPNAK:
-			BifEvent::generate_dhcp_nak(connection()->bro_analyzer(),
-					connection()->bro_analyzer()->Conn(),
-					dhcp_msg_val_->Ref());
-			break;
+			case DHCPACK:
+				BifEvent::generate_dhcp_ack(connection()->bro_analyzer(),
+							    connection()->bro_analyzer()->Conn(),
+							    dhcp_msg_val_->Ref(), new AddrVal(subnet_mask),
+							    router_list, lease, new AddrVal(serv_addr), host_name);
+				break;
 
-		}
+			case DHCPNAK:
+				BifEvent::generate_dhcp_nak(connection()->bro_analyzer(),
+							    connection()->bro_analyzer()->Conn(),
+							    dhcp_msg_val_->Ref(), host_name);
+				break;
+
+			}
 
 		return true;
 
@@ -195,7 +213,10 @@ flow DHCP_Flow(is_orig: bool) {
 		// DHCP or BOOTP.  If not, we are unable to interpret
 		// the message options.
 		if ( ${msg.cookie} != 0x63825363 )
+			{
+			connection()->bro_analyzer()->ProtocolViolation(fmt("bad cookie (%d)", ${msg.cookie}));
 			return false;
+			}
 
 		Unref(dhcp_msg_val_);
 		RecordVal* r = new RecordVal(dhcp_msg);
@@ -203,40 +224,44 @@ flow DHCP_Flow(is_orig: bool) {
 		r->Assign(0, new Val(${msg.op}, TYPE_COUNT));
 		r->Assign(1, new Val(${msg.type}, TYPE_COUNT));
 		r->Assign(2, new Val(${msg.xid}, TYPE_COUNT));
-
-		// We want only 6 bytes for Ethernet address.
-		r->Assign(3, new StringVal(6, (const char*) ${msg.chaddr}.begin()));
-
+		r->Assign(3, new StringVal(format_mac(${msg.chaddr}.data())));
 		r->Assign(4, new AddrVal(${msg.ciaddr}));
 		r->Assign(5, new AddrVal(${msg.yiaddr}));
 
 		dhcp_msg_val_ = r;
 
-		switch ( ${msg.op} ) {
-		case BOOTREQUEST:	// presumablye from client to server
-			if ( ${msg.type} == DHCPDISCOVER ||
-			     ${msg.type} == DHCPREQUEST ||
-			     ${msg.type} == DHCPDECLINE ||
-			     ${msg.type} == DHCPRELEASE ||
-			     ${msg.type} == DHCPINFORM )
-				parse_request(${msg.options}, ${msg.type});
-			else
-				interp->Weird("DHCP_wrong_msg_type");
-			break;
+		switch ( ${msg.op} )
+			{
+			case BOOTREQUEST:	// presumably from client to server
+				if ( ${msg.type} == DHCPDISCOVER ||
+			     	     ${msg.type} == DHCPREQUEST ||
+			     	     ${msg.type} == DHCPDECLINE ||
+			     	     ${msg.type} == DHCPRELEASE ||
+			     	     ${msg.type} == DHCPINFORM )
+					parse_request(${msg.options}, ${msg.type});
+				else
+					connection()->bro_analyzer()->ProtocolViolation(fmt("unknown DHCP message type option for BOOTREQUEST (%d)", 
+											    ${msg.type}));
+				break;
 
-		case BOOTREPLY:		// presumably from server to client
-			if ( ${msg.type} == DHCPOFFER ||
-			     ${msg.type} == DHCPACK || ${msg.type} == DHCPNAK )
-				parse_reply(${msg.options}, ${msg.type});
-			else
-				interp->Weird("DHCP_wrong_msg_type");
-			break;
-
-		default:
-			interp->Weird("DHCP_wrong_op_type");
-			break;
-		}
+			case BOOTREPLY:		// presumably from server to client
+				if ( ${msg.type} == DHCPOFFER ||
+			     	     ${msg.type} == DHCPACK || 
+				     ${msg.type} == DHCPNAK )
+					parse_reply(${msg.options}, ${msg.type});
+				else
+					connection()->bro_analyzer()->ProtocolViolation(fmt("unknown DHCP message type option for BOOTREPLY (%d)", 
+											    ${msg.type}));
+					
+				break;
 
+			default:
+				connection()->bro_analyzer()->ProtocolViolation(fmt("unknown DHCP message op code (%d). Known codes: 1=BOOTREQUEST, 2=BOOTREPLY", 
+										${msg.op}));
+				break;
+			}
+		
+		connection()->bro_analyzer()->ProtocolConfirmation();
 		return true;
 		%}
 };
diff --git a/src/analyzer/protocol/dhcp/dhcp-protocol.pac b/src/analyzer/protocol/dhcp/dhcp-protocol.pac
index d77780b1b3..1eb5b399e5 100644
--- a/src/analyzer/protocol/dhcp/dhcp-protocol.pac
+++ b/src/analyzer/protocol/dhcp/dhcp-protocol.pac
@@ -10,13 +10,14 @@ enum OP_type {
 # The option types are by no means complete.
 # Anyone can add a new option type in RFC 1533 to be parsed here.
 enum OPTION_type {
-	SUBNET_OPTION	= 1,
-	ROUTER_OPTION	= 3,
-	REQ_IP_OPTION	= 50,
-	LEASE_OPTION	= 51,
-	MSG_TYPE_OPTION = 53,	
-	SERV_ID_OPTION	= 54,	# Server address, actually :)
-	END_OPTION	= 255,
+	SUBNET_OPTION	 = 1,
+	ROUTER_OPTION	 = 3,
+	HOST_NAME_OPTION = 12,
+	REQ_IP_OPTION	 = 50,
+	LEASE_OPTION	 = 51,
+	MSG_TYPE_OPTION  = 53,	
+	SERV_ID_OPTION	 = 54,	# Server address, actually :)
+	END_OPTION	 = 255,
 };
 
 # Refer to RFC 1533 for message types (with option = 53).
@@ -34,21 +35,22 @@ enum DHCP_message_type {
 type Option_Info(code: uint8)  = record {
 	length		: uint8;
 	value		: case code of {
-		SUBNET_OPTION   -> mask	: uint32;
-		ROUTER_OPTION   -> router_list: uint32[length/4];
-		REQ_IP_OPTION   -> req_addr	: uint32;
-		LEASE_OPTION    -> lease	: uint32;
-		MSG_TYPE_OPTION -> msg_type : uint8;
-		SERV_ID_OPTION  -> serv_addr: uint32;
-		default		-> other: bytestring &length = length;
+		SUBNET_OPTION    -> mask	: uint32;
+		ROUTER_OPTION    -> router_list : uint32[length/4];
+		REQ_IP_OPTION    -> req_addr	: uint32;
+		LEASE_OPTION     -> lease	: uint32;
+		MSG_TYPE_OPTION  -> msg_type    : uint8;
+		SERV_ID_OPTION   -> serv_addr   : uint32;
+		HOST_NAME_OPTION -> host_name   : bytestring &length = length;
+		default		 -> other       : bytestring &length = length;
 	};
 };
 
 type DHCP_Option = record {
 	code		: uint8;
 	data		: case code of {
-		0, 255	->	none	: empty;
-		default	->	info	: Option_Info(code);	
+		0, 255	-> none	: empty;
+		default	-> info	: Option_Info(code);	
 	};
 } &let {
 	last: bool	= (code == 255);   # Mark the end of a list of options
diff --git a/src/analyzer/protocol/dhcp/dhcp.pac b/src/analyzer/protocol/dhcp/dhcp.pac
index c4a684badc..706be31e10 100644
--- a/src/analyzer/protocol/dhcp/dhcp.pac
+++ b/src/analyzer/protocol/dhcp/dhcp.pac
@@ -1,3 +1,4 @@
+%include binpac.pac
 %include bro.pac
 
 %extern{
diff --git a/src/analyzer/protocol/dhcp/events.bif b/src/analyzer/protocol/dhcp/events.bif
index 741504185e..987213dad3 100644
--- a/src/analyzer/protocol/dhcp/events.bif
+++ b/src/analyzer/protocol/dhcp/events.bif
@@ -1,8 +1,5 @@
-## Generated for DHCP messages of type *discover*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPDISCOVER* (client broadcast to locate 
+## available servers).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -10,33 +7,23 @@
 ##
 ## req_addr: The specific address requested by the client.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request dns_max_queries dns_session_timeout
-##    dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_nak 
+##              dhcp_release dhcp_inform
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%);
+event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string%);
 
-## Generated for DHCP messages of type *offer*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPOFFER* (server to client in response to 
+## DHCPDISCOVER with offer of configuration parameters).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
-## msg: TODO.
+## msg: The parsed type-independent part of the DHCP message.
 ##
 ## mask: The subnet mask specified by the message.
 ##
@@ -46,28 +33,21 @@ event dhcp_discover%(c: connection, msg: dhcp_msg, req_addr: addr%);
 ##
 ## serv_addr: The server address specified by the message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_discover dhcp_request dhcp_decline dhcp_ack dhcp_nak 
+##              dhcp_release dhcp_inform
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%);
+event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string%);
 
-## Generated for DHCP messages of type *request*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPREQUEST* (Client message to servers either 
+## (a) requesting offered parameters from one server and implicitly declining offers 
+## from all others, (b) confirming correctness of previously allocated address after,
+## e.g., system reboot, or (c) extending the lease on a particular network address.)
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -77,55 +57,37 @@ event dhcp_offer%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_
 ##
 ## serv_addr: The server address specified by the message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_decline dhcp_ack dhcp_nak 
+##              dhcp_release dhcp_inform
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr%);
+event dhcp_request%(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string%);
 
-## Generated for DHCP messages of type *decline*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPDECLINE* (Client to server indicating 
+## network address is already in use).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_ack dhcp_nak 
+##              dhcp_release dhcp_inform
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_decline%(c: connection, msg: dhcp_msg%);
+event dhcp_decline%(c: connection, msg: dhcp_msg, host_name: string%);
 
-## Generated for DHCP messages of type *acknowledgment*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPACK* (Server to client with configuration 
+## parameters, including committed network address).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
@@ -139,101 +101,62 @@ event dhcp_decline%(c: connection, msg: dhcp_msg%);
 ##
 ## serv_addr: The server address specified by the message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
 ##
-## .. note:: Bro does not support broadcast packets (as used by the DHCP
-##    protocol). It treats broadcast addresses just like any other and
-##    associates packets into transport-level flows in the same way as usual.
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_nak 
+##              dhcp_release dhcp_inform
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr%);
+event dhcp_ack%(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string%);
 
-## Generated for DHCP messages of type *negative acknowledgment*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPNAK* (Server to client indicating client's 
+## notion of network address is incorrect (e.g., client has moved to new subnet) or 
+## client's lease has expired).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_release 
+##              dhcp_inform
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_nak%(c: connection, msg: dhcp_msg%);
+event dhcp_nak%(c: connection, msg: dhcp_msg, host_name: string%);
 
-## Generated for DHCP messages of type *release*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPRELEASE* (Client to server relinquishing 
+## network address and cancelling remaining lease).
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
 ##
-## .. note:: Bro does not support broadcast packets (as used by the DHCP
-##    protocol). It treats broadcast addresses just like any other and
-##    associates packets into transport-level flows in the same way as usual.
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_nak 
+##              dhcp_inform
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_release%(c: connection, msg: dhcp_msg%);
+event dhcp_release%(c: connection, msg: dhcp_msg, host_name: string%);
 
-## Generated for DHCP messages of type *inform*.
-##
-## See `Wikipedia
-## <http://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol>`__ for
-## more information about the DHCP protocol.
+## Generated for DHCP messages of type *DHCPINFORM* (Client to server, asking only for 
+## local configuration parameters; client already has externally configured network 
+## address). 
 ##
 ## c: The connection record describing the underlying UDP flow.
 ##
 ## msg: The parsed type-independent part of the DHCP message.
 ##
-## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
-##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
-##    dns_SRV_reply dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_end
-##    dns_full_request dns_mapping_altered dns_mapping_lost_name dns_mapping_new_name
-##    dns_mapping_unverified dns_mapping_valid dns_message dns_query_reply
-##    dns_rejected dns_request non_dns_request
+## host_name: The value of the host name option, if specified by the client.
+##
+## .. bro:see:: dhcp_discover dhcp_offer dhcp_request dhcp_decline dhcp_ack dhcp_nak 
+##              dhcp_release
 ##
 ## .. note:: Bro does not support broadcast packets (as used by the DHCP
 ##    protocol). It treats broadcast addresses just like any other and
 ##    associates packets into transport-level flows in the same way as usual.
 ##
-## .. todo:: Bro's current default configuration does not activate the protocol
-##    analyzer that generates this event; the corresponding script has not yet
-##    been ported to Bro 2.x. To still enable this event, one needs to
-##    register a port for it or add a DPD payload signature.
-event dhcp_inform%(c: connection, msg: dhcp_msg%);
+event dhcp_inform%(c: connection, msg: dhcp_msg, host_name: string%);
 
diff --git a/src/util.cc b/src/util.cc
index 5a63be22cb..0a349ccce2 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -78,6 +78,26 @@ std::string extract_ip_and_len(const std::string& i, int* len)
 	return extract_ip(i.substr(0, pos));
 	}
 
+/**
+* Given a MAC address, formats it as 00:de:ad:be:ef
+* Supports both EUI-48 and EUI-64. If it's neither, returns
+* an empty string.
+*
+* @param m EUI-48 or EUI-64 MAC address to format, as a char array
+* @return A string of the formatted MAC
+*/
+char* format_mac(const unsigned char* m)
+{  
+	char* buf = new char[24];
+	if (m[6] == 0 && m[7] == 0) // EUI-48
+		snprintf(buf, 18, "%02x:%02x:%02x:%02x:%02x:%02x",
+			 m[0], m[1], m[2], m[3], m[4], m[5]);
+	else
+		snprintf(buf, 24, "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x",
+			 m[0], m[1], m[2], m[3], m[4], m[5], m[6], m[7]);
+	return buf;
+}
+
 /**
  * Takes a string, unescapes all characters that are escaped as hex codes
  * (\x##) and turns them into the equivalent ascii-codes. Returns a string
diff --git a/src/util.h b/src/util.h
index 91ed8f2888..8201456b45 100644
--- a/src/util.h
+++ b/src/util.h
@@ -106,6 +106,7 @@ std::string get_escaped_string(const std::string& str, bool escape_all);
 
 extern char* copy_string(const char* s);
 extern int streq(const char* s1, const char* s2);
+extern char* format_mac(const unsigned char* m);
 
 // Returns the character corresponding to the given escape sequence (s points
 // just past the '\'), and updates s to point just beyond the last character

From 4b9d8b2c733abc336d7a565c98909f4e8dd6449e Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 22 Jul 2013 15:38:04 -0400
Subject: [PATCH 08/25] Tiny fix to account for missing str field  (not sure
 how this happens yet)

---
 scripts/policy/protocols/ssh/detect-bruteforcing.bro | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index f8d41db2b8..7e8ec2d911 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -55,9 +55,11 @@ event bro_init()
 	                  	{
 	                  	local r = result["ssh.login.failure"];
 	                  	local sub_msg = fmt("Sampled servers: ");
-	                  	for ( i in r$samples )
+	                  	local samples = r$samples;
+	                  	for ( i in samples )
 	                  		{
-	                  		sub_msg = fmt("%s%s %s", sub_msg, i==0 ? "":",", r$samples[i]$str);
+	                  		if ( samples[i]?$str )
+	                  			sub_msg = fmt("%s%s %s", sub_msg, i==0 ? "":",", samples[i]$str);
 	                  		}
 	                  	# Generate the notice.
 	                  	NOTICE([$note=Password_Guessing,

From 0e23a8bc9e24e535b2fff61f63232c6360c14638 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 30 Jul 2013 11:46:51 -0400
Subject: [PATCH 09/25] Beginning rework of SumStats API.

---
 scripts/base/frameworks/sumstats/cluster.bro  | 133 +++++++++---------
 scripts/base/frameworks/sumstats/main.bro     |  31 +---
 .../base/frameworks/sumstats/non-cluster.bro  |  11 +-
 3 files changed, 84 insertions(+), 91 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index a69dd1ab54..0c005d72a6 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -39,10 +39,10 @@ export {
 	## a single key value from a sumstat.  It's typically used to get intermediate
 	## updates before the break interval triggers to speed detection of a value
 	## crossing a threshold.
-	global cluster_key_request: event(uid: string, ss_name: string, key: Key, cleanup: bool);
+	global cluster_keys_request: event(uid: string, ss_name: string, key: set[Key], cleanup: bool);
 
 	## This event is sent by nodes in response to a
-	## :bro:id:`SumStats::cluster_key_request` event.
+	## :bro:id:`SumStats::cluster_keys_request` event.
 	global cluster_key_response: event(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool);
 
 	## This is sent by workers to indicate that they crossed the percent
@@ -89,68 +89,71 @@ function data_added(ss: SumStat, key: Key, result: Result)
 		}
 	}
 
-event SumStats::send_data(uid: string, ss_name: string, data: ResultTable, cleanup: bool)
+#event SumStats::send_data(uid: string, ss_name: string, data: ResultTable, cleanup: bool)
+#	{
+#	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
+#
+#	local local_data: ResultTable = table();
+#	local incoming_data: ResultTable = cleanup ? data : copy(data);
+#
+#	local num_added = 0;
+#	for ( key in incoming_data )
+#		{
+#		local_data[key] = incoming_data[key];
+#		delete incoming_data[key];
+#
+#		# Only send cluster_send_in_groups_of at a time.  Queue another
+#		# event to send the next group.
+#		if ( cluster_send_in_groups_of == ++num_added )
+#			break;
+#		}
+#
+#	local done = F;
+#	# If data is empty, this sumstat is done.
+#	if ( |incoming_data| == 0 )
+#		done = T;
+#
+#	# Note: copy is needed to compensate serialization caching issue. This should be
+#	# changed to something else later. 
+#	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done, cleanup);
+#	if ( ! done )
+#		schedule 0.01 sec { SumStats::send_data(uid, ss_name, incoming_data, T) };
+#	}
+
+#event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
+#	{
+#	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
+#
+#	# Initiate sending all of the data for the requested stats.
+#	if ( ss_name in result_store )
+#		event SumStats::send_data(uid, ss_name, result_store[ss_name], cleanup);
+#	else
+#		event SumStats::send_data(uid, ss_name, table(), cleanup);
+#
+#	# Lookup the actual sumstats and reset it, the reference to the data
+#	# currently stored will be maintained internally by the send_data event.
+#	if ( ss_name in stats_store && cleanup )
+#		reset(stats_store[ss_name]);
+#	}
+
+event SumStats::cluster_keys_request(uid: string, ss_name: string, keys: set[Key], cleanup: bool)
 	{
-	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
-
-	local local_data: ResultTable = table();
-	local incoming_data: ResultTable = cleanup ? data : copy(data);
-
-	local num_added = 0;
-	for ( key in incoming_data )
+	for ( key in keys )
 		{
-		local_data[key] = incoming_data[key];
-		delete incoming_data[key];
+		if ( ss_name in result_store && key in result_store[ss_name] )
+			{
+			#print fmt("WORKER %s: received the cluster_keys_request event for %s=%s.", Cluster::node, key2str(key), data);
 
-		# Only send cluster_send_in_groups_of at a time.  Queue another
-		# event to send the next group.
-		if ( cluster_send_in_groups_of == ++num_added )
-			break;
-		}
-
-	local done = F;
-	# If data is empty, this sumstat is done.
-	if ( |incoming_data| == 0 )
-		done = T;
-
-	# Note: copy is needed to compensate serialization caching issue. This should be
-	# changed to something else later. 
-	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done, cleanup);
-	if ( ! done )
-		schedule 0.01 sec { SumStats::send_data(uid, ss_name, incoming_data, T) };
-	}
-
-event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
-	{
-	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
-
-	# Initiate sending all of the data for the requested stats.
-	if ( ss_name in result_store )
-		event SumStats::send_data(uid, ss_name, result_store[ss_name], cleanup);
-	else
-		event SumStats::send_data(uid, ss_name, table(), cleanup);
-
-	# Lookup the actual sumstats and reset it, the reference to the data
-	# currently stored will be maintained internally by the send_data event.
-	if ( ss_name in stats_store && cleanup )
-		reset(stats_store[ss_name]);
-	}
-
-event SumStats::cluster_key_request(uid: string, ss_name: string, key: Key, cleanup: bool)
-	{
-	if ( ss_name in result_store && key in result_store[ss_name] )
-		{
-		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
-
-		# Note: copy is needed to compensate serialization caching issue. This should be
-		# changed to something else later. 
-		event SumStats::cluster_key_response(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
-		}
-	else
-		{
-		# We need to send an empty response if we don't have the data so that the manager
-		# can know that it heard back from all of the workers.
-		event SumStats::cluster_key_response(uid, ss_name, key, table(), cleanup);
+			# Note: copy is needed to compensate serialization caching issue. This should be
+			# changed to something else later. 
+			event SumStats::cluster_key_response(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
+			}
+		else
+			{
+			# We need to send an empty response if we don't have the data so that the manager
+			# can know that it heard back from all of the workers.
+			event SumStats::cluster_key_response(uid, ss_name, key, table(), cleanup);
+			}
 		}
 	}
 
@@ -252,6 +255,10 @@ event SumStats::cluster_key_response(uid: string, ss_name: string, key: Key, res
 			threshold_crossed(ss, key, ir);
 			event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
 			}
+		if ()
+			{
+			
+			}
 
 		if ( cleanup )
 			{
@@ -289,7 +296,7 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 
 	local uid = unique_id("");
 	done_with[uid] = 0;
-	event SumStats::cluster_key_request(uid, ss_name, key, T);
+	event SumStats::cluster_keys_request(uid, ss_name, set(key), T);
 	}
 
 event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)
@@ -378,7 +385,7 @@ function request_key(ss_name: string, key: Key): Result
 	done_with[uid] = 0;
 	key_requests[uid] = table();
 
-	event SumStats::cluster_key_request(uid, ss_name, key, F);
+	event SumStats::cluster_keys_request(uid, ss_name, set(key), F);
 	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
 		{
 		local result = key_requests[uid];
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index 3afa507c7a..8d5a668cde 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -74,10 +74,6 @@ export {
 	## Type to store results for multiple reducers.
 	type Result: table[string] of ResultVal;
 
-	## Type to store a table of sumstats results indexed
-	## by keys.
-	type ResultTable: table[Key] of Result;
-
 	## SumStats represent an aggregation of reducers along with
 	## mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
@@ -92,7 +88,7 @@ export {
 		name:               string;
 		
 		## The interval at which this filter should be "broken"
-		## and the '$epoch_finished' callback called.  The
+		## and the '$epoch_result' callback called.  The
 		## results are also reset at this time so any threshold
 		## based detection needs to be set to a
 		## value that should be expected to happen within
@@ -119,15 +115,10 @@ export {
 		## A callback that is called when a threshold is crossed.
 		threshold_crossed:  function(key: SumStats::Key, result: SumStats::Result) &optional;
 
-		## A callback with the full collection of Results for
-		## this SumStat.
-		epoch_finished:     function(rt: SumStats::ResultTable) &optional;
-		#epoch_finished:     function(num_keys: count) &optional;
-
 		## A callback that receives each of the results at the
 		## end of the analysis epoch.  The function will be 
 		## called once for each key.
-		#epoch_finished_result:     function(key::SumStats::Key, result: SumStats::Result) &optional;
+		epoch_result:       function(ts: time, key::SumStats::Key, result: SumStats::Result) &optional;
 	};
 
 	## Create a summary statistic.
@@ -144,17 +135,6 @@ export {
 	## obs: The data point to send into the stream.
 	global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
 
-	## Dynamically request a sumstat.  This function should be
-	## used sparingly and not as a replacement for the callbacks 
-	## from the :bro:see:`SumStat` record.  The function is only
-	## available for use within "when" statements as an asynchronous
-	## function.
-	##
-	## ss_name: SumState name.
-	##
-	## Returns: The result table for the requested sumstat.
-	global request: function(ss_name: string): ResultTable;
-
 	## Dynamically request a sumstat key.  This function should be
 	## used sparingly and not as a replacement for the callbacks 
 	## from the :bro:see:`SumStat` record.  The function is only
@@ -182,6 +162,9 @@ export {
 	global key2str: function(key: SumStats::Key): string;
 }
 
+# Type to store a table of sumstats results indexed by keys.
+type ResultTable: table[Key] of Result;
+
 # The function prototype for plugins to do calculations.
 type ObserveFunc: function(r: Reducer, val: double, data: Observation, rv: ResultVal);
 
@@ -423,7 +406,7 @@ function observe(id: string, key: Key, obs: Observation)
 
 		local ss = stats_store[r$ssname];
 
-		# If there is a threshold and no epoch_finished callback
+		# If there is a threshold and no epoch_result callback
 		# we don't need to continue counting since the data will
 		# never be accessed.  This was leading
 		# to some state management issues when measuring
@@ -431,7 +414,7 @@ function observe(id: string, key: Key, obs: Observation)
 		# NOTE: this optimization could need removed in the
 		#       future if on demand access is provided to the
 		#       SumStats results.
-		if ( ! ss?$epoch_finished &&
+		if ( ! ss?$epoch_result &&
 			 r$ssname in threshold_tracker &&
 		     ( ss?$threshold &&
 		       key in threshold_tracker[r$ssname] &&
diff --git a/scripts/base/frameworks/sumstats/non-cluster.bro b/scripts/base/frameworks/sumstats/non-cluster.bro
index 265261f1bd..b7e18bd55a 100644
--- a/scripts/base/frameworks/sumstats/non-cluster.bro
+++ b/scripts/base/frameworks/sumstats/non-cluster.bro
@@ -6,10 +6,13 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( ss$name in result_store )
 		{
-		local data = result_store[ss$name];
-		if ( ss?$epoch_finished )
-			ss$epoch_finished(data);
-
+		if ( ss?$epoch_result )
+			{
+			local data = result_store[ss$name];
+			# TODO: don't block here.
+			for ( key in data )
+				ss$epoch_result(network_time(), key, data[key]);
+			}
 		reset(ss);
 		}
 

From 0f1b0e75b719c2b5755e3d20fa7e42b2f65726e7 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Wed, 31 Jul 2013 16:32:20 -0400
Subject: [PATCH 10/25] DHCP:

 - Added the known-devices log, and the script to log DHCP hostname per MAC address,
 - Added DPD sig,
 - Removed some scripts following a discussion with Seth.
---
 scripts/base/protocols/dhcp/__load__.bro      |   4 +-
 scripts/base/protocols/dhcp/dpd.sig           |   5 +
 scripts/base/protocols/dhcp/main.bro          |  12 +-
 scripts/policy/misc/known-devices.bro         |  37 +++
 .../protocols/dhcp/handle_extra_msg_types.bro | 256 ------------------
 .../dhcp/known-devices-and-hostnames.bro      |  22 ++
 6 files changed, 71 insertions(+), 265 deletions(-)
 create mode 100644 scripts/base/protocols/dhcp/dpd.sig
 create mode 100644 scripts/policy/misc/known-devices.bro
 delete mode 100644 scripts/policy/protocols/dhcp/handle_extra_msg_types.bro
 create mode 100644 scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro

diff --git a/scripts/base/protocols/dhcp/__load__.bro b/scripts/base/protocols/dhcp/__load__.bro
index 0098b81a7a..c04423a855 100644
--- a/scripts/base/protocols/dhcp/__load__.bro
+++ b/scripts/base/protocols/dhcp/__load__.bro
@@ -1,2 +1,4 @@
 @load ./consts
-@load ./main
\ No newline at end of file
+@load ./main
+
+@load-sigs ./dpd.sig
diff --git a/scripts/base/protocols/dhcp/dpd.sig b/scripts/base/protocols/dhcp/dpd.sig
new file mode 100644
index 0000000000..010920e2d8
--- /dev/null
+++ b/scripts/base/protocols/dhcp/dpd.sig
@@ -0,0 +1,5 @@
+signature dhcp_cookie {
+  ip-proto == udp
+  payload /^.*\x63\x82\x53\x63/
+  enable "dhcp"
+}
\ No newline at end of file
diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro
index 9fc63df152..05491361ff 100644
--- a/scripts/base/protocols/dhcp/main.bro
+++ b/scripts/base/protocols/dhcp/main.bro
@@ -2,8 +2,8 @@
 ##! This script ignores large swaths of the protocol, since it is rather
 ##! noisy on most networks, and focuses on the end-result: assigned leases.
 ##! 
-##! To enable further analysis and log output for DHCP, see the optional
-##! scripts in the policy/protocols/dhcp directory.
+##! If you'd like to track known DHCP devices and to log the hostname
+##! supplied by the client, see policy/protocols/dhcp/known-devices.bro
 
 @load ./utils.bro
 
@@ -45,13 +45,13 @@ redef record connection += {
 const ports = { 67/udp, 68/udp };
 redef likely_server_ports += { 67/udp };
 
-event bro_init() &priority=5
+event bro_init()
 	{
 	Log::create_stream(DHCP::LOG, [$columns=Info, $ev=log_dhcp]);
 	Analyzer::register_for_ports(Analyzer::ANALYZER_DHCP, ports);
 	}
 
-event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=5
+event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string)
 	{
 	local info: Info;
 	info$ts          = network_time();
@@ -65,10 +65,6 @@ event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_lis
 		info$mac = msg$h_addr;
 	
 	c$dhcp = info;
-	}
 
-# We let policy scripts add stuff too, so we run this at a lower priority
-event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=1
-	{
 	Log::write(DHCP::LOG, c$dhcp);
 	}
diff --git a/scripts/policy/misc/known-devices.bro b/scripts/policy/misc/known-devices.bro
new file mode 100644
index 0000000000..4c3ed9009c
--- /dev/null
+++ b/scripts/policy/misc/known-devices.bro
@@ -0,0 +1,37 @@
+##! This script logs devices for which Bro has been able to determine the MAC
+##! address and logs the MAC address once per day (by default). The log that 
+##! is output provides an easy way to determine a count of the devices in use
+##! on a network per day.
+
+##! NOTE: This script will not generate any logs. Scripts such as
+##! policy/protocols/dhcp/known-devices-and-hostnames are needed.
+module Known;
+
+export {
+	## The known-hosts logging stream identifier.
+	redef enum Log::ID += { DEVICES_LOG };
+
+	## The record type which contains the column fields of the known-devices log.
+	type DevicesInfo: record {
+		## The timestamp at which the host was detected.
+		ts:      time &log;
+		## The MAC address that was detected.
+		mac:     string &log;
+	};
+	
+	## The set of all known MAC addresses to store for preventing duplicate 
+	## logging of addresses.  It can also be used from other scripts to 
+	## inspect if an address has been seen in use.
+	## Maintain the list of known devices for 24 hours so that the existence
+	## of each individual address is logged each day.
+	global known_devices: set[string] &create_expire=1day &synchronized &redef;
+
+	## An event that can be handled to access the :bro:type:`Known::DevicesInfo`
+	## record as it is sent on to the logging framework.
+	global log_known_devices: event(rec: DevicesInfo);
+}
+
+event bro_init()
+	{
+	Log::create_stream(Known::DEVICES_LOG, [$columns=DevicesInfo, $ev=log_known_devices]);
+	}
diff --git a/scripts/policy/protocols/dhcp/handle_extra_msg_types.bro b/scripts/policy/protocols/dhcp/handle_extra_msg_types.bro
deleted file mode 100644
index e5fa713da4..0000000000
--- a/scripts/policy/protocols/dhcp/handle_extra_msg_types.bro
+++ /dev/null
@@ -1,256 +0,0 @@
-##! Handlers for DHCP message types other than DHCPACK, which is handled in base/protocols/dhcp.
-##! For networks that wish to get more details from their DHCP logs, at the expense
-##! of a significantly higher log rate.
-
-@load base/protocols/dhcp
-
-module DHCP;
-
-export {
-	redef record Info += {
-		## The value of the host name option, if seen
-		host_name:	string		&log &optional;
-		## The IP requested by the client, if any
-		requested_ip:	addr		&log &optional;
-		## The type of the DHCP message (DHCPOFFER, DHCPRELEASE, etc.)
-		msg_type:	string		&log &optional;
-	};
-
-	#### Enabled by default
-	
-	## A boolean value to determine if DHCPREQUEST messages are logged.
-	## Often useful to see client activity, and because host_name is often available.
-	const log_dhcprequest = T &redef;
-
-	## A boolean value to determine if DHCPDECLINE messages are logged.
-	## A client declines a lease if it detects that the IP is already in use (usually via ARP).
-	const log_dhcpdecline = T &redef;
-
-	## A boolean value to determine if DHCPNAK messages are logged.
-	## A server issues a DHCPNAK if a client DHCPREQUEST is invalid.
-	const log_dhcpnak = T &redef;
-
-	## A boolean value to determine if DHCPRELEASE messages are logged.
-	## A client issues a DHCPRELEASE when it no longer needs the lease (e.g. it's shutting down).
-	const log_dhcprelease = T &redef;
-
-	#### Not enabled by default
-
-	## A boolean value to determine if DHCPOFFER messages are logged.
-	## Used to profile server -> client communication.
-	const log_dhcpoffer = F &redef;
-	
-	## A boolean value to determine if DHCPDISCOVER messages are logged.
-	## Used to profile broadcast client discovery requests.
-	const log_dhcpdiscover = F &redef;
-	
-	## A boolean value to determine if DHCPINFORM messages are logged.
-	## Used to profile clients attempting to request/renew specific IPs.
-	const log_dhcpinform = F &redef;
-
-}
-
-event dhcp_offer(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=5
-	{
-	if ( ! log_dhcpoffer )
-		return;
-	
-	local info: Info;
-	info$ts          = network_time();
-	info$id          = c$id;
-	info$uid         = c$uid;
-	info$assigned_ip = reverse_ip(msg$yiaddr);
-	info$lease_time  = lease;
-	info$trans_id    = msg$xid;
-	info$msg_type    = "DHCPOFFER";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_discover(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &priority=5
-	{
-	if ( ! log_dhcpdiscover )
-		return;
-	
-	local info: Info;
-	info$ts           = network_time();
-	info$id           = c$id;
-	info$uid          = c$uid;
-	info$requested_ip = req_addr;
-	info$trans_id     = msg$xid;
-	info$msg_type     = "DHCPDISCOVER";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &priority=5
-	{
-	if ( ! log_dhcprequest )
-		return;
-	
-	local info: Info;
-	info$ts           = network_time();
-	info$id           = c$id;
-	info$uid          = c$uid;
-	info$requested_ip = req_addr;
-	info$trans_id     = msg$xid;
-	info$msg_type     = "DHCPREQUEST";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_decline(c: connection, msg: dhcp_msg, host_name: string) &priority=5
-	{
-	if ( ! log_dhcpdecline )
-		return;
-	
-	local info: Info;
-	info$ts           = network_time();
-	info$id           = c$id;
-	info$uid          = c$uid;
-	info$trans_id     = msg$xid;
-	info$msg_type     = "DHCPDECLINE";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_nak(c: connection, msg: dhcp_msg, host_name: string) &priority=5
-	{
-	if ( ! log_dhcpnak )
-		return;
-	
-	local info: Info;
-	info$ts           = network_time();
-	info$id           = c$id;
-	info$uid          = c$uid;
-	info$trans_id     = msg$xid;
-	info$msg_type     = "DHCPNAK";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_release(c: connection, msg: dhcp_msg, host_name: string) &priority=5
-	{
-	if ( ! log_dhcprelease )
-		return;
-	
-	local info: Info;
-	info$ts           = network_time();
-	info$id           = c$id;
-	info$uid          = c$uid;
-	info$trans_id     = msg$xid;
-	info$msg_type     = "DHCPRELEASE";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_inform(c: connection, msg: dhcp_msg, host_name: string) &priority=5
-	{
-	if ( ! log_dhcpinform )
-		return;
-	
-	local info: Info;
-	info$ts           = network_time();
-	info$id           = c$id;
-	info$uid          = c$uid;
-	info$trans_id     = msg$xid;
-	info$msg_type     = "DHCPINFORM";
-
-	if ( msg$h_addr != "" )
-		info$mac = msg$h_addr;
-
-	if ( host_name != "" )
-		info$host_name = host_name;
-
-	c$dhcp = info;
-	}
-
-event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=4
-	{
-	## For the sake of consistency, let's add msg_type to DHCPACK as well.
-	c$dhcp$msg_type = "DHCPACK";
-	## host_name is generally not in ACKs, but let's check anyway.
-	if ( host_name != "" )
-		c$dhcp$host_name = host_name;
-	}
-
-#### We log stuff at a lower priority, in case any other scripts would like to extend the Info record first.
-
-event dhcp_offer(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &priority=1
-	{
-	if ( log_dhcpoffer )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
-event dhcp_discover(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &priority=1
-	{
-	if ( log_dhcpdiscover )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
-event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &priority=1
-	{
-	if ( log_dhcprequest )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
-event dhcp_decline(c: connection, msg: dhcp_msg, host_name: string) &priority=1
-	{
-	if ( log_dhcpdecline )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
-event dhcp_nak(c: connection, msg: dhcp_msg, host_name: string) &priority=1
-	{
-	if ( log_dhcpnak )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
-event dhcp_release(c: connection, msg: dhcp_msg, host_name: string) &priority=1
-	{
-	if ( log_dhcprelease )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
-event dhcp_inform(c: connection, msg: dhcp_msg, host_name: string) &priority=1
-	{
-	if ( log_dhcpinform )
-		Log::write(DHCP::LOG, c$dhcp);
-	}
-
diff --git a/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro b/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
new file mode 100644
index 0000000000..95770ce273
--- /dev/null
+++ b/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
@@ -0,0 +1,22 @@
+@load policy/misc/known-devices
+
+module Known;
+
+export {
+	redef record DevicesInfo += {
+		## The value of the DHCP host name option, if seen
+		dhcp_host_name:	string		&log &optional;
+	};
+}
+
+event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string)
+	{
+	if ( msg$h_addr == "" )
+		return;
+
+	if ( msg$h_addr !in known_devices )
+		{
+		add known_devices[msg$h_addr];
+		Log::write(Known::DEVICES_LOG, [$ts=network_time(), $mac=msg$h_addr, $dhcp_host_name=host_name]);
+		}	
+	}

From 1786a3b92d8bd4f9929decfcf74f59ab59d65334 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <grigorescu@gmail.com>
Date: Wed, 31 Jul 2013 17:30:56 -0400
Subject: [PATCH 11/25] DHCP: Adding unit tests.

---
 scripts/base/protocols/dhcp/main.bro             |   8 ++++++--
 .../dhcp/known-devices-and-hostnames.bro         |  12 ++++++++++++
 .../dhcp.log                                     |  10 ++++++++++
 .../scripts.base.protocols.dhcp.inform/dhcp.log  |  10 ++++++++++
 .../known_devices.log                            |  11 +++++++++++
 testing/btest/Traces/dhcp/dhcp.trace             | Bin 0 -> 3140 bytes
 testing/btest/Traces/dhcp/dhcp_inform.trace      | Bin 0 -> 687 bytes
 .../base/protocols/dhcp/dhcp-all-msg-types.btest |   6 ++++++
 .../scripts/base/protocols/dhcp/inform.test      |   5 +++++
 .../dhcp/known-devices-and-hostnames/basic.test  |   8 ++++++++
 10 files changed, 68 insertions(+), 2 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.dhcp.dhcp-all-msg-types/dhcp.log
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.dhcp.inform/dhcp.log
 create mode 100644 testing/btest/Baseline/scripts.policy.protocols.dhcp.known-devices-and-hostnames.basic/known_devices.log
 create mode 100644 testing/btest/Traces/dhcp/dhcp.trace
 create mode 100644 testing/btest/Traces/dhcp/dhcp_inform.trace
 create mode 100644 testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.btest
 create mode 100644 testing/btest/scripts/base/protocols/dhcp/inform.test
 create mode 100644 testing/btest/scripts/policy/protocols/dhcp/known-devices-and-hostnames/basic.test

diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro
index 05491361ff..07bd437579 100644
--- a/scripts/base/protocols/dhcp/main.bro
+++ b/scripts/base/protocols/dhcp/main.bro
@@ -57,13 +57,17 @@ event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, router: dhcp_router_lis
 	info$ts          = network_time();
 	info$id          = c$id;
 	info$uid         = c$uid;
-	info$assigned_ip = reverse_ip(msg$yiaddr);
 	info$lease_time  = lease;
 	info$trans_id    = msg$xid;
 
 	if ( msg$h_addr != "" )
 		info$mac = msg$h_addr;
-	
+
+	if ( reverse_ip(msg$yiaddr) != 0.0.0.0 )
+		info$assigned_ip = reverse_ip(msg$yiaddr);
+	else
+		info$assigned_ip = c$id$orig_h;
+
 	c$dhcp = info;
 
 	Log::write(DHCP::LOG, c$dhcp);
diff --git a/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro b/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
index 95770ce273..ddb058f2e0 100644
--- a/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
+++ b/scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
@@ -20,3 +20,15 @@ event dhcp_request(c: connection, msg: dhcp_msg, req_addr: addr, serv_addr: addr
 		Log::write(Known::DEVICES_LOG, [$ts=network_time(), $mac=msg$h_addr, $dhcp_host_name=host_name]);
 		}	
 	}
+
+event dhcp_inform(c: connection, msg: dhcp_msg, host_name: string)
+	{
+	if ( msg$h_addr == "" )
+		return;
+
+	if ( msg$h_addr !in known_devices )
+		{
+		add known_devices[msg$h_addr];
+		Log::write(Known::DEVICES_LOG, [$ts=network_time(), $mac=msg$h_addr, $dhcp_host_name=host_name]);
+		}	
+	}      
diff --git a/testing/btest/Baseline/scripts.base.protocols.dhcp.dhcp-all-msg-types/dhcp.log b/testing/btest/Baseline/scripts.base.protocols.dhcp.dhcp-all-msg-types/dhcp.log
new file mode 100644
index 0000000000..b52d455a4a
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.dhcp.dhcp-all-msg-types/dhcp.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dhcp
+#open	2013-07-31-21-00-49
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mac	assigned_ip	lease_time	trans_id
+#types	time	string	addr	port	addr	port	string	addr	interval	count
+1370200444.371332	nQcgTWjvg4c	128.2.6.189	68	128.2.6.152	67	90:b1:1c:99:49:29	128.2.6.189	900.000000	1984
+#close	2013-07-31-21-00-50
diff --git a/testing/btest/Baseline/scripts.base.protocols.dhcp.inform/dhcp.log b/testing/btest/Baseline/scripts.base.protocols.dhcp.inform/dhcp.log
new file mode 100644
index 0000000000..d8f626efe3
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.dhcp.inform/dhcp.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dhcp
+#open	2013-07-31-21-00-55
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mac	assigned_ip	lease_time	trans_id
+#types	time	string	addr	port	addr	port	string	addr	interval	count
+1374432420.191205	FrJExwHcSal	128.2.6.122	68	128.2.6.152	67	90:b1:1c:99:49:29	128.2.6.122	0.000000	2754407505
+#close	2013-07-31-21-00-55
diff --git a/testing/btest/Baseline/scripts.policy.protocols.dhcp.known-devices-and-hostnames.basic/known_devices.log b/testing/btest/Baseline/scripts.policy.protocols.dhcp.known-devices-and-hostnames.basic/known_devices.log
new file mode 100644
index 0000000000..91d37f8950
--- /dev/null
+++ b/testing/btest/Baseline/scripts.policy.protocols.dhcp.known-devices-and-hostnames.basic/known_devices.log
@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_devices
+#open	2013-07-31-21-27-41
+#fields	ts	mac	dhcp_host_name
+#types	time	string	string
+1370200443.344965	90:b1:1c:99:49:29	btest.is.cool
+1374432420.186878	90:b1:1c:99:49:29	(empty)
+#close	2013-07-31-21-27-41
diff --git a/testing/btest/Traces/dhcp/dhcp.trace b/testing/btest/Traces/dhcp/dhcp.trace
new file mode 100644
index 0000000000000000000000000000000000000000..aeb00a133f1f70d493c3152f7c1f3a7ea458c2dd
GIT binary patch
literal 3140
zcmca|c+)~A1{MYw`2U}Qff2~5nz=eK^&Ja?F^~ho|G{9wMwywOnj8$S42-HEWe$Rs
zCqOzuk_;{k&WwUa%NZHjfFkS%K;q~Cq90v!=<<`Bf|E@d8I4#DthjuLhc~GtwYWqt
zvsf=VKR@R`C<u2jvNFsB1{4TGJ<p@*!1o>)gaVA+Adfc)>V!5hvCRPDL<VOD7shgC
zMkcf%1j(%te#Q<`KRknw$&3XQ2*$vK$lSun0!mKIAQ473js_-1ZXo7j0kK4YSk##}
zFSVpRzbIQbIX^EgT?VA^zZMTn8c2(2a~ql)=^0w+ndupu+ps$pl_n)R2Iv5d$dCnM
zIXy-ue)i<tQoYoa(*M=Kh-!6aWv~QB5(tBn`hTRft_@A=hd>bkPwV0n?~S~)&J2kt
z>?s};kZ*li8Nz^p2*QwnWI&{N4^RL$2yXffifnK|f>OMRojfANH-MCY0Z0vqF+5W|
z8zdkt<cm@ZOM%5kYKlT)N=i{`aj`;Peu+Y2Sz=~RVp2{juslWsHNa3l*~v<B(R^pi
z2u!X<pu%A<deICD!j@Hpi)QVgphO5V0a7&o_dq06NUVV5hIi4-I*>(kEij^luCOx5
z0ZS7Q#weNvphfdtP;|hGW`;lPp(8J?v;40EhSmpFHj+b&r4kxibI^(!Fn4&CzwH0(
gfuUtz$VPIt|6~m`w2q9N(Bd#=VbnW(0DEZ)0CO2oasU7T

literal 0
HcmV?d00001

diff --git a/testing/btest/Traces/dhcp/dhcp_inform.trace b/testing/btest/Traces/dhcp/dhcp_inform.trace
new file mode 100644
index 0000000000000000000000000000000000000000..798ca84149dece44eaf6f1cb91874a5dba63d86a
GIT binary patch
literal 687
zcmca|c+)~A1{MYw`2U}Qff2}AqVp#3-%Tb4MIZ-+8F(}u_})+0C^OSjlY_yPfl-)&
z!GXa+Q0GVk6I&G!&tPz2aAx@b{177}8(25UM35Yah5?9v7;os)$xXq@ri>isjEw(5
z0rvD26GIp<us|5<RdBF@1Iz;&VEv%L0tbvUgA1d{rd>=}0&I=&Gj>q$49@^#HDdt<
P0wc?RU~&T{IN|{S|HnC|

literal 0
HcmV?d00001

diff --git a/testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.btest b/testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.btest
new file mode 100644
index 0000000000..752ab91780
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/dhcp/dhcp-all-msg-types.btest
@@ -0,0 +1,6 @@
+# This tests that DHCP leases are logged in dhcp.log
+# The trace has a message of each DHCP message type,
+# but only one lease should show up in the logs.
+
+# @TEST-EXEC: bro -r $TRACES/dhcp/dhcp.trace %INPUT
+# @TEST-EXEC: btest-diff dhcp.log
diff --git a/testing/btest/scripts/base/protocols/dhcp/inform.test b/testing/btest/scripts/base/protocols/dhcp/inform.test
new file mode 100644
index 0000000000..652fd1ae45
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/dhcp/inform.test
@@ -0,0 +1,5 @@
+# DHCPINFORM leases are special-cased in the code.
+# This tests that those leases are correctly logged.
+
+# @TEST-EXEC: bro -r $TRACES/dhcp/dhcp_inform.trace %INPUT
+# @TEST-EXEC: btest-diff dhcp.log
diff --git a/testing/btest/scripts/policy/protocols/dhcp/known-devices-and-hostnames/basic.test b/testing/btest/scripts/policy/protocols/dhcp/known-devices-and-hostnames/basic.test
new file mode 100644
index 0000000000..c2fcc1397c
--- /dev/null
+++ b/testing/btest/scripts/policy/protocols/dhcp/known-devices-and-hostnames/basic.test
@@ -0,0 +1,8 @@
+# This tests that the known_devices log is created,
+# that devices are logged by MAC address, and that
+# the DHCP hostname is added, if available.
+
+# @TEST-EXEC: bro -r $TRACES/dhcp/dhcp.trace -r $TRACES/dhcp/dhcp_inform.trace %INPUT
+# @TEST-EXEC: btest-diff known_devices.log
+
+@load policy/protocols/dhcp/known-devices-and-hostnames
\ No newline at end of file

From 4f8100774ca0b23af29a5b928e065b402e6e2cec Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 2 Aug 2013 12:44:33 -0400
Subject: [PATCH 12/25] Updates for SumStats API to deal with high memory
 stats.

  - The code is a mess and will need to be cleaned up, but the
    tests do pass.
---
 doc/scripts/DocSourcesList.cmake              |   8 +-
 scripts/base/frameworks/sumstats/cluster.bro  | 446 ++++++++++++------
 scripts/base/frameworks/sumstats/main.bro     |   7 +-
 .../base/frameworks/sumstats/non-cluster.bro  |   8 +-
 scripts/policy/misc/app-stats/main.bro        |  20 +-
 scripts/test-all-policy.bro                   |  10 +-
 .../manager-1..stdout                         |   2 +-
 .../manager-1..stdout                         |   5 -
 .../.stdout                                   |   3 -
 .../frameworks/sumstats/basic-cluster.bro     |  13 +-
 .../base/frameworks/sumstats/basic.bro        |   9 +-
 .../sumstats/cluster-intermediate-update.bro  |   8 +-
 .../frameworks/sumstats/on-demand-cluster.bro |  21 +-
 .../base/frameworks/sumstats/on-demand.bro    |  25 +-
 .../frameworks/sumstats/sample-cluster.bro    |  23 +-
 .../base/frameworks/sumstats/sample.bro       |  13 +-
 16 files changed, 391 insertions(+), 230 deletions(-)

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index 2efa45ef38..97150f84aa 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -203,7 +203,13 @@ rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/app-metrics.bro)
+rest_target(${psd} policy/misc/app-stats/main.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/detect-traceroute/main.bro)
 rest_target(${psd} policy/misc/load-balancing.bro)
diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index 0c005d72a6..d5c5bc440a 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -33,17 +33,17 @@ export {
 
 	## Event sent by nodes that are collecting sumstats after receiving a request for
 	## the sumstat from the manager.
-	global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool);
+	#global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool);
 
 	## This event is sent by the manager in a cluster to initiate the collection of
 	## a single key value from a sumstat.  It's typically used to get intermediate
 	## updates before the break interval triggers to speed detection of a value
 	## crossing a threshold.
-	global cluster_keys_request: event(uid: string, ss_name: string, key: set[Key], cleanup: bool);
+	global cluster_get_result: event(uid: string, ss_name: string, key: Key, cleanup: bool);
 
 	## This event is sent by nodes in response to a
-	## :bro:id:`SumStats::cluster_keys_request` event.
-	global cluster_key_response: event(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool);
+	## :bro:id:`SumStats::cluster_get_result` event.
+	global cluster_send_result: event(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool);
 
 	## This is sent by workers to indicate that they crossed the percent
 	## of the current threshold by the percentage defined globally in
@@ -53,14 +53,20 @@ export {
 	## This event is scheduled internally on workers to send result chunks.
 	global send_data: event(uid: string, ss_name: string, data: ResultTable, cleanup: bool);
 
+	global get_a_key: event(uid: string, ss_name: string);
+
+	global send_a_key: event(uid: string, ss_name: string, key: Key);
+	global send_no_key: event(uid: string, ss_name: string);
+
 	## This event is generated when a threshold is crossed.
 	global cluster_threshold_crossed: event(ss_name: string, key: SumStats::Key, thold_index: count);
 }
 
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request|threshold_crossed)/;
-redef Cluster::manager2worker_events += /SumStats::thresholds_reset/;
-redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
+redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|get_result|threshold_crossed)/;
+redef Cluster::manager2worker_events += /SumStats::(thresholds_reset|get_a_key)/;
+redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|send_result|key_intermediate_response)/;
+redef Cluster::worker2manager_events += /SumStats::(send_a_key|send_no_key)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # This variable is maintained to know what keys have recently sent as
@@ -69,6 +75,10 @@ redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_resp
 # an intermediate result has been received.
 global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
 
+# Result tables indexed on a uid that are currently being sent to the
+# manager.
+global sending_results: table[string] of ResultTable = table();
+
 # This is done on all non-manager node types in the event that a sumstat is
 # being collected somewhere other than a worker.
 function data_added(ss: SumStat, key: Key, result: Result)
@@ -89,7 +99,7 @@ function data_added(ss: SumStat, key: Key, result: Result)
 		}
 	}
 
-#event SumStats::send_data(uid: string, ss_name: string, data: ResultTable, cleanup: bool)
+#event SumStats::send_data(uid: string, ss_name: string, cleanup: bool)
 #	{
 #	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 #
@@ -117,42 +127,86 @@ function data_added(ss: SumStat, key: Key, result: Result)
 #	# changed to something else later. 
 #	event SumStats::cluster_ss_response(uid, ss_name, copy(local_data), done, cleanup);
 #	if ( ! done )
-#		schedule 0.01 sec { SumStats::send_data(uid, ss_name, incoming_data, T) };
+#		schedule 0.01 sec { SumStats::send_data(uid, T) };
 #	}
 
-#event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
-#	{
-#	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
-#
-#	# Initiate sending all of the data for the requested stats.
-#	if ( ss_name in result_store )
-#		event SumStats::send_data(uid, ss_name, result_store[ss_name], cleanup);
-#	else
-#		event SumStats::send_data(uid, ss_name, table(), cleanup);
-#
-#	# Lookup the actual sumstats and reset it, the reference to the data
-#	# currently stored will be maintained internally by the send_data event.
-#	if ( ss_name in stats_store && cleanup )
-#		reset(stats_store[ss_name]);
-#	}
-
-event SumStats::cluster_keys_request(uid: string, ss_name: string, keys: set[Key], cleanup: bool)
+event SumStats::get_a_key(uid: string, ss_name: string)
 	{
-	for ( key in keys )
+	if ( uid in sending_results )
 		{
-		if ( ss_name in result_store && key in result_store[ss_name] )
-			{
-			#print fmt("WORKER %s: received the cluster_keys_request event for %s=%s.", Cluster::node, key2str(key), data);
+		if ( |sending_results[uid]| == 0 )
+			event SumStats::send_no_key(uid, ss_name);
 
+		for ( key in sending_results[uid] )
+			{
+			event SumStats::send_a_key(uid, ss_name, key);
+			# break to only send one.
+			break;
+			}
+		}
+	else if ( ss_name in result_store && |result_store[ss_name]| > 0 )
+		{
+		if ( |result_store[ss_name]| == 0 )
+			event SumStats::send_no_key(uid, ss_name);
+
+		for ( key in result_store[ss_name] )
+			{
+			event SumStats::send_a_key(uid, ss_name, key);
+			# break to only send one.
+			break;
+			}
+		}
+	else
+		{
+		event SumStats::send_no_key(uid, ss_name);
+		}
+	}
+
+event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
+	{
+	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
+
+	# Create a back store for the result
+	sending_results[uid] = (ss_name in result_store) ? copy(result_store[ss_name]) : table();
+
+	# Lookup the actual sumstats and reset it, the reference to the data
+	# currently stored will be maintained internally from the 
+	# sending_results table.
+	if ( cleanup && ss_name in stats_store )
+		reset(stats_store[ss_name]);
+	}
+
+event SumStats::cluster_get_result(uid: string, ss_name: string, key: Key, cleanup: bool)
+	{
+	#print fmt("WORKER %s: received the cluster_get_result event for %s=%s.", Cluster::node, key2str(key), data);
+
+	if ( cleanup ) # data will implicitly be in sending_results (i know this isn't great)
+		{
+		if ( uid in sending_results && key in sending_results[uid] )
+			{
 			# Note: copy is needed to compensate serialization caching issue. This should be
 			# changed to something else later. 
-			event SumStats::cluster_key_response(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
+			event SumStats::cluster_send_result(uid, ss_name, key, copy(sending_results[uid][key]), cleanup);
+			delete sending_results[uid][key];
 			}
 		else
 			{
 			# We need to send an empty response if we don't have the data so that the manager
 			# can know that it heard back from all of the workers.
-			event SumStats::cluster_key_response(uid, ss_name, key, table(), cleanup);
+			event SumStats::cluster_send_result(uid, ss_name, key, table(), cleanup);
+			}
+		}
+	else 
+		{
+		if ( ss_name in result_store && key in result_store[ss_name] )
+			{
+			event SumStats::cluster_send_result(uid, ss_name, key, copy(result_store[ss_name][key]), cleanup);
+			}
+		else
+			{
+			# We need to send an empty response if we don't have the data so that the manager
+			# can know that it heard back from all of the workers.
+			event SumStats::cluster_send_result(uid, ss_name, key, table(), cleanup);
 			}
 		}
 	}
@@ -205,7 +259,7 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( network_time() > zero_time )
 		{
-		#print fmt("%.6f MANAGER: breaking %s sumstat for %s sumstat", network_time(), ss$name, ss$id);
+		#print fmt("%.6f MANAGER: breaking %s sumstat", network_time(), ss$name);
 		local uid = unique_id("");
 
 		if ( uid in stats_results )
@@ -214,6 +268,9 @@ event SumStats::finish_epoch(ss: SumStat)
 
 		# Request data from peers.
 		event SumStats::cluster_ss_request(uid, ss$name, T);
+
+		done_with[uid] = 0;
+		event SumStats::get_a_key(uid, ss$name);
 		}
 
 	# Schedule the next finish_epoch event.
@@ -231,8 +288,129 @@ function data_added(ss: SumStat, key: Key, result: Result)
 		}
 	}
 
-event SumStats::cluster_key_response(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool)
+function handle_end_of_result_collection(uid: string, ss_name: string, key: Key, cleanup: bool)
 	{
+	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
+	local ss = stats_store[ss_name];
+	local ir = key_requests[uid];
+	if ( check_thresholds(ss, key, ir, 1.0) )
+		{
+		threshold_crossed(ss, key, ir);
+		event SumStats::cluster_threshold_crossed(ss_name, key, threshold_tracker[ss_name][key]);
+		}
+
+	delete key_requests[uid];
+	delete done_with[uid];
+
+	if ( cleanup )
+		{
+		# This is done here because "cleanup" implicitly means
+		# it's the end of an epoch.
+		if ( ss?$epoch_result && |ir| > 0 )
+			{
+			local now = network_time();
+			ss$epoch_result(now, key, ir);
+			}
+
+		# Check that there is an outstanding view before subtracting.
+		# Global views only apply to non-dynamic requests. Dynamic
+		# requests must be serviced.
+		if ( outstanding_global_views[ss_name] > 0 )
+			--outstanding_global_views[ss_name];
+		}
+
+	if ( uid in stats_results )
+		delete stats_results[uid][key];
+	}
+
+function request_all_current_keys(uid: string, ss_name: string, cleanup: bool)
+	{
+	#print "request_all_current_keys";
+	if ( uid in stats_results && |stats_results[uid]| > 0 )
+		{
+		#print fmt("    -- %d remaining keys here", |stats_results[uid]|);
+		for ( key in stats_results[uid] )
+			{
+			done_with[uid] = 0;
+			event SumStats::cluster_get_result(uid, ss_name, key, cleanup);
+			when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+				{
+				handle_end_of_result_collection(uid, ss_name, key, cleanup);
+				request_all_current_keys(uid, ss_name, cleanup);
+				}
+			break; # only a single key
+			}
+		}
+	else
+		{
+		# Get more keys!  And this breaks us out of the evented loop.
+		done_with[uid] = 0;
+		event SumStats::get_a_key(uid, ss_name);
+		}
+	}
+
+event SumStats::send_no_key(uid: string, ss_name: string)
+	{
+	#print "send_no_key";
+	++done_with[uid];
+	if ( Cluster::worker_count == done_with[uid] )
+		{
+		delete done_with[uid];
+
+		if ( |stats_results[uid]| > 0 )
+			{
+			#print "we need more keys!";
+			# Now that we have a key from each worker, lets
+			# grab all of the results.
+			request_all_current_keys(uid, ss_name, T);
+			}
+		else
+			{
+			#print "we're out of keys!";
+			local ss = stats_store[ss_name];
+			if ( ss?$epoch_finished )
+				ss$epoch_finished(network_time());
+			}
+		}
+	}
+
+event SumStats::send_a_key(uid: string, ss_name: string, key: Key)
+	{
+	#print fmt("send_a_key %s", key);
+	if ( uid !in stats_results )
+		{
+		# no clue what happened here
+		return;
+		}
+
+	if ( key !in stats_results[uid] )
+		stats_results[uid][key] = table();
+
+	++done_with[uid];
+	if ( Cluster::worker_count == done_with[uid] )
+		{
+		delete done_with[uid];
+
+		if ( |stats_results[uid]| > 0 )
+			{
+			#print "we need more keys!";
+			# Now that we have a key from each worker, lets
+			# grab all of the results.
+			request_all_current_keys(uid, ss_name, T);
+			}
+		else
+			{
+			#print "we're out of keys!";
+			local ss = stats_store[ss_name];
+			if ( ss?$epoch_finished )
+				ss$epoch_finished(network_time());
+			}
+		}
+	}
+
+event SumStats::cluster_send_result(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool)
+	{
+	#print "cluster_send_result";
 	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
 
 	# We only want to try and do a value merge if there are actually measured datapoints
@@ -244,37 +422,6 @@ event SumStats::cluster_key_response(uid: string, ss_name: string, key: Key, res
 
 	# Mark that a worker is done.
 	++done_with[uid];
-
-	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
-	if ( Cluster::worker_count == done_with[uid] )
-		{
-		local ss = stats_store[ss_name];
-		local ir = key_requests[uid];
-		if ( check_thresholds(ss, key, ir, 1.0) )
-			{
-			threshold_crossed(ss, key, ir);
-			event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
-			}
-		if ()
-			{
-			
-			}
-
-		if ( cleanup )
-			{
-			# We only want to delete the data if this is a non dynamic 
-			# request because the dynamic requests use when statements
-			# and the data needs to remain available.
-			delete key_requests[uid];
-			delete done_with[uid];
-
-			# Check that there is an outstanding view before subtracting.
-			# Global views only apply to non-dynamic requests. Dynamic
-			# requests must be serviced.
-			if ( outstanding_global_views[ss_name] > 0 )
-				--outstanding_global_views[ss_name];
-			}
-		}
 	}
 
 # Managers handle intermediate updates here.
@@ -296,96 +443,103 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 
 	local uid = unique_id("");
 	done_with[uid] = 0;
-	event SumStats::cluster_keys_request(uid, ss_name, set(key), T);
+	event SumStats::cluster_get_result(uid, ss_name, key, T);
 	}
 
-event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)
-	{
-	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
+#event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)
+#	{
+#	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
+#
+#	# Mark another worker as being "done" for this uid.
+#	if ( done )
+#		++done_with[uid];
+#
+#	# We had better only be getting requests for stuff that exists.
+#	if ( ss_name !in stats_store )
+#		return;
+#
+#	if ( uid !in stats_results )
+#		stats_results[uid] = table();
+#
+#	local local_data = stats_results[uid];
+#	local ss = stats_store[ss_name];
+#
+#	for ( key in data )
+#		{
+#		if ( key in local_data )
+#			local_data[key] = compose_results(local_data[key], data[key]);
+#		else
+#			local_data[key] = data[key];
+#
+#		# If a stat is done being collected, thresholds for each key
+#		# need to be checked so we're doing it here to avoid doubly
+#		# iterating over each key.
+#		if ( Cluster::worker_count == done_with[uid] )
+#			{
+#			if ( check_thresholds(ss, key, local_data[key], 1.0) )
+#				{
+#				threshold_crossed(ss, key, local_data[key]);
+#				event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
+#				}
+#			}
+#		}
+#
+#	# If the data has been collected from all peers, we are done and ready to finish.
+#	if ( cleanup && Cluster::worker_count == done_with[uid] )
+#		{
+#		local now = network_time();
+#		if ( ss?$epoch_result )
+#			{
+#			for ( key in local_data )
+#				ss$epoch_result(now, key, local_data[key]);
+#			}
+#
+#		if ( ss?$epoch_finished )
+#			ss$epoch_finished(now);
+#
+#		# Clean up
+#		delete stats_results[uid];
+#		delete done_with[uid];
+#		reset(ss);
+#		}
+#	}
 
-	# Mark another worker as being "done" for this uid.
-	if ( done )
-		++done_with[uid];
-
-	# We had better only be getting requests for stuff that exists.
-	if ( ss_name !in stats_store )
-		return;
-
-	if ( uid !in stats_results )
-		stats_results[uid] = table();
-
-	local local_data = stats_results[uid];
-	local ss = stats_store[ss_name];
-
-	for ( key in data )
-		{
-		if ( key in local_data )
-			local_data[key] = compose_results(local_data[key], data[key]);
-		else
-			local_data[key] = data[key];
-
-		# If a stat is done being collected, thresholds for each key
-		# need to be checked so we're doing it here to avoid doubly
-		# iterating over each key.
-		if ( Cluster::worker_count == done_with[uid] )
-			{
-			if ( check_thresholds(ss, key, local_data[key], 1.0) )
-				{
-				threshold_crossed(ss, key, local_data[key]);
-				event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
-				}
-			}
-		}
-
-	# If the data has been collected from all peers, we are done and ready to finish.
-	if ( cleanup && Cluster::worker_count == done_with[uid] )
-		{
-		if ( ss?$epoch_finished )
-			ss$epoch_finished(local_data);
-
-		# Clean up
-		delete stats_results[uid];
-		delete done_with[uid];
-		reset(ss);
-		}
-	}
-
-function request(ss_name: string): ResultTable
-	{
-	# This only needs to be implemented this way for cluster compatibility.
-	local uid = unique_id("dyn-");
-	stats_results[uid] = table();
-	done_with[uid] = 0;
-	event SumStats::cluster_ss_request(uid, ss_name, F);
-
-	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
-		{
-		if ( uid in stats_results )
-			{
-			local ss_result = stats_results[uid];
-			# Clean up
-			delete stats_results[uid];
-			delete done_with[uid];
-			reset(stats_store[ss_name]);
-			return ss_result;
-			}
-		else
-			return table();
-		}
-	timeout 1.1min
-		{
-		Reporter::warning(fmt("Dynamic SumStat request for %s took longer than 1 minute and was automatically cancelled.", ss_name));
-		return table();
-		}
-	}
+#function request(ss_name: string): ResultTable
+#	{
+#	# This only needs to be implemented this way for cluster compatibility.
+#	local uid = unique_id("dyn-");
+#	stats_results[uid] = table();
+#	done_with[uid] = 0;
+#	event SumStats::cluster_ss_request(uid, ss_name, F);
+#
+#	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+#		{
+#		if ( uid in stats_results )
+#			{
+#			local ss_result = stats_results[uid];
+#			# Clean up
+#			delete stats_results[uid];
+#			delete done_with[uid];
+#			reset(stats_store[ss_name]);
+#			return ss_result;
+#			}
+#		else
+#			return table();
+#		}
+#	timeout 1.1min
+#		{
+#		Reporter::warning(fmt("Dynamic SumStat request for %s took longer than 1 minute and was automatically cancelled.", ss_name));
+#		return table();
+#		}
+#	}
 
 function request_key(ss_name: string, key: Key): Result
 	{
-	local uid = unique_id("dyn-");
+	local uid = unique_id("");
 	done_with[uid] = 0;
 	key_requests[uid] = table();
 
-	event SumStats::cluster_keys_request(uid, ss_name, set(key), F);
+	event SumStats::cluster_get_result(uid, ss_name, key, F);
 	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
 		{
 		local result = key_requests[uid];
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index 7a98783df8..282b03da6b 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -118,7 +118,12 @@ export {
 		## A callback that receives each of the results at the
 		## end of the analysis epoch.  The function will be 
 		## called once for each key.
-		epoch_result:       function(ts: time, key::SumStats::Key, result: SumStats::Result) &optional;
+		epoch_result:       function(ts: time, key: SumStats::Key, result: SumStats::Result) &optional;
+	
+		## A callback that will be called when a single collection 
+		## interval is completed.  The ts value will be the time of 
+		## when the collection started.
+		epoch_finished:     function(ts:time) &optional;
 	};
 
 	## Create a summary statistic.
diff --git a/scripts/base/frameworks/sumstats/non-cluster.bro b/scripts/base/frameworks/sumstats/non-cluster.bro
index b7e18bd55a..97e1817598 100644
--- a/scripts/base/frameworks/sumstats/non-cluster.bro
+++ b/scripts/base/frameworks/sumstats/non-cluster.bro
@@ -6,13 +6,19 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( ss$name in result_store )
 		{
+		local now = network_time();
+
 		if ( ss?$epoch_result )
 			{
 			local data = result_store[ss$name];
 			# TODO: don't block here.
 			for ( key in data )
-				ss$epoch_result(network_time(), key, data[key]);
+				ss$epoch_result(now, key, data[key]);
 			}
+
+		if ( ss?$epoch_finished )
+			ss$epoch_finished(now);
+
 		reset(ss);
 		}
 
diff --git a/scripts/policy/misc/app-stats/main.bro b/scripts/policy/misc/app-stats/main.bro
index e4a38c6893..24c9ac2ade 100644
--- a/scripts/policy/misc/app-stats/main.bro
+++ b/scripts/policy/misc/app-stats/main.bro
@@ -45,20 +45,16 @@ event bro_init() &priority=3
 	SumStats::create([$name="app-metrics",
 	                  $epoch=break_interval,
 	                  $reducers=set(r1, r2),
-	                  $epoch_finished(data: SumStats::ResultTable) =
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
 	                  	{
 	                  	local l: Info;
-	                  	l$ts = network_time();
-	                  	l$ts_delta = break_interval;
-	                  	for ( key in data )
-	                  		{
-	                  		local result = data[key];
-	                  		l$app        = key$str;
-	                  		l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
-	                  		l$hits       = result["apps.hits"]$num;
-	                  		l$uniq_hosts = result["apps.hits"]$unique;
-	                  		Log::write(LOG, l);
-	                  		}
+	                  	l$ts         = network_time();
+	                  	l$ts_delta   = break_interval;
+	                  	l$app        = key$str;
+	                  	l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
+	                  	l$hits       = result["apps.hits"]$num;
+	                  	l$uniq_hosts = result["apps.hits"]$unique;
+	                  	Log::write(LOG, l);
 	                  	}]);
 	}
 
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index dcf50b538e..f0900fda07 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -35,7 +35,15 @@
 @load integration/barnyard2/types.bro
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
-@load misc/app-metrics.bro
+@load misc/app-stats/__load__.bro
+@load misc/app-stats/main.bro
+@load misc/app-stats/plugins/__load__.bro
+@load misc/app-stats/plugins/facebook.bro
+@load misc/app-stats/plugins/gmail.bro
+@load misc/app-stats/plugins/google.bro
+@load misc/app-stats/plugins/netflix.bro
+@load misc/app-stats/plugins/pandora.bro
+@load misc/app-stats/plugins/youtube.bro
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro
diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout
index a5428dd3b7..810cdb0ae8 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout
@@ -1,3 +1,3 @@
 A test metric threshold was crossed with a value of: 101.0
-End of epoch handler was called
 101.0
+End of epoch handler was called
diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout
index ed14a1c753..0445fc68b2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand-cluster/manager-1..stdout
@@ -1,7 +1,2 @@
-Complete SumStat request
-    Host: 6.5.4.3 -> 1
-    Host: 10.10.10.10 -> 5
-    Host: 1.2.3.4 -> 169
-    Host: 7.2.1.5 -> 145
 SumStat key request
     Host: 7.2.1.5 -> 145
diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout
index 876c368eb3..7d62edb7f7 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.on-demand/.stdout
@@ -1,5 +1,2 @@
-Complete SumStat request
-    Host: 1.2.3.4 -> 42
-    Host: 4.3.2.1 -> 7
 Key request for 1.2.3.4
     Host: 1.2.3.4 -> 42
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
index 956c43a57b..2206673c3c 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
@@ -26,14 +26,13 @@ event bro_init() &priority=5
 	SumStats::create([$name="test",
 	                  $epoch=5secs,
 	                  $reducers=set(r1),
-	                  $epoch_finished(rt: SumStats::ResultTable) =
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["test"];
+	                  	print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
+	                  	},
+	                  $epoch_finished(ts: time) =
 	                  	{
-	                  	for ( key in rt )
-	                  		{
-	                  		local r = rt[key]["test"];
-	                  		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
-	                  		}
-
 	                  	terminate();
 	                  	}]);
 	}
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic.bro b/testing/btest/scripts/base/frameworks/sumstats/basic.bro
index 54160cbf54..906d69a6f3 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/basic.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic.bro
@@ -14,13 +14,10 @@ event bro_init() &priority=5
 	SumStats::create([$name="test",
 	                  $epoch=3secs,
 	                  $reducers=set(r1),
-	                  $epoch_finished(data: SumStats::ResultTable) = 
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	for ( key in data )
-	                  		{
-	                  		local r = data[key]["test.metric"];
-	                  		print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
-	                  		}
+	                  	local r = result["test.metric"];
+	                  	print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
 	                  	}
 		 ]);
 
diff --git a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
index 97986aeddf..4fb6b817d3 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
@@ -23,11 +23,13 @@ event bro_init() &priority=5
 	SumStats::create([$name="test",
 	                  $epoch=10secs,
 	                  $reducers=set(r1),
-	                  $epoch_finished(data: SumStats::ResultTable) = 
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	print result["test.metric"]$sum;
+	                  	},
+	                  $epoch_finished(ts: time) = 
 	                  	{
 	                  	print "End of epoch handler was called";
-	                  	for ( res in data ) 
-	                  		print data[res]["test.metric"]$sum;
 	                  	terminate();
 	                  	},
 	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
diff --git a/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro
index 29fcd14c64..48068d8cfe 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro
@@ -22,7 +22,7 @@ global n = 0;
 
 event bro_init() &priority=5
 	{
-	local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
+	local r1 = SumStats::Reducer($stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE));
 	SumStats::create([$name="test sumstat",
 	                  $epoch=1hr,
 	                  $reducers=set(r1)]);
@@ -61,23 +61,24 @@ event on_demand2()
 	when ( local result = SumStats::request_key("test sumstat", [$host=host]) )
 		{
 		print "SumStat key request";
-		print fmt("    Host: %s -> %.0f", host, result["test"]$sum);
+		if ( "test" in result )
+			print fmt("    Host: %s -> %.0f", host, result["test"]$sum);
 		terminate();
 		}
 	}
 
 event on_demand()
 	{
-	when ( local results = SumStats::request("test sumstat") )
-		{
-		print "Complete SumStat request";
-		print fmt("    Host: %s -> %.0f", 6.5.4.3, results[[$host=6.5.4.3]]["test"]$sum);
-		print fmt("    Host: %s -> %.0f", 10.10.10.10, results[[$host=10.10.10.10]]["test"]$sum);
-		print fmt("    Host: %s -> %.0f", 1.2.3.4, results[[$host=1.2.3.4]]["test"]$sum);
-		print fmt("    Host: %s -> %.0f", 7.2.1.5, results[[$host=7.2.1.5]]["test"]$sum);
+	#when ( local results = SumStats::request("test sumstat") )
+	#	{
+	#	print "Complete SumStat request";
+	#	print fmt("    Host: %s -> %.0f", 6.5.4.3, results[[$host=6.5.4.3]]["test"]$sum);
+	#	print fmt("    Host: %s -> %.0f", 10.10.10.10, results[[$host=10.10.10.10]]["test"]$sum);
+	#	print fmt("    Host: %s -> %.0f", 1.2.3.4, results[[$host=1.2.3.4]]["test"]$sum);
+	#	print fmt("    Host: %s -> %.0f", 7.2.1.5, results[[$host=7.2.1.5]]["test"]$sum);
 
 		event on_demand2();
-		}
+	#	}
 	}
 
 global peer_count = 0;
diff --git a/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro b/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro
index f93e1a72dc..78aba726ca 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/on-demand.bro
@@ -4,17 +4,18 @@
 redef exit_only_after_terminate=T;
 
 
-event on_demand()
-	{
-	when ( local results = SumStats::request("test") )
-		{
-		print "Complete SumStat request";
-		for ( key in results )
-			{
-			print fmt("    Host: %s -> %.0f", key$host, results[key]["test.reducer"]$sum);
-			}
-		}
-	}
+## Requesting a full sumstats resulttable is not supported yet.
+#event on_demand()
+#	{
+#	when ( local results = SumStats::request("test") )
+#		{
+#		print "Complete SumStat request";
+#		for ( key in results )
+#			{
+#			print fmt("    Host: %s -> %.0f", key$host, results[key]["test.reducer"]$sum);
+#			}
+#		}
+#	}
 
 event on_demand_key()
 	{
@@ -39,7 +40,7 @@ event bro_init() &priority=5
 	SumStats::observe("test.reducer", [$host=1.2.3.4], [$num=42]);
 	SumStats::observe("test.reducer", [$host=4.3.2.1], [$num=7]);
 
-	schedule 0.1 secs { on_demand() };
+	#schedule 0.1 secs { on_demand() };
 	schedule 1 secs { on_demand_key() };
 	}
 
diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro
index 794625c25b..1f2bab0229 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro
@@ -23,21 +23,18 @@ event bro_init() &priority=5
 	SumStats::create([$name="test",
 	                  $epoch=5secs,
 	                  $reducers=set(r1),
-	                  $epoch_finished(rt: SumStats::ResultTable) =
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	local hosts: vector of addr = vector(6.5.4.3, 10.10.10.10, 1.2.3.4, 7.2.1.5);
-	                  	for ( i in hosts )
-	                  		{
-	                  		local key = [$host=hosts[i]];
-	                  		local r = rt[key]["test"];
+	                  	local r = result["test"];
+	                  	print fmt("Host: %s  Sampled observations: %d", key$host, r$sample_elements);
+	                  	local sample_nums: vector of count = vector();
+	                  	for ( sample in r$samples ) 
+	                  		sample_nums[|sample_nums|] =r$samples[sample]$num;
 
-	                  		print fmt("Host: %s  Sampled observations: %d", key$host, r$sample_elements);
-	                  		local sample_nums: vector of count = vector();
-	                  		for ( sample in r$samples ) 
-	                  			sample_nums[|sample_nums|] =r$samples[sample]$num;
-
-	                  		print fmt("    %s", sort(sample_nums));
-	                  		}
+	                  	print fmt("    %s", sort(sample_nums));
+	                  	},
+                      $epoch_finished(ts: time) =
+                      	{
 	                  	terminate();
 	                  	}]);
 	}
diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample.bro b/testing/btest/scripts/base/frameworks/sumstats/sample.bro
index 595c1e710a..4ba395b463 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/sample.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/sample.bro
@@ -8,15 +8,12 @@ event bro_init() &priority=5
 	SumStats::create([$name="test",
 	                  $epoch=3secs,
 	                  $reducers=set(r1),
-	                  $epoch_finished(data: SumStats::ResultTable) = 
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	for ( key in data )
-	                  		{
-	                  		print key$host;
-	                  		local r = data[key]["test.metric"];
-	                  		print r$samples;
-	                  		print r$sample_elements;
-	                  		}
+	                  	print key$host;
+	                  	local r = result["test.metric"];
+	                  	print r$samples;
+	                  	print r$sample_elements;
 	                  	}]);
 
 	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=5]);

From 7b8073556e7ca04944b931e7be1498e1699ea737 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 2 Aug 2013 16:30:20 -0400
Subject: [PATCH 13/25] Fix the SumStats top-k plugin and test.

---
 .../base/frameworks/sumstats/plugins/topk.bro | 14 ++++---
 .../base/frameworks/sumstats/topk-cluster.bro | 39 +++++++++----------
 .../scripts/base/frameworks/sumstats/topk.bro | 35 +++++++----------
 3 files changed, 41 insertions(+), 47 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/plugins/topk.bro b/scripts/base/frameworks/sumstats/plugins/topk.bro
index 58f8168f5b..cb90af962e 100644
--- a/scripts/base/frameworks/sumstats/plugins/topk.bro
+++ b/scripts/base/frameworks/sumstats/plugins/topk.bro
@@ -18,18 +18,20 @@ export {
 
 }
 
+hook register_observe_plugins()
+	{
+	register_observe_plugin(TOPK, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
+		topk_add(rv$topk, obs);
+		});
+	}
+
 hook init_resultval_hook(r: Reducer, rv: ResultVal)
 	{
 	if ( TOPK in r$apply && ! rv?$topk )
 		rv$topk = topk_init(r$topk_size);
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
-	{
-	if ( TOPK in r$apply )
-		topk_add(rv$topk, obs);
-	}
-
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$topk )
diff --git a/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro
index 0ade38e86c..d26cee4244 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/topk-cluster.bro
@@ -23,27 +23,24 @@ event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test.metric", 
 	                               $apply=set(SumStats::TOPK)];
-	SumStats::create([$epoch=5secs,
-	                     $reducers=set(r1),
-	                     $epoch_finished(data: SumStats::ResultTable) = 
-	                     	{
-	                     	for ( key in data )
-	                     		{
-	                     		local r = data[key]["test.metric"];
-					
-					local s: vector of SumStats::Observation;
-					s = topk_get_top(r$topk, 5);
-				
-					print fmt("Top entries for key %s", key$str);
-					for ( element in s ) 
-						{
-						print fmt("Num: %d, count: %d, epsilon: %d", s[element]$num, topk_count(r$topk, s[element]), topk_epsilon(r$topk, s[element]));
-						}
-
-					terminate();
-	                     		}
-	                     	}
-		 ]);
+	SumStats::create([$name="topk-test", 
+	                  $epoch=5secs,
+	                  $reducers=set(r1),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["test.metric"];
+	                  	local s: vector of SumStats::Observation;
+	                  	s = topk_get_top(r$topk, 5);
+	                  	print fmt("Top entries for key %s", key$str);
+	                  	for ( element in s ) 
+	                  		{
+	                  		print fmt("Num: %d, count: %d, epsilon: %d", s[element]$num, topk_count(r$topk, s[element]), topk_epsilon(r$topk, s[element]));
+	                  		}
+	                  	},
+	                  $epoch_finished(ts: time) = 
+	                  	{
+	                  	terminate();
+	                  	}]);
 
 
 	}
diff --git a/testing/btest/scripts/base/frameworks/sumstats/topk.bro b/testing/btest/scripts/base/frameworks/sumstats/topk.bro
index 22a5af1bc7..99c301c669 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/topk.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/topk.bro
@@ -5,26 +5,21 @@ event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test.metric", 
 	                               $apply=set(SumStats::TOPK)];
-	SumStats::create([$epoch=3secs,
-	                     $reducers=set(r1),
-	                     $epoch_finished(data: SumStats::ResultTable) = 
-	                     	{
-	                     	for ( key in data )
-	                     		{
-	                     		local r = data[key]["test.metric"];
-					
-					local s: vector of SumStats::Observation;
-					s = topk_get_top(r$topk, 5);
-				
-					print fmt("Top entries for key %s", key$str);
-					for ( element in s ) 
-						{
-						print fmt("Num: %d, count: %d, epsilon: %d", s[element]$num, topk_count(r$topk, s[element]), topk_epsilon(r$topk, s[element]));
-						}
-
-	                     		}
-	                     	}
-		 ]);
+	SumStats::create([$name="topk-test",
+	                  $epoch=3secs,
+	                  $reducers=set(r1),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["test.metric"];
+	                  	local s: vector of SumStats::Observation;
+	                  	s = topk_get_top(r$topk, 5);
+	                  	
+	                  	print fmt("Top entries for key %s", key$str);
+	                  	for ( element in s ) 
+	                  		{
+	                  		print fmt("Num: %d, count: %d, epsilon: %d", s[element]$num, topk_count(r$topk, s[element]), topk_epsilon(r$topk, s[element]));
+	                  		}
+	                  	}]);
 
 
 	const loop_v: vector of count = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100};

From 135094428e9992d42b1208e3f1181775e7d4e22f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 2 Aug 2013 16:30:34 -0400
Subject: [PATCH 14/25] Hopefully fix the SumStats cluster support.

---
 scripts/base/frameworks/sumstats/cluster.bro | 104 +++++++++++--------
 1 file changed, 58 insertions(+), 46 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index d5c5bc440a..b13fff0431 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -10,10 +10,6 @@
 module SumStats;
 
 export {
-	## Allows a user to decide how large of result groups the workers should transmit
-	## values for cluster stats aggregation.
-	const cluster_send_in_groups_of = 50 &redef;
-
 	## The percent of the full threshold value that needs to be met on a single worker
 	## for that worker to send the value to its manager in order for it to request a
 	## global view for that value.  There is no requirement that the manager requests
@@ -53,7 +49,7 @@ export {
 	## This event is scheduled internally on workers to send result chunks.
 	global send_data: event(uid: string, ss_name: string, data: ResultTable, cleanup: bool);
 
-	global get_a_key: event(uid: string, ss_name: string);
+	global get_a_key: event(uid: string, ss_name: string, cleanup: bool &default=F);
 
 	global send_a_key: event(uid: string, ss_name: string, key: Key);
 	global send_no_key: event(uid: string, ss_name: string);
@@ -130,30 +126,38 @@ function data_added(ss: SumStat, key: Key, result: Result)
 #		schedule 0.01 sec { SumStats::send_data(uid, T) };
 #	}
 
-event SumStats::get_a_key(uid: string, ss_name: string)
+event SumStats::get_a_key(uid: string, ss_name: string, cleanup: bool)
 	{
 	if ( uid in sending_results )
 		{
 		if ( |sending_results[uid]| == 0 )
-			event SumStats::send_no_key(uid, ss_name);
-
-		for ( key in sending_results[uid] )
 			{
-			event SumStats::send_a_key(uid, ss_name, key);
-			# break to only send one.
-			break;
+			event SumStats::send_no_key(uid, ss_name);
+			}
+		else
+			{
+			for ( key in sending_results[uid] )
+				{
+				event SumStats::send_a_key(uid, ss_name, key);
+				# break to only send one.
+				break;
+				}
 			}
 		}
-	else if ( ss_name in result_store && |result_store[ss_name]| > 0 )
+	else if ( !cleanup && ss_name in result_store && |result_store[ss_name]| > 0 )
 		{
 		if ( |result_store[ss_name]| == 0 )
-			event SumStats::send_no_key(uid, ss_name);
-
-		for ( key in result_store[ss_name] )
 			{
-			event SumStats::send_a_key(uid, ss_name, key);
-			# break to only send one.
-			break;
+			event SumStats::send_no_key(uid, ss_name);
+			}
+		else
+			{
+			for ( key in result_store[ss_name] )
+				{
+				event SumStats::send_a_key(uid, ss_name, key);
+				# break to only send one.
+				break;
+				}
 			}
 		}
 	else
@@ -232,7 +236,7 @@ event SumStats::thresholds_reset(ss_name: string)
 # This variable is maintained by manager nodes as they collect and aggregate
 # results.
 # Index on a uid.
-global stats_results: table[string] of ResultTable &create_expire=1min;
+global stats_keys: table[string] of set[Key] &create_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid
@@ -262,15 +266,16 @@ event SumStats::finish_epoch(ss: SumStat)
 		#print fmt("%.6f MANAGER: breaking %s sumstat", network_time(), ss$name);
 		local uid = unique_id("");
 
-		if ( uid in stats_results )
-			delete stats_results[uid];
-		stats_results[uid] = table();
+		if ( uid in stats_keys )
+			delete stats_keys[uid];
+		stats_keys[uid] = set();
 
 		# Request data from peers.
 		event SumStats::cluster_ss_request(uid, ss$name, T);
 
 		done_with[uid] = 0;
-		event SumStats::get_a_key(uid, ss$name);
+		#print fmt("get_key by uid: %s", uid);
+		event SumStats::get_a_key(uid, ss$name, T);
 		}
 
 	# Schedule the next finish_epoch event.
@@ -299,9 +304,6 @@ function handle_end_of_result_collection(uid: string, ss_name: string, key: Key,
 		event SumStats::cluster_threshold_crossed(ss_name, key, threshold_tracker[ss_name][key]);
 		}
 
-	delete key_requests[uid];
-	delete done_with[uid];
-
 	if ( cleanup )
 		{
 		# This is done here because "cleanup" implicitly means
@@ -319,25 +321,27 @@ function handle_end_of_result_collection(uid: string, ss_name: string, key: Key,
 			--outstanding_global_views[ss_name];
 		}
 
-	if ( uid in stats_results )
-		delete stats_results[uid][key];
+	delete key_requests[uid];
+	delete done_with[uid];
 	}
 
 function request_all_current_keys(uid: string, ss_name: string, cleanup: bool)
 	{
 	#print "request_all_current_keys";
-	if ( uid in stats_results && |stats_results[uid]| > 0 )
+	if ( uid in stats_keys && |stats_keys[uid]| > 0 )
 		{
-		#print fmt("    -- %d remaining keys here", |stats_results[uid]|);
-		for ( key in stats_results[uid] )
+		#print fmt("    -- %d remaining keys here", |stats_keys[uid]|);
+		for ( key in stats_keys[uid] )
 			{
 			done_with[uid] = 0;
 			event SumStats::cluster_get_result(uid, ss_name, key, cleanup);
 			when ( uid in done_with && Cluster::worker_count == done_with[uid] )
 				{
+				#print "done getting result";
 				handle_end_of_result_collection(uid, ss_name, key, cleanup);
 				request_all_current_keys(uid, ss_name, cleanup);
 				}
+			delete stats_keys[uid][key];
 			break; # only a single key
 			}
 		}
@@ -345,7 +349,8 @@ function request_all_current_keys(uid: string, ss_name: string, cleanup: bool)
 		{
 		# Get more keys!  And this breaks us out of the evented loop.
 		done_with[uid] = 0;
-		event SumStats::get_a_key(uid, ss_name);
+		#print fmt("get_key by uid: %s", uid);
+		event SumStats::get_a_key(uid, ss_name, cleanup);
 		}
 	}
 
@@ -357,7 +362,7 @@ event SumStats::send_no_key(uid: string, ss_name: string)
 		{
 		delete done_with[uid];
 
-		if ( |stats_results[uid]| > 0 )
+		if ( |stats_keys[uid]| > 0 )
 			{
 			#print "we need more keys!";
 			# Now that we have a key from each worker, lets
@@ -377,21 +382,21 @@ event SumStats::send_no_key(uid: string, ss_name: string)
 event SumStats::send_a_key(uid: string, ss_name: string, key: Key)
 	{
 	#print fmt("send_a_key %s", key);
-	if ( uid !in stats_results )
+	if ( uid !in stats_keys )
 		{
 		# no clue what happened here
 		return;
 		}
 
-	if ( key !in stats_results[uid] )
-		stats_results[uid][key] = table();
+	if ( key !in stats_keys[uid] )
+		add stats_keys[uid][key];
 
 	++done_with[uid];
 	if ( Cluster::worker_count == done_with[uid] )
 		{
 		delete done_with[uid];
 
-		if ( |stats_results[uid]| > 0 )
+		if ( |stats_keys[uid]| > 0 )
 			{
 			#print "we need more keys!";
 			# Now that we have a key from each worker, lets
@@ -422,6 +427,12 @@ event SumStats::cluster_send_result(uid: string, ss_name: string, key: Key, resu
 
 	# Mark that a worker is done.
 	++done_with[uid];
+
+	#if ( Cluster::worker_count == done_with[uid] )
+	#	{
+	#	print "done";
+	#	handle_end_of_result_collection(uid, ss_name, key, cleanup);
+	#	}
 	}
 
 # Managers handle intermediate updates here.
@@ -458,10 +469,10 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 #	if ( ss_name !in stats_store )
 #		return;
 #
-#	if ( uid !in stats_results )
-#		stats_results[uid] = table();
+#	if ( uid !in stats_keys )
+#		stats_keys[uid] = table();
 #
-#	local local_data = stats_results[uid];
+#	local local_data = stats_keys[uid];
 #	local ss = stats_store[ss_name];
 #
 #	for ( key in data )
@@ -498,7 +509,7 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 #			ss$epoch_finished(now);
 #
 #		# Clean up
-#		delete stats_results[uid];
+#		delete stats_keys[uid];
 #		delete done_with[uid];
 #		reset(ss);
 #		}
@@ -508,17 +519,17 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 #	{
 #	# This only needs to be implemented this way for cluster compatibility.
 #	local uid = unique_id("dyn-");
-#	stats_results[uid] = table();
+#	stats_keys[uid] = table();
 #	done_with[uid] = 0;
 #	event SumStats::cluster_ss_request(uid, ss_name, F);
 #
 #	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
 #		{
-#		if ( uid in stats_results )
+#		if ( uid in stats_keys )
 #			{
-#			local ss_result = stats_results[uid];
+#			local ss_result = stats_keys[uid];
 #			# Clean up
-#			delete stats_results[uid];
+#			delete stats_keys[uid];
 #			delete done_with[uid];
 #			reset(stats_store[ss_name]);
 #			return ss_result;
@@ -542,6 +553,7 @@ function request_key(ss_name: string, key: Key): Result
 	event SumStats::cluster_get_result(uid, ss_name, key, F);
 	return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
 		{
+		#print "done with request_key";
 		local result = key_requests[uid];
 		# Clean up
 		delete key_requests[uid];

From 6b58ef12d7f917cacb5a29a7c0fd7fda6ad41267 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 2 Aug 2013 16:53:56 -0400
Subject: [PATCH 15/25] Still fixing bugs in sumstats updated api cluster
 support.

---
 scripts/base/frameworks/sumstats/cluster.bro | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index b13fff0431..d2db09f312 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -454,7 +454,16 @@ event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
 
 	local uid = unique_id("");
 	done_with[uid] = 0;
-	event SumStats::cluster_get_result(uid, ss_name, key, T);
+	event SumStats::cluster_get_result(uid, ss_name, key, F);
+	when ( uid in done_with && Cluster::worker_count == done_with[uid] )
+		{
+		handle_end_of_result_collection(uid, ss_name, key, F);
+		}
+	timeout 1.1min
+		{
+		Reporter::warning(fmt("Dynamic SumStat intermediate key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
+		}
+
 	}
 
 #event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool, cleanup: bool)

From 1531980f3d2f2a6f55c7c58c222b0853f90ea2a1 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@illinois.edu>
Date: Fri, 2 Aug 2013 17:09:17 -0500
Subject: [PATCH 16/25] Exec module and raw input reader fixes.

- Do stream mode for commands done by exec module, it seems important
  in some cases (e.g. ensure requested stdin is fully written).

- For cases where the raw input reader knows the child process has been
  reaped, set the childpid member to a sentinel value to indicate such
  so we don't later think we should kill it or wait on it anymore.

- More error checking on dup2/close calls.  Set sentinel values when
  closing ends of pipes to prevent double closing a fd.

- Signal flag not set when raw input reader's child exits as a result
  of a signal.  Left out a test for this -- might be portability issues
  (e.g. Ubuntu seems to do things different regarding the exit code and
  also is printing "Killed" to stderr where other platforms don't).
---
 scripts/base/utils/dir.bro                    |  2 +-
 scripts/base/utils/exec.bro                   |  1 +
 src/input/readers/Raw.cc                      | 39 ++++++++++++-------
 .../scripts.base.utils.exec/bro..stdout       |  1 -
 .../frameworks/input/raw/executestdin.bro     |  1 -
 .../btest/scripts/base/utils/active-http.test |  3 +-
 testing/btest/scripts/base/utils/dir.test     | 14 +++++--
 testing/btest/scripts/base/utils/exec.test    |  9 +++--
 8 files changed, 42 insertions(+), 28 deletions(-)

diff --git a/scripts/base/utils/dir.bro b/scripts/base/utils/dir.bro
index 4f3ee94945..1ade4a47f7 100644
--- a/scripts/base/utils/dir.bro
+++ b/scripts/base/utils/dir.bro
@@ -28,7 +28,7 @@ event Dir::monitor_ev(dir: string, last_files: set[string],
                       callback: function(fname: string),
                       poll_interval: interval)
 	{
-	when ( local result = Exec::run([$cmd=fmt("ls -i \"%s/\"", str_shell_escape(dir))]) )
+	when ( local result = Exec::run([$cmd=fmt("ls -i -1 \"%s/\"", str_shell_escape(dir))]) )
 		{
 		if ( result$exit_code != 0 )
 			{
diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro
index 732bbcf34c..d505b424c7 100644
--- a/scripts/base/utils/exec.bro
+++ b/scripts/base/utils/exec.bro
@@ -163,6 +163,7 @@ function run(cmd: Command): Result
 	Input::add_event([$name=cmd$uid,
 	                  $source=fmt("%s |", cmd$cmd),
 	                  $reader=Input::READER_RAW,
+	                  $mode=Input::STREAM,
 	                  $fields=Exec::OneLine,
 	                  $ev=Exec::line,
 	                  $want_record=F,
diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc
index 2820923a25..0f4c4ca7d1 100644
--- a/src/input/readers/Raw.cc
+++ b/src/input/readers/Raw.cc
@@ -95,29 +95,32 @@ bool Raw::Execute()
 	else if ( childpid == 0 )
 		{
 		// we are the child.
-		close(pipes[stdout_in]);
-		dup2(pipes[stdout_out], stdout_fileno);
+		safe_close(pipes[stdout_in]);
+		if ( dup2(pipes[stdout_out], stdout_fileno) == -1 )
+			Error(Fmt("Error on dup2 stdout_out: %d", errno));
 
 		if ( stdin_towrite )
 			{
-			close(pipes[stdin_out]);
-			dup2(pipes[stdin_in], stdin_fileno);
+			safe_close(pipes[stdin_out]);
+			if ( dup2(pipes[stdin_in], stdin_fileno) == -1 )
+				Error(Fmt("Error on dup2 stdin_in: %d", errno));
 			}
 
 		if ( use_stderr )
 			{
-			close(pipes[stderr_in]);
-			dup2(pipes[stderr_out], stderr_fileno);
+			safe_close(pipes[stderr_in]);
+			if ( dup2(pipes[stderr_out], stderr_fileno) == -1 )
+				Error(Fmt("Error on dup2 stderr_out: %d", errno));
 			}
 
-		execl("/bin/sh", "sh", "-c", fname.c_str(), NULL);
+		execl("/bin/sh", "sh", "-c", fname.c_str(), (char*) NULL);
 		fprintf(stderr, "Exec failed :(......\n");
 		exit(255);
 		}
 	else
 		{
 		// we are the parent
-		close(pipes[stdout_out]);
+		safe_close(pipes[stdout_out]);
 		pipes[stdout_out] = -1;
 
 		if ( Info().mode == MODE_STREAM )
@@ -125,7 +128,7 @@ bool Raw::Execute()
 
 		if ( stdin_towrite )
 			{
-			close(pipes[stdin_in]);
+			safe_close(pipes[stdin_in]);
 			pipes[stdin_in] = -1;
 			fcntl(pipes[stdin_out], F_SETFL, O_NONBLOCK); // ya, just always set this to nonblocking. we do not want to block on a program receiving data.
 			// note that there is a small gotcha with it. More data is queued when more data is read from the program output. Hence, when having
@@ -134,7 +137,7 @@ bool Raw::Execute()
 
 		if ( use_stderr )
 			{
-			close(pipes[stderr_out]);
+			safe_close(pipes[stderr_out]);
 			pipes[stderr_out] = -1;
 			fcntl(pipes[stderr_in], F_SETFL, O_NONBLOCK); // true for this too.
 			}
@@ -195,7 +198,10 @@ bool Raw::CloseInput()
 		{
 		for ( int i = 0; i < 6; i ++ )
 			if ( pipes[i] != -1 )
-				close(pipes[i]);
+				{
+				safe_close(pipes[i]);
+				pipes[i] = -1;
+				}
 		}
 
 	file = 0;
@@ -393,11 +399,13 @@ void Raw::WriteToStdin()
 		{
 		Error(Fmt("Writing to child process stdin failed: %d. Stopping writing at position %d", errno, pos));
 		stdin_towrite = 0;
-		close(pipes[stdin_out]);
 		}
 
 	if ( stdin_towrite == 0 ) // send EOF when we are done.
-		close(pipes[stdin_out]);
+		{
+		safe_close(pipes[stdin_out]);
+		pipes[stdin_out] = -1;
+		}
 
 	if ( Info().mode == MODE_MANUAL && stdin_towrite != 0 )
 		{
@@ -528,6 +536,7 @@ bool Raw::DoUpdate()
 	if ( childpid != -1 && waitpid(childpid, &return_code, WNOHANG) != 0 )
 		{
 		// child died
+		childpid = -1;
 		bool signal = false;
 		int code = 0;
 		if ( WIFEXITED(return_code) )
@@ -539,7 +548,7 @@ bool Raw::DoUpdate()
 
 		else if ( WIFSIGNALED(return_code) )
 			{
-			signal = false;
+			signal = true;
 			code = WTERMSIG(return_code);
 			Error(Fmt("Child process exited due to signal %d", code));
 			}
@@ -564,7 +573,7 @@ bool Raw::DoUpdate()
 			EndCurrentSend();
 
 		SendEvent("InputRaw::process_finished", 4, vals);
-	}
+		}
 
 
 
diff --git a/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout b/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout
index 5352d15d18..3cfdaafb4c 100644
--- a/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout
+++ b/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout
@@ -3,5 +3,4 @@ test1, [exit_code=0, signal_exit=F, stdout=[done, exit, stop], stderr=<uninitial
 [out2] = [insert more text here, and there]
 }]
 test2, [exit_code=1, signal_exit=F, stdout=[here's something on stdout, some more stdout, last stdout], stderr=[and some stderr, more stderr, last stderr], files=<uninitialized>]
-test3, [exit_code=9, signal_exit=F, stdout=[FML], stderr=<uninitialized>, files=<uninitialized>]
 test4, [exit_code=0, signal_exit=F, stdout=[hibye], stderr=<uninitialized>, files=<uninitialized>]
diff --git a/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro b/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro
index 729844e4b4..f6513dc6aa 100644
--- a/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro
+++ b/testing/btest/scripts/base/frameworks/input/raw/executestdin.bro
@@ -39,6 +39,5 @@ event bro_init()
 	try = 0;
 	outfile = open("../out");
 	Input::add_event([$source="cat > ../test.txt |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
-	Input::remove("input");
 	Input::add_event([$source="cat |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input2", $fields=Val, $ev=line, $want_record=F, $config=config_strings]);
 	}
diff --git a/testing/btest/scripts/base/utils/active-http.test b/testing/btest/scripts/base/utils/active-http.test
index 127b21d77e..f547e7dd15 100644
--- a/testing/btest/scripts/base/utils/active-http.test
+++ b/testing/btest/scripts/base/utils/active-http.test
@@ -1,4 +1,3 @@
-# @TEST-REQUIRES: which httpd
 # @TEST-REQUIRES: which python
 #
 # @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 1
@@ -8,7 +7,7 @@
 # @TEST-EXEC: btest-diff bro/.stdout
 
 @load base/utils/active-http
-
+@load base/frameworks/communication # let network-time run. otherwise there are no heartbeats...
 redef exit_only_after_terminate = T;
 
 event bro_init()
diff --git a/testing/btest/scripts/base/utils/dir.test b/testing/btest/scripts/base/utils/dir.test
index 44fee3860f..aa9ee62315 100644
--- a/testing/btest/scripts/base/utils/dir.test
+++ b/testing/btest/scripts/base/utils/dir.test
@@ -1,11 +1,11 @@
 # @TEST-EXEC: btest-bg-run bro bro -b ../dirtest.bro
-# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout
 
 @TEST-START-FILE dirtest.bro
 
 @load base/utils/dir
-
+@load base/frameworks/communication # let network-time run. otherwise there are no heartbeats...
 redef exit_only_after_terminate = T;
 
 global c: count = 0;
@@ -33,14 +33,20 @@ function new_file2(fname: string)
 event change_things()
 	{
 	system("touch ../testdir/newone");
-	system("rm ../testdir/bye && touch ../testdir/bye");
+	system("rm ../testdir/bye");
+	}
+
+event change_things2()
+	{
+	system("touch ../testdir/bye");
 	}
 
 event bro_init()
 	{
 	Dir::monitor("../testdir", new_file1, .5sec);
 	Dir::monitor("../testdir", new_file2, 1sec);
-	schedule 1sec { change_things() };
+	schedule 3sec { change_things() };
+	schedule 6sec { change_things2() };
 	}
 
 @TEST-END-FILE
diff --git a/testing/btest/scripts/base/utils/exec.test b/testing/btest/scripts/base/utils/exec.test
index 8876f0f49b..33ba10f97a 100644
--- a/testing/btest/scripts/base/utils/exec.test
+++ b/testing/btest/scripts/base/utils/exec.test
@@ -1,11 +1,11 @@
 # @TEST-EXEC: btest-bg-run bro bro -b ../exectest.bro
-# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout
 
 @TEST-START-FILE exectest.bro
 
 @load base/utils/exec
-
+@load base/frameworks/communication # let network-time run. otherwise there are no heartbeats...
 redef exit_only_after_terminate = T;
 
 global c: count = 0;
@@ -14,7 +14,7 @@ function check_exit_condition()
 	{
 	c += 1;
 
-	if ( c == 4 )
+	if ( c == 3 )
 		terminate();
 	}
 
@@ -32,7 +32,8 @@ event bro_init()
 	test_cmd("test1", [$cmd="bash ../somescript.sh",
 	                   $read_files=set("out1", "out2")]);
 	test_cmd("test2", [$cmd="bash ../nofiles.sh"]);
-	test_cmd("test3", [$cmd="bash ../suicide.sh"]);
+	# Not sure of a portable way to test signals yet.
+	#test_cmd("test3", [$cmd="bash ../suicide.sh"]);
 	test_cmd("test4", [$cmd="bash ../stdin.sh", $stdin="hibye"]);
 	}
 

From 1eadeaec3cf13a809d2c8720d9a6a3b895585796 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 3 Aug 2013 01:57:37 -0400
Subject: [PATCH 17/25] Fix a major memory issue in the SumStats framework.

 - There are still problems, but this should prevent a deadlock issue
   and help with memory use.
---
 CHANGES                                      | 4 ++++
 VERSION                                      | 2 +-
 scripts/base/frameworks/sumstats/cluster.bro | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index d1f1dde6a6..824eb0c436 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.1-1027 | 2013-08-03 01:57:37 -0400
+
+  * Fix a major memory issue in the SumStats framework.
+
 2.1-1026 | 2013-08-02 22:35:09 -0400
 
   * Fix the SumStats top-k plugin and test. (Seth Hall)
diff --git a/VERSION b/VERSION
index b6505c675b..7d3375bd3d 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-1026
+2.1-1027
diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index d2db09f312..9c343ad15d 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -73,7 +73,7 @@ global recent_global_view_keys: table[string, Key] of count &create_expire=1min
 
 # Result tables indexed on a uid that are currently being sent to the
 # manager.
-global sending_results: table[string] of ResultTable = table();
+global sending_results: table[string] of ResultTable = table() &create_expire=1min;
 
 # This is done on all non-manager node types in the event that a sumstat is
 # being collected somewhere other than a worker.
@@ -171,7 +171,7 @@ event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
 	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
 
 	# Create a back store for the result
-	sending_results[uid] = (ss_name in result_store) ? copy(result_store[ss_name]) : table();
+	sending_results[uid] = (ss_name in result_store) ? result_store[ss_name] : table();
 
 	# Lookup the actual sumstats and reset it, the reference to the data
 	# currently stored will be maintained internally from the 

From 2f0671aeeb05203890630e0c968730c1ac1781ad Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sat, 3 Aug 2013 20:27:43 -0700
Subject: [PATCH 18/25] Updating tests for DHCP.

---
 CHANGES                                                |  8 ++++++++
 VERSION                                                |  2 +-
 doc/scripts/DocSourcesList.cmake                       |  5 +++++
 scripts/test-all-policy.bro                            |  2 ++
 testing/btest/Baseline/core.print-bpf-filters/output2  | 10 ++++++----
 testing/btest/Baseline/core.tunnels.teredo/conn.log    |  6 +++---
 .../canonified_loaded_scripts.log                      |  5 +++--
 .../canonified_loaded_scripts.log                      | 10 ++++++++--
 8 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/CHANGES b/CHANGES
index 824eb0c436..3f11c55338 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,12 @@
 
+2.1-1034 | 2013-08-03 20:27:43 -0700
+
+  * A set of DHCP extensions. (Vlad Grigorescu)
+
+     - Leases are logged to dhcp.log as they are seen.
+     - scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro
+     - Added DPD sig.
+
 2.1-1027 | 2013-08-03 01:57:37 -0400
 
   * Fix a major memory issue in the SumStats framework.
diff --git a/VERSION b/VERSION
index 7d3375bd3d..5ed761d1ae 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-1027
+2.1-1034
diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index e405834ac6..bd88f5cd54 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -139,6 +139,9 @@ rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
 rest_target(${psd} base/protocols/conn/polling.bro)
+rest_target(${psd} base/protocols/dhcp/consts.bro)
+rest_target(${psd} base/protocols/dhcp/main.bro)
+rest_target(${psd} base/protocols/dhcp/utils.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/files.bro)
@@ -215,6 +218,7 @@ rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
 rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/detect-traceroute/main.bro)
+rest_target(${psd} policy/misc/known-devices.bro)
 rest_target(${psd} policy/misc/load-balancing.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
@@ -224,6 +228,7 @@ rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
+rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index f0900fda07..7d582bf82f 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -47,6 +47,7 @@
 @load misc/capture-loss.bro
 @load misc/detect-traceroute/__load__.bro
 @load misc/detect-traceroute/main.bro
+@load misc/known-devices.bro
 @load misc/load-balancing.bro
 @load misc/loaded-scripts.bro
 @load misc/profiling.bro
@@ -56,6 +57,7 @@
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro
 @load protocols/conn/weirds.bro
+@load protocols/dhcp/known-devices-and-hostnames.bro
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
 @load protocols/ftp/detect-bruteforcing.bro
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output2 b/testing/btest/Baseline/core.print-bpf-filters/output2
index 99ad929fbf..d7d8c8b05b 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output2
+++ b/testing/btest/Baseline/core.print-bpf-filters/output2
@@ -26,6 +26,8 @@
 1 6667
 1 6668
 1 6669
+1 67
+1 68
 1 80
 1 8000
 1 8080
@@ -36,8 +38,8 @@
 1 992
 1 993
 1 995
-40 and
-39 or
-40 port
+42 and
+41 or
+42 port
 31 tcp
-9 udp
+11 udp
diff --git a/testing/btest/Baseline/core.tunnels.teredo/conn.log b/testing/btest/Baseline/core.tunnels.teredo/conn.log
index b71e56f073..8bb55e11d2 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/conn.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#open	2008-05-16-15-50-57
+#open	2013-08-04-03-28-45
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1210953047.736921	arKYeMETxOg	192.168.2.16	1576	75.126.130.163	80	tcp	-	0.000357	0	0	SHR	-	0	fA	1	40	1	40	(empty)
@@ -21,10 +21,10 @@
 1210953074.570439	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	tcp	http	0.466677	469	3916	SF	-	0	ShADadFf	7	757	6	4164	(empty)
 1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	udp	teredo	8.928880	129	48	SF	-	0	Dd	2	185	1	76	(empty)
 1210953060.829233	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	udp	teredo	13.293994	2359	11243	SF	-	0	Dd	12	2695	13	11607	(empty)
-1210953058.933954	iE6yhOq3SF	0.0.0.0	68	255.255.255.255	67	udp	-	-	-	-	S0	-	0	D	1	328	0	0	(empty)
+1210953058.933954	iE6yhOq3SF	0.0.0.0	68	255.255.255.255	67	udp	dhcp	-	-	-	S0	-	0	D	1	328	0	0	(empty)
 1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	udp	-	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
 1210953046.591933	UWkUyAuUGXf	192.168.2.16	138	192.168.2.255	138	udp	-	28.448321	416	0	S0	-	0	D	2	472	0	0	(empty)
 1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
 1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c
 1210953052.202579	j4u32Pc5bif	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	nQcgTWjvg4c
-#close	2008-05-16-15-51-16
+#close	2013-08-04-03-28-45
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index e65b72a30b..6f85862bd7 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-07-29-22-37-52
+#open	2013-08-04-03-27-22
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -91,6 +91,7 @@ scripts/base/init-bare.bro
       scripts/base/utils/site.bro
         scripts/base/utils/patterns.bro
   build/scripts/base/bif/__load__.bro
+    build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-07-29-22-37-52
+#close	2013-08-04-03-27-22
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 572173bd97..0d9a490080 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-07-29-22-37-53
+#open	2013-08-04-03-27-23
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -91,6 +91,7 @@ scripts/base/init-bare.bro
       scripts/base/utils/site.bro
         scripts/base/utils/patterns.bro
   build/scripts/base/bif/__load__.bro
+    build/scripts/base/bif/top-k.bif.bro
 scripts/base/init-default.bro
   scripts/base/utils/active-http.bro
     scripts/base/utils/exec.bro
@@ -147,6 +148,7 @@ scripts/base/init-default.bro
       scripts/base/frameworks/sumstats/plugins/std-dev.bro
         scripts/base/frameworks/sumstats/plugins/variance.bro
       scripts/base/frameworks/sumstats/plugins/sum.bro
+      scripts/base/frameworks/sumstats/plugins/topk.bro
       scripts/base/frameworks/sumstats/plugins/unique.bro
     scripts/base/frameworks/sumstats/non-cluster.bro
   scripts/base/frameworks/tunnels/__load__.bro
@@ -156,6 +158,10 @@ scripts/base/init-default.bro
     scripts/base/protocols/conn/contents.bro
     scripts/base/protocols/conn/inactivity.bro
     scripts/base/protocols/conn/polling.bro
+  scripts/base/protocols/dhcp/__load__.bro
+    scripts/base/protocols/dhcp/consts.bro
+    scripts/base/protocols/dhcp/main.bro
+      scripts/base/protocols/dhcp/utils.bro
   scripts/base/protocols/dns/__load__.bro
     scripts/base/protocols/dns/consts.bro
     scripts/base/protocols/dns/main.bro
@@ -202,4 +208,4 @@ scripts/base/init-default.bro
     scripts/base/files/extract/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-07-29-22-37-53
+#close	2013-08-04-03-27-23

From 595e2f3c8a6829d44673a368ab13dd28bd4aab85 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 5 Aug 2013 16:44:50 -0400
Subject: [PATCH 19/25] Change to SSL log delay to cause the log to write even
 if delay times out.

---
 CHANGES                             | 4 ++++
 VERSION                             | 2 +-
 scripts/base/protocols/ssl/main.bro | 4 ++--
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3f11c55338..287f74286d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.1-1035 | 2013-08-05 16:44:50 -0400
+
+  * Change to SSL log delay to cause the log to write even if delay times out. (Seth Hall)
+
 2.1-1034 | 2013-08-03 20:27:43 -0700
 
   * A set of DHCP extensions. (Vlad Grigorescu)
diff --git a/VERSION b/VERSION
index 5ed761d1ae..491bca5672 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-1034
+2.1-1035
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 0d4a8435f0..67293c5bca 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -137,8 +137,8 @@ function log_record(info: Info)
 			}
 		timeout 15secs
 			{
-			Reporter::info(fmt("SSL delay tokens not released in time (%s tokens remaining)",
-			                   |info$delay_tokens|));
+			# We are just going to log the record anyway.
+			log_record(info);
 			}
 		}
 	}

From e52b1745943865e90fa95c2a98242f2da8efaf3f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 5 Aug 2013 17:29:11 -0400
Subject: [PATCH 20/25] Fix the SSL infinite loop I just created.

---
 CHANGES                             | 4 ++++
 VERSION                             | 2 +-
 scripts/base/protocols/ssl/main.bro | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 287f74286d..f9151efd79 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.1-1036 | 2013-08-05 17:29:11 -0400
+
+  * Fix the SSL infinite loop I just created. (Seth Hall)
+
 2.1-1035 | 2013-08-05 16:44:50 -0400
 
   * Change to SSL log delay to cause the log to write even if delay times out. (Seth Hall)
diff --git a/VERSION b/VERSION
index 491bca5672..1843674cb9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-1035
+2.1-1036
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index 67293c5bca..2381b356e4 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -138,6 +138,7 @@ function log_record(info: Info)
 		timeout 15secs
 			{
 			# We are just going to log the record anyway.
+			delete info$delay_tokens;
 			log_record(info);
 			}
 		}

From db7b6661fe705c0a359aa02ed5cac443a12bfa28 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 9 Aug 2013 09:50:05 -0500
Subject: [PATCH 21/25] Fix a unit test outdated by recent sumstats changes.

---
 testing/btest/core/leaks/basic-cluster.bro | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index d25af55b3f..2c13c2315c 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -26,16 +26,16 @@ global n = 0;
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
-	SumStats::create([$epoch=5secs,
+	SumStats::create([$name="test",
+	                  $epoch=5secs,
 	                  $reducers=set(r1),
-	                  $epoch_finished(rt: SumStats::ResultTable) =
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["test"];
+	                  	print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
+	                  	},
+	                  $epoch_finished(ts: time) =
 	                  	{
-	                  	for ( key in rt )
-	                  		{
-	                  		local r = rt[key]["test"];
-	                  		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
-	                  		}
-
 	                  	terminate();
 	                  	}]);
 	}

From b2392aa452089ecbbeb3186c252041dd9a919bef Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 9 Aug 2013 09:51:09 -0500
Subject: [PATCH 22/25] Fix mem leak in DHCP analyzer.

---
 src/analyzer/protocol/dhcp/dhcp-analyzer.pac | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/analyzer/protocol/dhcp/dhcp-analyzer.pac b/src/analyzer/protocol/dhcp/dhcp-analyzer.pac
index f58a2d9b5e..889d5ad7a5 100644
--- a/src/analyzer/protocol/dhcp/dhcp-analyzer.pac
+++ b/src/analyzer/protocol/dhcp/dhcp-analyzer.pac
@@ -224,7 +224,9 @@ flow DHCP_Flow(is_orig: bool) {
 		r->Assign(0, new Val(${msg.op}, TYPE_COUNT));
 		r->Assign(1, new Val(${msg.type}, TYPE_COUNT));
 		r->Assign(2, new Val(${msg.xid}, TYPE_COUNT));
-		r->Assign(3, new StringVal(fmt_mac(${msg.chaddr}.data(), ${msg.chaddr}.length())));
+		const char* mac_str = fmt_mac(${msg.chaddr}.data(), ${msg.chaddr}.length());
+		r->Assign(3, new StringVal(mac_str));
+		delete [] mac_str;
 		r->Assign(4, new AddrVal(${msg.ciaddr}));
 		r->Assign(5, new AddrVal(${msg.yiaddr}));
 

From 4bdbd1762d89bc9ff8a1e4c4826f0e22cca72710 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@illinois.edu>
Date: Fri, 9 Aug 2013 11:26:49 -0500
Subject: [PATCH 23/25] Update coverage baselines for canonical load order of
 scripts.

---
 .../canonified_loaded_scripts.log                  | 14 +++++++-------
 .../canonified_loaded_scripts.log                  | 14 +++++++-------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index 6f85862bd7..ca22ba568c 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-04-03-27-22
+#open	2013-08-09-16-13-58
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -23,28 +23,28 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_DCE_RPC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DHCP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DNS.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_File.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_Gnutella.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro
     build/scripts/base/bif/plugins/Bro_HTTP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_HTTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_ICMP.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Ident.events.bif.bro
     build/scripts/base/bif/plugins/Bro_InterConn.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Login.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Login.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_MIME.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Modbus.events.bif.bro
     build/scripts/base/bif/plugins/Bro_NCP.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_NTP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_NetBIOS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_NetBIOS.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_NetFlow.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_NTP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_PIA.events.bif.bro
     build/scripts/base/bif/plugins/Bro_POP3.events.bif.bro
     build/scripts/base/bif/plugins/Bro_RPC.events.bif.bro
@@ -94,4 +94,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/top-k.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2013-08-04-03-27-22
+#close	2013-08-09-16-13-58
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 0d9a490080..ab30771ef3 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-08-04-03-27-23
+#open	2013-08-09-16-13-37
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -23,28 +23,28 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_DCE_RPC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DHCP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_DNS.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_File.events.bif.bro
     build/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Finger.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_Gnutella.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro
     build/scripts/base/bif/plugins/Bro_HTTP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_HTTP.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_ICMP.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Ident.events.bif.bro
     build/scripts/base/bif/plugins/Bro_InterConn.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_IRC.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Login.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Login.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_MIME.events.bif.bro
     build/scripts/base/bif/plugins/Bro_Modbus.events.bif.bro
     build/scripts/base/bif/plugins/Bro_NCP.events.bif.bro
-    build/scripts/base/bif/plugins/Bro_NTP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_NetBIOS.events.bif.bro
     build/scripts/base/bif/plugins/Bro_NetBIOS.functions.bif.bro
     build/scripts/base/bif/plugins/Bro_NetFlow.events.bif.bro
+    build/scripts/base/bif/plugins/Bro_NTP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_PIA.events.bif.bro
     build/scripts/base/bif/plugins/Bro_POP3.events.bif.bro
     build/scripts/base/bif/plugins/Bro_RPC.events.bif.bro
@@ -208,4 +208,4 @@ scripts/base/init-default.bro
     scripts/base/files/extract/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-08-04-03-27-23
+#close	2013-08-09-16-13-37

From 937afb3e7b841f08f368dd992c9d8bfd9942ca78 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 9 Aug 2013 15:34:32 -0700
Subject: [PATCH 24/25] Updating submodule.

---
 cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake b/cmake
index 026639f836..370f7efc6e 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 026639f8368e56742c0cb5d9fb390ea64e60ec50
+Subproject commit 370f7efc6e144f978e2309ad80b1a10c83feaaa9

From b71dc5d8ff005a4cc25b59b99261facedd660dc9 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 9 Aug 2013 17:06:30 -0700
Subject: [PATCH 25/25] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 314fa8f65f..00674ed07d 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 314fa8f65fc240e960c23c3bba98623436a72b98
+Subproject commit 00674ed07d702252b00675b5060647d9e811cdd7
diff --git a/aux/bro-aux b/aux/bro-aux
index d9963983c0..0e2d74e488 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit d9963983c0b4d426b24836f8d154d014d5aecbba
+Subproject commit 0e2d74e488195170e4648037e22b51e122dc7b0e
diff --git a/aux/broccoli b/aux/broccoli
index d59c73b6e0..7ddfa3212d 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit d59c73b6e0966ad63bbc63a35741b5f68263e7b1
+Subproject commit 7ddfa3212d1fd0822588d4a96158f1a30c755afe
diff --git a/aux/broctl b/aux/broctl
index 090d4553ac..c9293bad3b 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 090d4553ace0f9acf2d86eafab07ecfdcc534878
+Subproject commit c9293bad3bf4d6fc3e1808a315e791140a632961