From 5904043e8572b17404026cf391755bfd3a2d74c1 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 22 Mar 2012 16:46:37 -0400
Subject: [PATCH 01/70] Modifications to the metrics framework API and new
 features.

- Metrics:ID enum has been replaced with strings.

- Uniqueness can now be measured with the Metrics::add_unique function.

- Filters can change the index value with the $normalize_func field.
---
 scripts/base/frameworks/metrics/cluster.bro |  30 +++---
 scripts/base/frameworks/metrics/main.bro    | 112 +++++++++++++-------
 2 files changed, 90 insertions(+), 52 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 4804bc5005..a17993cd27 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -27,29 +27,29 @@ export {
 	
 	## Event sent by the manager in a cluster to initiate the 
 	## collection of metrics values for a filter.
-	global cluster_filter_request: event(uid: string, id: ID, filter_name: string);
+	global cluster_filter_request: event(uid: string, id: string, filter_name: string);
 
 	## Event sent by nodes that are collecting metrics after receiving
 	## a request for the metric filter from the manager.
-	global cluster_filter_response: event(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool);
+	global cluster_filter_response: event(uid: string, id: string, filter_name: string, data: MetricTable, done: bool);
 
 	## This event is sent by the manager in a cluster to initiate the
 	## collection of a single index value from a filter.  It's typically
 	## used to get intermediate updates before the break interval triggers
 	## to speed detection of a value crossing a threshold.
-	global cluster_index_request: event(uid: string, id: ID, filter_name: string, index: Index);
+	global cluster_index_request: event(uid: string, id: string, filter_name: string, index: Index);
 
 	## This event is sent by nodes in response to a 
 	## :bro:id:`Metrics::cluster_index_request` event.
-	global cluster_index_response: event(uid: string, id: ID, filter_name: string, index: Index, val: count);
+	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, val: count);
 
 	## This is sent by workers to indicate that they crossed the percent of the 
 	## current threshold by the percentage defined globally in 
 	## :bro:id:`Metrics::cluster_request_global_view_percent`
-	global cluster_index_intermediate_response: event(id: Metrics::ID, filter_name: string, index: Metrics::Index, val: count);
+	global cluster_index_intermediate_response: event(id: string, filter_name: string, index: Metrics::Index, val: count);
 
 	## This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, id: ID, filter_name: string, data: MetricTable);
+	global send_data: event(uid: string, id: string, filter_name: string, data: MetricTable);
 	
 }
 
@@ -65,7 +65,7 @@ global requested_results: table[string] of time = table() &create_expire=5mins;
 
 # This variable is maintained by manager nodes as they collect and aggregate 
 # results.
-global filter_results: table[string, ID, string] of MetricTable &create_expire=5mins;
+global filter_results: table[string, string, string] of MetricTable &create_expire=5mins;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid 
@@ -76,7 +76,7 @@ global done_with: table[string] of count &create_expire=5mins &default=0;
 
 # This variable is maintained by managers to track intermediate responses as 
 # they are getting a global view for a certain index.
-global index_requests: table[string, ID, string, Index] of count &create_expire=5mins &default=0;
+global index_requests: table[string, string, string, Index] of count &create_expire=5mins &default=0;
 
 # This variable is maintained by all hosts for different purposes. Non-managers
 # maintain it to know what indexes they have recently sent as intermediate
@@ -86,7 +86,7 @@ global index_requests: table[string, ID, string, Index] of count &create_expire=
 # an intermediate result has been received. The manager may optionally request
 # the index again before data expires from here if too many workers are crossing
 # the percentage threshold (not implemented yet!).
-global recent_global_view_indexes: table[ID, string, Index] of count &create_expire=5mins &default=0;
+global recent_global_view_indexes: table[string, string, Index] of count &create_expire=5mins &default=0;
 
 # Add events to the cluster framework to make this work.
 redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
@@ -116,7 +116,7 @@ function data_added(filter: Filter, index: Index, val: count)
 		}
 	}
 
-event Metrics::send_data(uid: string, id: ID, filter_name: string, data: MetricTable)
+event Metrics::send_data(uid: string, id: string, filter_name: string, data: MetricTable)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 	
@@ -143,7 +143,7 @@ event Metrics::send_data(uid: string, id: ID, filter_name: string, data: MetricT
 		event Metrics::send_data(uid, id, filter_name, data);
 	}
 
-event Metrics::cluster_filter_request(uid: string, id: ID, filter_name: string)
+event Metrics::cluster_filter_request(uid: string, id: string, filter_name: string)
 	{
 	#print fmt("WORKER %s: received the cluster_filter_request event.", Cluster::node);
 	
@@ -155,7 +155,7 @@ event Metrics::cluster_filter_request(uid: string, id: ID, filter_name: string)
 	reset(filter_store[id, filter_name]);
 	}
 	
-event Metrics::cluster_index_request(uid: string, id: ID, filter_name: string, index: Index)
+event Metrics::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
 	{
 	local val=0;
 	if ( index in store[id, filter_name] )
@@ -195,7 +195,7 @@ function data_added(filter: Filter, index: Index, val: count)
 		do_notice(filter, index, val);
 	}
 	
-event Metrics::cluster_index_response(uid: string, id: ID, filter_name: string, index: Index, val: count)
+event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: count)
 	{
 	#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
 
@@ -216,7 +216,7 @@ event Metrics::cluster_index_response(uid: string, id: ID, filter_name: string,
 	}
 
 # Managers handle intermediate updates here.
-event Metrics::cluster_index_intermediate_response(id: ID, filter_name: string, index: Index, val: count)
+event Metrics::cluster_index_intermediate_response(id: string, filter_name: string, index: Index, val: count)
 	{
 	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting index data for %s", index2str(index));
@@ -226,7 +226,7 @@ event Metrics::cluster_index_intermediate_response(id: ID, filter_name: string,
 	++recent_global_view_indexes[id, filter_name, index];
 	}
 
-event Metrics::cluster_filter_response(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool)
+event Metrics::cluster_filter_response(uid: string, id: string, filter_name: string, data: MetricTable, done: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 	
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index d322d128fe..e80054353a 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -8,12 +8,6 @@ export {
 	## The metrics logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	## Identifiers for metrics to collect.
-	type ID: enum {
-		## Blank placeholder value.
-		NOTHING,
-	};
-	
 	## The default interval used for "breaking" metrics and writing the 
 	## current value to the logging stream.
 	const default_break_interval = 15mins &redef;
@@ -23,12 +17,8 @@ export {
 	const renotice_interval = 1hr &redef;
 	
 	## Represents a thing which is having metrics collected for it.  An instance
-	## of this record type and a :bro:type:`Metrics::ID` together represent a 
-	## single measurement.
+	## of this record type and an id together represent a single measurement.
 	type Index: record {
-		## Host is the value to which this metric applies.
-		host:         addr &optional;
-		
 		## A non-address related metric or a sub-key for an address based metric.
 		## An example might be successful SSH connections by client IP address
 		## where the client string would be the index value.
@@ -36,7 +26,10 @@ export {
 		## value in a Host header.  This is an example of a non-host based
 		## metric since multiple IP addresses could respond for the same Host
 		## header value.
-		str:        string &optional;
+		str:          string &optional;
+	
+		## Host is the value to which this metric applies.
+		host:         addr &optional;
 		
 		## The CIDR block that this metric applies to.  This is typically
 		## only used internally for host based aggregation.
@@ -46,17 +39,19 @@ export {
 	## The record type that is used for logging metrics.
 	type Info: record {
 		## Timestamp at which the metric was "broken".
-		ts:           time   &log;
-		## What measurement the metric represents.
-		metric_id:    ID     &log;
-		## The name of the filter being logged.  :bro:type:`Metrics::ID` values
+		ts:           time     &log;
+		## Interval between logging of this filter and the last time it was logged.
+		ts_delta:     interval &log;
+		## The name of the filter being logged.  Values
 		## can have multiple filters which represent different perspectives on
 		## the data so this is necessary to understand the value.
-		filter_name:  string &log;
+		filter_name:  string   &log;
+		## What measurement the metric represents.
+		metric_id:    string   &log;
 		## What the metric value applies to.
-		index:        Index  &log;
+		index:        Index    &log;
 		## The simple numeric value of the metric.
-		value:        count  &log;
+		value:        count    &log;
 	};
 	
     # TODO: configure a metrics filter logging stream to log the current
@@ -68,15 +63,19 @@ export {
 	## and logged or how the data within them is aggregated.  It's also 
 	## possible to disable logging and use filters for thresholding.
 	type Filter: record {
-		## The :bro:type:`Metrics::ID` that this filter applies to.
-		id:                ID                      &optional;
 		## The name for this filter so that multiple filters can be
 		## applied to a single metrics to get a different view of the same
 		## metric data being collected (different aggregation, break, etc).
 		name:              string                  &default="default";
+		## The :bro:type:`Metrics::ID` that this filter applies to.
+		id:                string                  &optional;
 		## A predicate so that you can decide per index if you would like
 		## to accept the data being inserted.
 		pred:              function(index: Index): bool &optional;
+		## A function to normalize the index.  This can be used to normalize
+		## any field in the index and is likely most useful to normalize
+		## the $str field.
+		normalize_func:    function(index: Index): Index &optional;
 		## Global mask by which you'd like to aggregate traffic.
 		aggregation_mask:  count                   &optional;
 		## This is essentially a mapping table between addresses and subnets.
@@ -111,7 +110,7 @@ export {
 	## id: The metric ID that the filter should be associated with.
 	##
 	## filter: The record representing the filter configuration.
-	global add_filter: function(id: ID, filter: Filter);
+	global add_filter: function(id: string, filter: Filter);
 	
 	## Add data into a :bro:type:`Metrics::ID`.  This should be called when
 	## a script has measured some point value and is ready to increment the
@@ -122,7 +121,9 @@ export {
 	## index: The metric index that the value is to be added to.
 	##
 	## increment: How much to increment the counter by.
-	global add_data: function(id: ID, index: Index, increment: count);
+	global add_data: function(id: string, index: Index, increment: count);
+	
+	global add_unique: function(id: string, index: Index, data: string);
 	
 	## Helper function to represent a :bro:type:`Metrics::Index` value as 
 	## a simple string
@@ -141,19 +142,24 @@ export {
 	## Event to access metrics records as they are passed to the logging framework.
 	global log_metrics: event(rec: Info);
 	
+	## Internal use only
+	type MetricMeasurement: record {
+		num:        count       &optional;
+		unique_vals: set[string] &optional;
+	};
 	## Type to store a table of metrics values.  Interal use only!
-	type MetricTable: table[Index] of count &default=0;
+	type MetricTable: table[Index] of MetricMeasurement;
 }
 
 redef record Notice::Info += {
 	metric_index: Index &log &optional;
 };
 
-global metric_filters: table[ID] of vector of Filter = table();
-global filter_store: table[ID, string] of Filter = table();
+global metric_filters: table[string] of vector of Filter = table();
+global filter_store: table[string, string] of Filter = table();
 
 # This is indexed by metric ID and stream filter name.
-global store: table[ID, string] of MetricTable = table() &default=table();
+global store: table[string, string] of MetricTable = table() &default=table();
 
 # This function checks if a threshold has been crossed and generates a 
 # notice if it has.  It is also used as a method to implement 
@@ -166,7 +172,7 @@ global data_added: function(filter: Filter, index: Index, val: count);
 
 # This stores the current threshold index for filters using the
 # $notice_threshold and $notice_thresholds elements.
-global thresholds: table[ID, string, Index] of count = {} &create_expire=renotice_interval &default=0;
+global thresholds: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0;
 
 event bro_init() &priority=5
 	{
@@ -189,8 +195,13 @@ function write_log(ts: time, filter: Filter, data: MetricTable)
 	{
 	for ( index in data )
 		{
-		local val = data[index];
+		local val = 0;
+		if ( data[index]?$unique_vals )
+			val = |data[index]$unique_vals|;
+		else
+			val = data[index]$num;
 		local m: Info = [$ts=ts,
+		                 $ts_delta=filter$break_interval,
 		                 $metric_id=filter$id,
 		                 $filter_name=filter$name,
 		                 $index=index,
@@ -207,7 +218,7 @@ function reset(filter: Filter)
 	store[filter$id, filter$name] = table();
 	}
 
-function add_filter(id: ID, filter: Filter)
+function add_filter(id: string, filter: Filter)
 	{
 	if ( filter?$aggregation_table && filter?$aggregation_mask )
 		{
@@ -237,8 +248,8 @@ function add_filter(id: ID, filter: Filter)
 	
 	schedule filter$break_interval { Metrics::log_it(filter) };
 	}
-	
-function add_data(id: ID, index: Index, increment: count)
+
+function add_it(id: string, index: Index, integer_value: bool, num: count, str: string)
 	{
 	if ( id !in metric_filters )
 		return;
@@ -257,6 +268,11 @@ function add_data(id: ID, index: Index, increment: count)
 		
 		if ( index?$host )
 			{
+			if ( filter?$normalize_func )
+				{
+				index = filter$normalize_func(copy(index));
+				}
+			
 			if ( filter?$aggregation_mask )
 				{
 				index$network = mask_addr(index$host, filter$aggregation_mask);
@@ -274,14 +290,36 @@ function add_data(id: ID, index: Index, increment: count)
 			}
 		
 		local metric_tbl = store[id, filter$name];
-		if ( index !in metric_tbl )
-			metric_tbl[index] = 0;
-		metric_tbl[index] += increment;
-		
-		data_added(filter, index, metric_tbl[index]);
+		if ( integer_value )
+			{
+			if ( index !in metric_tbl )
+				metric_tbl[index] = [$num=0];
+			metric_tbl[index]$num += num;
+			data_added(filter, index, metric_tbl[index]$num);
+			}
+		else
+			{
+			if ( index !in metric_tbl )
+				{
+				local empty_ss: set[string] = set();
+				metric_tbl[index] = [$unique_vals=empty_ss];
+				}
+			add metric_tbl[index]$unique_vals[str];
+			data_added(filter, index, |metric_tbl[index]$unique_vals|);
+			}
 		}
 	}
 
+function add_data(id: string, index: Index, increment: count)
+	{
+	add_it(id, index, T, increment, "");
+	}
+	
+function add_unique(id: string, index: Index, data: string)
+	{
+	add_it(id, index, F, 0, data);
+	}
+	
 function check_notice(filter: Filter, index: Index, val: count): bool
 	{
 	if ( (filter?$notice_threshold &&

From 6600e62ea386d7801e52a3646c5761df7f10cba0 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Mar 2012 11:39:27 -0400
Subject: [PATCH 02/70] Ported scripts using metrics framework and added a new
 smtp script.

- New script measures a couple of aspects of SMTP traffic.

- Existing metrics scripts had a small amount of work done
  to make them work with changes to metrics framework.
---
 scripts/base/frameworks/metrics/main.bro      | 13 ++++----
 .../frameworks/metrics/conn-example.bro       | 16 ++++------
 scripts/policy/protocols/http/detect-sqli.bro | 27 ++++++----------
 scripts/policy/protocols/smtp/metrics.bro     | 32 +++++++++++++++++++
 .../protocols/ssh/detect-bruteforcing.bro     | 17 ++++------
 5 files changed, 61 insertions(+), 44 deletions(-)
 create mode 100644 scripts/policy/protocols/smtp/metrics.bro

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index e80054353a..e5ed63dccd 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -36,6 +36,12 @@ export {
 		network:      subnet &optional;
 	} &log;
 	
+	## Represents data being added for a single metric data point.  Used internally.
+	type DataPoint: record {
+		num:        count       &optional;
+		unique_vals: set[string] &optional;
+	};
+	
 	## The record type that is used for logging metrics.
 	type Info: record {
 		## Timestamp at which the metric was "broken".
@@ -142,13 +148,8 @@ export {
 	## Event to access metrics records as they are passed to the logging framework.
 	global log_metrics: event(rec: Info);
 	
-	## Internal use only
-	type MetricMeasurement: record {
-		num:        count       &optional;
-		unique_vals: set[string] &optional;
-	};
 	## Type to store a table of metrics values.  Interal use only!
-	type MetricTable: table[Index] of MetricMeasurement;
+	type MetricTable: table[Index] of DataPoint;
 }
 
 redef record Notice::Info += {
diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro
index 974012963b..00c82f840d 100644
--- a/scripts/policy/frameworks/metrics/conn-example.bro
+++ b/scripts/policy/frameworks/metrics/conn-example.bro
@@ -4,22 +4,18 @@
 @load base/frameworks/metrics
 @load base/utils/site
 
-redef enum Metrics::ID += { 
-	CONNS_ORIGINATED, 
-	CONNS_RESPONDED 
-};
-
 event bro_init()
 	{
-	Metrics::add_filter(CONNS_ORIGINATED, [$aggregation_mask=24, $break_interval=1mins]);
+	#Metrics::add_filter("conns.originated", [$aggregation_mask=24, $break_interval=1mins]);
+	Metrics::add_filter("conns.originated",  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
+	
 	
 	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter(CONNS_RESPONDED,  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
+	Metrics::add_filter("conns.responded",  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
 	}
 
 event connection_established(c: connection)
 	{
-	Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h], 1);
-	Metrics::add_data(CONNS_RESPONDED,  [$host=c$id$resp_h], 1);
+	Metrics::add_data("conns.originated", [$host=c$id$orig_h], 1);
+	Metrics::add_data("conns.responded",  [$host=c$id$resp_h], 1);
 	}
-	
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index a92565c63a..193a4b9614 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -15,13 +15,6 @@ export {
 		SQL_Injection_Victim,
 	};
 	
-	redef enum Metrics::ID += {
-		## Metric to track SQL injection attackers.
-		SQLI_ATTACKER,
-		## Metrics to track SQL injection victims.
-		SQLI_VICTIM,
-	};
-
 	redef enum Tags += {
 		## Indicator of a URI based SQL injection attack.
 		URI_SQLI,
@@ -58,14 +51,14 @@ event bro_init() &priority=3
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
 	
-	Metrics::add_filter(SQLI_ATTACKER, [$log=F,
-	                                   $notice_threshold=sqli_requests_threshold,
-	                                   $break_interval=sqli_requests_interval,
-	                                   $note=SQL_Injection_Attacker]);
-	Metrics::add_filter(SQLI_VICTIM, [$log=F,
-	                                 $notice_threshold=sqli_requests_threshold,
-	                                 $break_interval=sqli_requests_interval,
-	                                 $note=SQL_Injection_Victim]);
+	Metrics::add_filter("http.sqli.attacker", [$log=F,
+	                                           $notice_threshold=sqli_requests_threshold,
+	                                           $break_interval=sqli_requests_interval,
+	                                           $note=SQL_Injection_Attacker]);
+	Metrics::add_filter("http.sqli.victim", [$log=F,
+	                                         $notice_threshold=sqli_requests_threshold,
+	                                         $break_interval=sqli_requests_interval,
+	                                         $note=SQL_Injection_Victim]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -75,7 +68,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data(SQLI_ATTACKER, [$host=c$id$orig_h], 1);
-		Metrics::add_data(SQLI_VICTIM, [$host=c$id$resp_h], 1);
+		Metrics::add_data("http.sqli.attacker", [$host=c$id$orig_h], 1);
+		Metrics::add_data("http.sqli.victim", [$host=c$id$resp_h], 1);
 		}
 	}
diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro
new file mode 100644
index 0000000000..c3d1bb1e20
--- /dev/null
+++ b/scripts/policy/protocols/smtp/metrics.bro
@@ -0,0 +1,32 @@
+##! This script is meant to answer the following questions...
+##!   "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?"
+##!   "How much mail is being sent from each local mail server per hour?"
+
+@load base/frameworks/metrics
+
+module SMTPMetrics;
+
+export {
+	## Define the break intervals for all of the metrics collected and logged by this script.
+	const breaks = 1hr &redef;
+}
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter("smtp.mailfrom", [$pred(index: Metrics::Index) = { 
+	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); }, 
+	                                      $break_interval=breaks]);
+	Metrics::add_filter("smtp.messages", [$pred(index: Metrics::Index) = { 
+	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); }, 
+	                                      $break_interval=breaks]);
+	}
+
+event SMTP::log_smtp(rec: SMTP::Info)
+	{
+	Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], 1);
+	
+	if ( rec?$mailfrom )
+		Metrics::add_unique("smtp.mailfrom", [$host=rec$id$orig_h], rec$mailfrom);
+	}
+
+
diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index aa6e920c12..7939f00c72 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -20,11 +20,6 @@ export {
 		Login_By_Password_Guesser,
 	};
 	
-	redef enum Metrics::ID  += {
-		## Metric is to measure failed logins.
-		FAILED_LOGIN,
-	};
-
 	## The number of failed SSH connections before a host is designated as
 	## guessing passwords.
 	const password_guesses_limit = 30 &redef;
@@ -46,11 +41,11 @@ export {
 
 event bro_init()
 	{
-	Metrics::add_filter(FAILED_LOGIN, [$name="detect-bruteforcing", $log=F,
-	                                   $note=Password_Guessing,
-	                                   $notice_threshold=password_guesses_limit,
-	                                   $notice_freq=1hr,
-	                                   $break_interval=guessing_timeout]);
+	Metrics::add_filter("ssh.login.failure", [$name="detect-bruteforcing", $log=F,
+	                                          $note=Password_Guessing,
+	                                          $notice_threshold=password_guesses_limit,
+	                                          $notice_freq=1hr,
+	                                          $break_interval=guessing_timeout]);
 	}
 
 event SSH::heuristic_successful_login(c: connection)
@@ -75,5 +70,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		Metrics::add_data(FAILED_LOGIN, [$host=id$orig_h], 1);
+		Metrics::add_data("ssh.login.failure", [$host=id$orig_h], 1);
 	}

From 47f58e6340b81b4573d9cbdb0988799f317b23a4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Mar 2012 14:00:59 -0400
Subject: [PATCH 03/70] Cluster support for the metrics framework returns and
 all tests work again.

---
 scripts/base/frameworks/metrics/cluster.bro   | 31 +++++++++----------
 scripts/base/frameworks/metrics/main.bro      | 26 ++++++++++++++++
 .../manager-1.metrics.log                     | 10 +++---
 .../metrics.log                               | 10 +++---
 .../manager-1.notice.log                      |  6 ++--
 .../notice.log                                |  8 ++---
 .../base/frameworks/metrics/basic-cluster.bro | 16 ++++------
 .../scripts/base/frameworks/metrics/basic.bro | 12 +++----
 .../metrics/cluster-intermediate-update.bro   | 12 +++----
 .../base/frameworks/metrics/notice.bro        | 12 +++----
 10 files changed, 76 insertions(+), 67 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index a17993cd27..03bbcdf584 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -41,7 +41,7 @@ export {
 
 	## This event is sent by nodes in response to a 
 	## :bro:id:`Metrics::cluster_index_request` event.
-	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, val: count);
+	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, data: DataPoint);
 
 	## This is sent by workers to indicate that they crossed the percent of the 
 	## current threshold by the percentage defined globally in 
@@ -76,7 +76,7 @@ global done_with: table[string] of count &create_expire=5mins &default=0;
 
 # This variable is maintained by managers to track intermediate responses as 
 # they are getting a global view for a certain index.
-global index_requests: table[string, string, string, Index] of count &create_expire=5mins &default=0;
+global index_requests: table[string, string, string, Index] of DataPoint &create_expire=5mins &default=[];
 
 # This variable is maintained by all hosts for different purposes. Non-managers
 # maintain it to know what indexes they have recently sent as intermediate
@@ -157,12 +157,12 @@ event Metrics::cluster_filter_request(uid: string, id: string, filter_name: stri
 	
 event Metrics::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
 	{
-	local val=0;
+	local data: DataPoint;
 	if ( index in store[id, filter_name] )
-		val = store[id, filter_name][index];
+		data = store[id, filter_name][index];
 	
 	# fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val);
-	event Metrics::cluster_index_response(uid, id, filter_name, index, val);
+	event Metrics::cluster_index_response(uid, id, filter_name, index, data);
 	}
 
 @endif
@@ -195,21 +195,19 @@ function data_added(filter: Filter, index: Index, val: count)
 		do_notice(filter, index, val);
 	}
 	
-event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: count)
+event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint)
 	{
 	#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
-
-	if ( [uid, id, filter_name, index] !in index_requests )
-		index_requests[uid, id, filter_name, index] = 0;
 	
-	index_requests[uid, id, filter_name, index] += val;
+	index_requests[uid, id, filter_name, index] = merge_data_points(index_requests[uid, id, filter_name, index], data);
 	local ir = index_requests[uid, id, filter_name, index];
 	
 	++done_with[uid];
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		if ( check_notice(filter_store[id, filter_name], index, ir) )
-			do_notice(filter_store[id, filter_name], index, ir);
+		local size = ir?$num ? ir$num : |ir$unique_vals|;
+		if ( check_notice(filter_store[id, filter_name], index, size) )
+			do_notice(filter_store[id, filter_name], index, size);
 		delete done_with[uid];
 		delete index_requests[uid, id, filter_name, index];
 		}
@@ -233,12 +231,13 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 	local local_data = filter_results[uid, id, filter_name];
 	for ( index in data )
 		{
-		if ( index !in local_data )
-			local_data[index] = 0;
-		local_data[index] += data[index];
+		if ( index in local_data )
+			local_data[index] = merge_data_points(local_data[index], data[index]);
+		else
+			local_data[index] = data[index];
 		}
 	
-	# Mark another worker as being "done" for this uid.
+		# Mark another worker as being "done" for this uid.
 	if ( done )
 		++done_with[uid];
 	
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index e5ed63dccd..0e2496ef16 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -192,6 +192,32 @@ function index2str(index: Index): string
 	return fmt("metric_index(%s)", out);
 	}
 	
+function merge_data_points(dp1: DataPoint, dp2: DataPoint): DataPoint
+	{
+	local result: DataPoint;
+	if ( dp1?$num || dp2?$num )
+		{
+		result$num = 0;
+		if ( dp1?$num )
+			result$num += dp1$num;
+		if ( dp2?$num )
+			result$num += dp2$num;
+		}
+		
+	if ( dp1?$unique_vals || dp2?$unique_vals )
+		{
+		result$unique_vals = set();
+		if ( dp1?$unique_vals )
+			for ( val1 in dp1$unique_vals )
+				add result$unique_vals[val1];
+		if ( dp2?$unique_vals )
+			for ( val2 in dp2$unique_vals )
+				add result$unique_vals[val2];
+			}
+			
+	return result;
+	}
+	
 function write_log(ts: time, filter: Filter, data: MetricTable)
 	{
 	for ( index in data )
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index a22deb26e4..26ee55e20f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
-#types	time	enum	string	addr	string	subnet	count
-1328303679.867377	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1328303679.867377	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
-1328303679.867377	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+#fields	ts	ts_delta	filter_name	metric_id	index.str	index.host	index.network	value
+#types	time	interval	string	string	string	addr	subnet	count
+1332957065.172883	3.000000	foo-bar	test.metric	-	6.5.4.3	-	4
+1332957065.172883	3.000000	foo-bar	test.metric	-	1.2.3.4	-	6
+1332957065.172883	3.000000	foo-bar	test.metric	-	7.2.1.5	-	2
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index 4bfb6964ea..7c625fea5b 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
-#types	time	enum	string	addr	string	subnet	count
-1328303763.333948	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
-1328303763.333948	TEST_METRIC	foo-bar	7.2.1.5	-	-	1
-1328303763.333948	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
+#fields	ts	ts_delta	filter_name	metric_id	index.str	index.host	index.network	value
+#types	time	interval	string	string	string	addr	subnet	count
+1332956138.267655	3.000000	foo-bar	test.metric	-	6.5.4.3	-	2
+1332956138.267655	3.000000	foo-bar	test.metric	-	1.2.3.4	-	3
+1332956138.267655	3.000000	foo-bar	test.metric	-	7.2.1.5	-	1
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
index 59d70896fb..d5af4008d0 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
@@ -3,6 +3,6 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1325633225.777902	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.str	metric_index.host	metric_index.network
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	string	addr	subnet
+1332957572.934499	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	1.2.3.4	-
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
index 58346b79e6..bb25f5b0ea 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1325633274.875473	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
-1325633274.875473	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	6.5.4.3	-	-
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.str	metric_index.host	metric_index.network
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	string	addr	subnet
+1332956197.821031	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	1.2.3.4	-
+1332956197.821031	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	6.5.4.3	-
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index b801074b33..425c91fb53 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -10,8 +10,8 @@
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -19,20 +19,16 @@ redef Cluster::nodes = {
 
 redef Log::default_rotation_interval = 0secs;
 
-redef enum Metrics::ID += {
-	TEST_METRIC,
-};
-
 event bro_init() &priority=5
 	{
-	Metrics::add_filter(TEST_METRIC, 
+	Metrics::add_filter("test.metric", 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
 		
 	if ( Cluster::local_node_type() == Cluster::WORKER )
 		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
+		Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
+		Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
 		}
 	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic.bro b/testing/btest/scripts/base/frameworks/metrics/basic.bro
index 43e7ac28ef..23a79d2bd3 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic.bro
@@ -1,16 +1,12 @@
 # @TEST-EXEC: bro %INPUT
 # @TEST-EXEC: btest-diff metrics.log
 
-redef enum Metrics::ID += {
-	TEST_METRIC,
-};
-
 event bro_init() &priority=5
 	{
-	Metrics::add_filter(TEST_METRIC, 
+	Metrics::add_filter("test.metric", 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
+	Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
+	Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
 	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
index 701d2ea378..f442a100f0 100644
--- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
@@ -10,8 +10,8 @@
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -23,13 +23,9 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
-redef enum Metrics::ID += {
-	TEST_METRIC,
-};
-
 event bro_init() &priority=5
 	{
-	Metrics::add_filter(TEST_METRIC,
+	Metrics::add_filter("test.metric",
 		[$name="foo-bar",
 		 $break_interval=1hr,
 		 $note=Test_Notice,
@@ -44,7 +40,7 @@ event do_metrics(i: count)
 	# Worker-1 will trigger an intermediate update and then if everything
 	# works correctly, the data from worker-2 will hit the threshold and
 	# should trigger the notice.
-	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], i);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], i);
 	}
 
 event bro_init()
diff --git a/testing/btest/scripts/base/frameworks/metrics/notice.bro b/testing/btest/scripts/base/frameworks/metrics/notice.bro
index 0ac9faa956..1ed11a968c 100644
--- a/testing/btest/scripts/base/frameworks/metrics/notice.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/notice.bro
@@ -6,19 +6,15 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
-redef enum Metrics::ID += {
-	TEST_METRIC,
-};
-
 event bro_init() &priority=5
 	{
-	Metrics::add_filter(TEST_METRIC, 
+	Metrics::add_filter("test.metric", 
 		[$name="foo-bar",
 		 $break_interval=3secs,
 		 $note=Test_Notice,
 		 $notice_threshold=2,
 		 $log=F]);
-	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
+	Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
+	Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
 	}

From df6a1800233721b78c8ca8c5ac720a5cea8390cf Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Mar 2012 15:52:20 -0400
Subject: [PATCH 04/70] Some scripts for collecting connection stats and "app"
 stats.

- App stats are considered stats for applications on the internet.
  Services like facebook, youtube, etc.
---
 scripts/policy/misc/app-metrics.bro       | 75 +++++++++++++++++++++++
 scripts/policy/protocols/conn/metrics.bro | 21 +++++++
 2 files changed, 96 insertions(+)
 create mode 100644 scripts/policy/misc/app-metrics.bro
 create mode 100644 scripts/policy/protocols/conn/metrics.bro

diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
new file mode 100644
index 0000000000..40b8264233
--- /dev/null
+++ b/scripts/policy/misc/app-metrics.bro
@@ -0,0 +1,75 @@
+@load base/protocols/http
+@load base/protocols/ssl
+
+@load base/frameworks/metrics
+
+module AppMetrics;
+
+event bro_init() &priority=3
+	{
+	Metrics::add_filter("apps.bytes", [$break_interval=1hr]);
+	Metrics::add_filter("apps.views", [$break_interval=1hr]);
+	Metrics::add_filter("apps.users", [$break_interval=1hr]);
+	}
+
+function do_metric(id: conn_id, hostname: string, size: count)
+	{
+	if ( /youtube/ in hostname && size > 512*1024 ) 
+		{
+		Metrics::add_data("apps.bytes", [$str="youtube"], size);
+		Metrics::add_data("apps.views", [$str="youtube"], 1);
+		Metrics::add_unique("apps.users", [$str="youtube"], cat(id$orig_h));
+		}
+	else if ( /facebook.com|fbcdn.net/ in hostname && size > 20 )
+		{
+		Metrics::add_data("apps.bytes", [$str="facebook"], size);
+		Metrics::add_data("apps.views", [$str="facebook"], 1);
+		Metrics::add_unique("apps.users", [$str="facebook"], cat(id$orig_h));
+		}
+	else if ( /google.com/ in hostname && size > 20 ) 
+		{
+		Metrics::add_data("apps.bytes", [$str="google"], size);
+		Metrics::add_data("apps.views", [$str="google"], 1);
+		Metrics::add_unique("apps.users", [$str="google"], cat(id$orig_h));
+		}
+	else if ( /nflximg.com/ in hostname && size > 200*1024 ) 
+		{
+		Metrics::add_data("apps.bytes", [$str="netflix"], size);
+		Metrics::add_data("apps.views", [$str="netflix"], 1);
+		Metrics::add_unique("apps.users", [$str="netflix"], cat(id$orig_h));
+		}
+	else if ( /pandora.com/ in hostname && size > 512*1024 )
+		{
+		Metrics::add_data("apps.bytes", [$str="pandora"], size);
+		Metrics::add_data("apps.views", [$str="pandora"], 1);
+		Metrics::add_unique("apps.users", [$str="pandora"], cat(id$orig_h));
+		}
+	else if ( /gmail.com/ in hostname && size > 20 )
+		{
+		Metrics::add_data("apps.bytes", [$str="gmail"], size);
+		Metrics::add_data("apps.views", [$str="gmail"], 1);
+		Metrics::add_unique("apps.users", [$str="gmail"], cat(id$orig_h));
+		}
+}
+
+redef record connection += {
+	resp_hostname: string &optional;
+};
+
+event ssl_established(c: connection)
+	{
+	if ( c?$ssl && c$ssl?$server_name )
+		c$resp_hostname = c$ssl$server_name;
+	}
+
+event connection_finished(c: connection)
+	{
+	if ( c?$resp_hostname )
+		do_metric(c$id, c$resp_hostname, c$resp$num_bytes_ip);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if( rec?$host )
+		do_metric(rec$id, rec$host, rec$response_body_len);
+	}
diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro
new file mode 100644
index 0000000000..910ae4aa6e
--- /dev/null
+++ b/scripts/policy/protocols/conn/metrics.bro
@@ -0,0 +1,21 @@
+@load base/frameworks/metrics
+
+event bro_init() &priority=3
+	{
+	Metrics::add_filter("conns.country", [$break_interval=1hr]);
+	Metrics::add_filter("hosts.active", [$break_interval=1hr]);
+	}
+
+event connection_established(c: connection) &priority=3
+	{
+	if ( Site::is_local_addr(c$id$orig_h) )
+		{
+		local loc = lookup_location(c$id$resp_h);
+		if ( loc?$country_code )
+			Metrics::add_data("conns.country", [$str=loc$country_code], 1);
+		}
+		
+	local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h;
+	# There is no index for this.
+	Metrics::add_unique("hosts.active", [], cat(the_host));
+	}

From 77694cc88479b2f16e6d9c5bce2f8522eaefcb85 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Mar 2012 16:06:09 -0400
Subject: [PATCH 05/70] The app metrics break interval is now configurable.

---
 scripts/policy/misc/app-metrics.bro | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index 40b8264233..5cb108ea73 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -5,11 +5,16 @@
 
 module AppMetrics;
 
+export {
+	## The metric break interval for the default stats collected by this script.
+	const break_interval = 1hr &redef;
+}
+
 event bro_init() &priority=3
 	{
-	Metrics::add_filter("apps.bytes", [$break_interval=1hr]);
-	Metrics::add_filter("apps.views", [$break_interval=1hr]);
-	Metrics::add_filter("apps.users", [$break_interval=1hr]);
+	Metrics::add_filter("apps.bytes", [$break_interval=break_interval]);
+	Metrics::add_filter("apps.views", [$break_interval=break_interval]);
+	Metrics::add_filter("apps.users", [$break_interval=break_interval]);
 	}
 
 function do_metric(id: conn_id, hostname: string, size: count)

From 83910eeb08679df068f910fa8edb04d9c92a43ae Mon Sep 17 00:00:00 2001
From: Sheharbano Khattak <sheharbano.k@gmail.com>
Date: Tue, 9 Oct 2012 05:33:37 +0500
Subject: [PATCH 06/70] Added function to intercept threshold checking

---
 aux/binpac                               |  2 +-
 aux/bro-aux                              |  2 +-
 aux/broccoli                             |  2 +-
 aux/broctl                               |  2 +-
 aux/btest                                |  2 +-
 cmake                                    |  2 +-
 scripts/base/frameworks/metrics/main.bro | 72 +++++++++++++++++++-----
 7 files changed, 65 insertions(+), 19 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 3034da8f08..a93ef13735 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 3034da8f082b61157e234237993ffd7a95be6e62
+Subproject commit a93ef1373512c661ffcd0d0a61bd19b96667e0d5
diff --git a/aux/bro-aux b/aux/bro-aux
index f53bcb2b49..6748ec3a96 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit f53bcb2b492cb0db3dd288384040abc2ab711767
+Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff
diff --git a/aux/broccoli b/aux/broccoli
index a08ca90727..ebfa4de45a 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit a08ca90727c5c4b90aa8633106ec33a5cf7378d4
+Subproject commit ebfa4de45a839e58aec200e7e4bad33eaab4f1ed
diff --git a/aux/broctl b/aux/broctl
index 954538514d..b0e3c0d846 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 954538514d71983e7ef3f0e109960466096e1c1d
+Subproject commit b0e3c0d84643878c135dcb8a9774ed78147dd648
diff --git a/aux/btest b/aux/btest
index 9c9fde204d..44a43e6245 160000
--- a/aux/btest
+++ b/aux/btest
@@ -1 +1 @@
-Subproject commit 9c9fde204dd5518bdfdb8b4a86d38ed06e597209
+Subproject commit 44a43e62452302277f88e8fac08d1f979dc53f98
diff --git a/cmake b/cmake
index 2cc1055770..125f9a5fa8 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 2cc105577044a2d214124568f3f2496ed2ccbb34
+Subproject commit 125f9a5fa851381d0350efa41a4d14f27be263a2
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 0e2496ef16..744eaf731d 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -103,12 +103,20 @@ export {
 		notice_threshold:  count                   &optional;
 		## A series of thresholds at which to generate notices.
 		notice_thresholds: vector of count         &optional;
-		## How often this notice should be raised for this filter.  It 
-		## will be generated everytime it crosses a threshold, but if the 
-		## $break_interval is set to 5mins and this is set to 1hr the notice
-		## only be generated once per hour even if something crosses the
-		## threshold in every break interval.
-		notice_freq:       interval                &optional;
+		
+		## Sheharbano's additions
+		##--------------------------------------------
+		## A straight threshold for generating a notice.
+		default_threshold:  count                   &optional;
+		## Represents Index specific thresholds, that is we can
+		## have different thresholds for different Index values.
+		## If the threshold for an Index is not specified in <dynamic_thresholds>,
+		## <threshold> will be used as default.
+		custom_thresholds: table[Index] of count         &optional;
+		## A predicate so that you can decide when to flexibly declare when 
+		## a threshold crossed, and do extra stuff
+		check_threshold:     function(index: Index, default_thresh: count, 
+					custom_thresh: table[Index] of count, val: count ): bool &optional;
 	};
 	
 	## Function to associate a metric filter with a metric ID.
@@ -262,6 +270,11 @@ function add_filter(id: string, filter: Filter)
 		print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds";
 		return;
 		}
+	if ( !filter?$default_threshold && !filter?$custom_thresholds )
+		{
+		print "INVALID Metric filter: Must define one of $default_threshold and $custom_thresholds";
+		return;
+		}
 	
 	if ( ! filter?$id )
 		filter$id = id;
@@ -349,15 +362,43 @@ function add_unique(id: string, index: Index, data: string)
 	
 function check_notice(filter: Filter, index: Index, val: count): bool
 	{
-	if ( (filter?$notice_threshold &&
-	      [filter$id, filter$name, index] !in thresholds &&
-	      val >= filter$notice_threshold) ||
-	     (filter?$notice_thresholds &&
-	      |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
-	      val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
-		return T;
+	## It's possible for a user to skip defining either default_threshold or custom_thresholds.
+	## Therefore must check which one is missing, so we can craft and send a dummy value in the function
+
+	local cust_thresh: table[Index] of count;
+	local def_thresh = 0;
+
+	if ( filter?$custom_thresholds )
+		cust_thresh = filter$custom_thresholds;
+
+	if ( filter?$default_threshold )
+		def_thresh = filter$default_threshold;
+
+	if ( filter?$check_threshold )
+		return filter$check_threshold( index, def_thresh, cust_thresh, val );
+
 	else
+		{
+		if ( index in cust_thresh )
+			{
+			if ( val > cust_thresh[index] )
+				return T;
+			}
+		else if ( val > def_thresh)
+			return T;
+
 		return F;
+		}
+
+	#if ( (filter?$notice_threshold &&
+		 #     [filter$id, filter$name, index] !in thresholds &&
+		  #    val >= filter$notice_threshold) ||
+		   #  (filter?$notice_thresholds &&
+		    #  |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
+		     # val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
+			#return T;
+		#else
+			#return F;
 	}
 		
 function do_notice(filter: Filter, index: Index, val: count)
@@ -377,7 +418,12 @@ function do_notice(filter: Filter, index: Index, val: count)
 	# TODO: not sure where to put the network yet.
 	
 	NOTICE(n);
+
+	# Resetting unique values
+	local metric_tbl = store[filter$id, filter$name];
+	metric_tbl[index]$unique_vals = set();
 	
+
 	# This just needs set to some value so that it doesn't refire the 
 	# notice until it expires from the table or it crosses the next 
 	# threshold in the case of vectors of thresholds.

From d510702078e89a1c6914794f902437614aca48d2 Mon Sep 17 00:00:00 2001
From: Sheharbano Khattak <sheharbano.k@gmail.com>
Date: Tue, 16 Oct 2012 05:54:38 +0500
Subject: [PATCH 07/70] Added the branch /testing

---
 scripts/base/frameworks/metrics/cluster.bro   |  15 +-
 scripts/base/frameworks/metrics/main.bro      | 154 ++++++++++++------
 .../base/frameworks/metrics/non-cluster.bro   |   5 +-
 3 files changed, 116 insertions(+), 58 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 03bbcdf584..19f431460f 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -107,7 +107,7 @@ function data_added(filter: Filter, index: Index, val: count)
 	# intermediate update.
 	local pct_val = double_to_count(val / cluster_request_global_view_percent);
 	
-	if ( check_notice(filter, index, pct_val) ) 
+	if ( check_threshold(filter, index, pct_val) ) 
 		{
 		# kick off intermediate update
 		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val);
@@ -137,6 +137,9 @@ event Metrics::send_data(uid: string, id: string, filter_name: string, data: Met
 	# If data is empty, this metric is done.
 	if ( |data| == 0 )
 		done = T;
+
+	#print "Here is local_data";
+	#print local_data;
 	
 	event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
 	if ( ! done )
@@ -191,8 +194,9 @@ event Metrics::log_it(filter: Filter)
 # being collected by managers.
 function data_added(filter: Filter, index: Index, val: count)
 	{
-	if ( check_notice(filter, index, val) )
-		do_notice(filter, index, val);
+	if ( check_threshold(filter, index, val) )
+		threshold_crossed_alert( filter, index, val );
+		#do_notice(filter, index, val);
 	}
 	
 event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint)
@@ -206,8 +210,9 @@ event Metrics::cluster_index_response(uid: string, id: string, filter_name: stri
 	if ( Cluster::worker_count == done_with[uid] )
 		{
 		local size = ir?$num ? ir$num : |ir$unique_vals|;
-		if ( check_notice(filter_store[id, filter_name], index, size) )
-			do_notice(filter_store[id, filter_name], index, size);
+		if ( check_threshold(filter_store[id, filter_name], index, size) )
+			threshold_crossed_alert( filter_store[id, filter_name], index, size );
+			#do_notice(filter_store[id, filter_name], index, size);
 		delete done_with[uid];
 		delete index_requests[uid, id, filter_name, index];
 		}
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 744eaf731d..d4bfd8bd5f 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -14,7 +14,7 @@ export {
 	
 	## This is the interval for how often threshold based notices will happen 
 	## after they have already fired.
-	const renotice_interval = 1hr &redef;
+	const renotice_interval = 12hr &redef;
 	
 	## Represents a thing which is having metrics collected for it.  An instance
 	## of this record type and an id together represent a single measurement.
@@ -98,11 +98,7 @@ export {
 		log:               bool                    &default=T;
 		## If this and a $notice_threshold value are set, this notice type
 		## will be generated by the metrics framework.
-		note:              Notice::Type            &optional;
-		## A straight threshold for generating a notice.
-		notice_threshold:  count                   &optional;
-		## A series of thresholds at which to generate notices.
-		notice_thresholds: vector of count         &optional;
+		note:              Notice::Type            &optional;		
 		
 		## Sheharbano's additions
 		##--------------------------------------------
@@ -115,8 +111,20 @@ export {
 		custom_thresholds: table[Index] of count         &optional;
 		## A predicate so that you can decide when to flexibly declare when 
 		## a threshold crossed, and do extra stuff
-		check_threshold:     function(index: Index, default_thresh: count, 
+		custom_check_threshold:     function(index: Index, default_thresh: count, 
 					custom_thresh: table[Index] of count, val: count ): bool &optional;
+		## Even if custom_check_threshold has been defined, we don't want 
+		## to call it every time because of function call overhead.
+		## Metrics::Filter$trigger_custom_check_threshold describes how often
+		## custom_check_threshold will be called
+		## e.g. call custom_check_threshold for every 5 items seen by the metrics fw
+		trigger_custom_check_threshold: count   &default=1;
+		## A predicate that is called whenever a threshold is crossed
+		## ToDo: Also have a parameter here that is a sample of the
+		## observed trackable items 
+		threshold_crossed:     function(index: Index, val: count ) &optional;
+		## A series of thresholds at which to generate notices.
+		threshold_series: vector of count         &optional;
 	};
 	
 	## Function to associate a metric filter with a metric ID.
@@ -136,6 +144,13 @@ export {
 	##
 	## increment: How much to increment the counter by.
 	global add_data: function(id: string, index: Index, increment: count);
+
+	# This function does the following:
+	# If index (src,) doesn't exist, it creates an entry for this index. It
+	# adds data (c$id$orig_h) to a set associated with this index. If the number
+	# of unique data values for an index exceeds threshold, a notice is generated.
+	# So the threshold applies to the number of unique data values associated with
+	# an index.
 	
 	global add_unique: function(id: string, index: Index, data: string);
 	
@@ -173,15 +188,13 @@ global store: table[string, string] of MetricTable = table() &default=table();
 # This function checks if a threshold has been crossed and generates a 
 # notice if it has.  It is also used as a method to implement 
 # mid-break-interval threshold crossing detection for cluster deployments.
-global check_notice: function(filter: Filter, index: Index, val: count): bool;
-
+global check_threshold: function(filter: Filter, index: Index, val: count): bool;
 # This is hook for watching thresholds being crossed.  It is called whenever
 # index values are updated and the new val is given as the `val` argument.
 global data_added: function(filter: Filter, index: Index, val: count);
 
-# This stores the current threshold index for filters using the
-# $notice_threshold and $notice_thresholds elements.
-global thresholds: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0;
+# This stores the current threshold index for filters using $threshold_series.
+global threshold_series_index: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0;
 
 event bro_init() &priority=5
 	{
@@ -265,16 +278,23 @@ function add_filter(id: string, filter: Filter)
 		print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
 		return;
 		}
-	if ( filter?$notice_threshold && filter?$notice_thresholds )
+	if ( filter?$threshold_series && ( filter?$default_threshold || filter?$custom_thresholds ) )
 		{
-		print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds";
+		print "INVALID Metric filter: Cannot define $custom_thresholds and $default_threshold with $threshold_series";
 		return;
 		}
-	if ( !filter?$default_threshold && !filter?$custom_thresholds )
+	if ( !filter?$default_threshold && !filter?$custom_thresholds && !filter?$threshold_series )
 		{
-		print "INVALID Metric filter: Must define one of $default_threshold and $custom_thresholds";
+		print "INVALID Metric filter: Must define one of $default_threshold, $custom_thresholds and threshold_series";
 		return;
 		}
+
+	#Bro throws error anyway when a non-optional record field is missing
+	#if ( !filter?$threshold_crossed )
+	#	{
+	#	print "INVALID Metric filter: Must define the function $threshold_crossed";
+	#	return;
+	#	}
 	
 	if ( ! filter?$id )
 		filter$id = id;
@@ -345,6 +365,8 @@ function add_it(id: string, index: Index, integer_value: bool, num: count, str:
 				metric_tbl[index] = [$unique_vals=empty_ss];
 				}
 			add metric_tbl[index]$unique_vals[str];
+			#print metric_tbl[index]$unique_vals;
+			#print "-------------------------------------";
 			data_added(filter, index, |metric_tbl[index]$unique_vals|);
 			}
 		}
@@ -360,7 +382,7 @@ function add_unique(id: string, index: Index, data: string)
 	add_it(id, index, F, 0, data);
 	}
 	
-function check_notice(filter: Filter, index: Index, val: count): bool
+function check_threshold(filter: Filter, index: Index, val: count): bool
 	{
 	## It's possible for a user to skip defining either default_threshold or custom_thresholds.
 	## Therefore must check which one is missing, so we can craft and send a dummy value in the function
@@ -373,59 +395,89 @@ function check_notice(filter: Filter, index: Index, val: count): bool
 
 	if ( filter?$default_threshold )
 		def_thresh = filter$default_threshold;
+ 
+	if ( filter?$custom_check_threshold && ( val%filter$trigger_custom_check_threshold == 0 ) )
+		return filter$custom_check_threshold( index, def_thresh, cust_thresh, val );
 
-	if ( filter?$check_threshold )
-		return filter$check_threshold( index, def_thresh, cust_thresh, val );
-
-	else
+	# No custom check threshold defined
+	else if ( !filter?$custom_check_threshold )
 		{
 		if ( index in cust_thresh )
 			{
 			if ( val > cust_thresh[index] )
 				return T;
 			}
-		else if ( val > def_thresh)
-			return T;
+		else if ( filter?$default_threshold )
+			{
+			if ( val > def_thresh)
+				{
+				if ( index$str == "80/tcp")
+					print fmt("default threshold--val is %d for index %s",val,index);
+				return T;
+				}
+			}
 
+		else if ( filter?$threshold_series )
+			{
+			#print threshold_series_index[filter$id, filter$name, index];
+			if ( |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] &&
+			     val >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] )
+				{
+				if ( index$str == "80/tcp")
+					print fmt("series threshold--val is %d for index %s",val,index);
+				return T;
+				}
+			}
+	
 		return F;
 		}
+	#else if ( !filter?$custom_check_threshold )
+	#	{
+	#	if ( index in cust_thresh )
+	#		{
+	#		if ( val > cust_thresh[index] )
+	#			return T;
+	#		}
+	#	else if ( val > def_thresh)
+	#		return T;
+	#	}
 
-	#if ( (filter?$notice_threshold &&
-		 #     [filter$id, filter$name, index] !in thresholds &&
-		  #    val >= filter$notice_threshold) ||
-		   #  (filter?$notice_thresholds &&
-		    #  |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
-		     # val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
-			#return T;
-		#else
-			#return F;
+	return F;
 	}
 		
-function do_notice(filter: Filter, index: Index, val: count)
+function threshold_crossed_alert(filter: Filter, index: Index, val: count)
 	{
 	# We include $peer_descr here because the a manager count have actually 
 	# generated the notice even though the current remote peer for the event 
 	# calling this could be a worker if this is running as a cluster.
-	local n: Notice::Info = [$note=filter$note, 
-	                         $n=val, 
-	                         $metric_index=index, 
-	                         $peer_descr=peer_description];
-	n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
-	if ( index?$str )
-		n$sub = index$str;
-	if ( index?$host )
-		n$src = index$host;
+	#local n: Notice::Info = [$note=filter$note, 
+	#                        $n=val, 
+	#                         $metric_index=index, 
+	#                        $peer_descr=peer_description];
+	#n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
+	#if ( index?$str )
+	#	n$sub = index$str;
+	#if ( index?$host )
+	#	n$src = index$host;
 	# TODO: not sure where to put the network yet.
 	
-	NOTICE(n);
+	#NOTICE(n);
+
+	if ( filter?$threshold_crossed )
+		filter$threshold_crossed( index, val );
 
 	# Resetting unique values
-	local metric_tbl = store[filter$id, filter$name];
-	metric_tbl[index]$unique_vals = set();
-	
-
-	# This just needs set to some value so that it doesn't refire the 
-	# notice until it expires from the table or it crosses the next 
-	# threshold in the case of vectors of thresholds.
-	++thresholds[filter$id, filter$name, index];
+	#local metric_tbl = store[filter$id, filter$name];
+	#metric_tbl[index]$unique_vals = set();
+	# If I don't reset here, the value just keeps
+	# retriggering once the threshold has been exceeded
+	if ( !filter?$threshold_series )
+		reset(filter);
+	else
+		{
+		# This just needs set to some value so that it doesn't refire the 
+		# notice until it expires from the table or it crosses the next 
+		# threshold in the case of vectors of thresholds.
+		++threshold_series_index[filter$id, filter$name, index];
+		}
 	}
diff --git a/scripts/base/frameworks/metrics/non-cluster.bro b/scripts/base/frameworks/metrics/non-cluster.bro
index 85c050fb25..dc0daea2be 100644
--- a/scripts/base/frameworks/metrics/non-cluster.bro
+++ b/scripts/base/frameworks/metrics/non-cluster.bro
@@ -16,6 +16,7 @@ event Metrics::log_it(filter: Filter)
 	
 function data_added(filter: Filter, index: Index, val: count)
 	{
-	if ( check_notice(filter, index, val) )
-		do_notice(filter, index, val);
+	if ( check_threshold(filter, index, val) )
+		threshold_crossed_alert( filter, index, val );
+		#do_notice(filter, index, val);
 	}

From 6244bf43088c459450496fbead708bef980407d2 Mon Sep 17 00:00:00 2001
From: Sheharbano Khattak <sheharbano.k@gmail.com>
Date: Wed, 17 Oct 2012 04:13:13 +0500
Subject: [PATCH 08/70] Started the branch testing2

---
 scripts/base/frameworks/metrics/main.bro      | 118 ++++++------------
 .../base/frameworks/metrics/non-cluster.bro   |   1 -
 2 files changed, 35 insertions(+), 84 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index d4bfd8bd5f..ebe5170062 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -11,10 +11,14 @@ export {
 	## The default interval used for "breaking" metrics and writing the 
 	## current value to the logging stream.
 	const default_break_interval = 15mins &redef;
+
+	## The default number of metric items which trigger 
+	## filter$custom_check_threshold 
+	const default_trigger_custom_check_threshold = 10 &redef;
 	
 	## This is the interval for how often threshold based notices will happen 
 	## after they have already fired.
-	const renotice_interval = 12hr &redef;
+	const threshold_series_restart_interval = 1hr &redef;
 	
 	## Represents a thing which is having metrics collected for it.  An instance
 	## of this record type and an id together represent a single measurement.
@@ -96,33 +100,22 @@ export {
 		## thresholding and statistics gathering utility that is meant to
 		## never log but rather to generate notices and derive data.
 		log:               bool                    &default=T;
-		## If this and a $notice_threshold value are set, this notice type
-		## will be generated by the metrics framework.
-		note:              Notice::Type            &optional;		
-		
-		## Sheharbano's additions
-		##--------------------------------------------
 		## A straight threshold for generating a notice.
 		default_threshold:  count                   &optional;
-		## Represents Index specific thresholds, that is we can
-		## have different thresholds for different Index values.
-		## If the threshold for an Index is not specified in <dynamic_thresholds>,
-		## <threshold> will be used as default.
-		custom_thresholds: table[Index] of count         &optional;
 		## A predicate so that you can decide when to flexibly declare when 
 		## a threshold crossed, and do extra stuff
 		custom_check_threshold:     function(index: Index, default_thresh: count, 
-					custom_thresh: table[Index] of count, val: count ): bool &optional;
+						     val: count ): bool &optional;
 		## Even if custom_check_threshold has been defined, we don't want 
 		## to call it every time because of function call overhead.
 		## Metrics::Filter$trigger_custom_check_threshold describes how often
 		## custom_check_threshold will be called
-		## e.g. call custom_check_threshold for every 5 items seen by the metrics fw
-		trigger_custom_check_threshold: count   &default=1;
+		## e.g. call custom_check_threshold for every 10 items seen by the metrics fw
+		trigger_custom_check_threshold: count   &default=default_trigger_custom_check_threshold;
 		## A predicate that is called whenever a threshold is crossed
 		## ToDo: Also have a parameter here that is a sample of the
 		## observed trackable items 
-		threshold_crossed:     function(index: Index, val: count ) &optional;
+		threshold_crossed:     function(index: Index, val: count );
 		## A series of thresholds at which to generate notices.
 		threshold_series: vector of count         &optional;
 	};
@@ -194,7 +187,7 @@ global check_threshold: function(filter: Filter, index: Index, val: count): bool
 global data_added: function(filter: Filter, index: Index, val: count);
 
 # This stores the current threshold index for filters using $threshold_series.
-global threshold_series_index: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0;
+global threshold_series_index: table[string, string, Index] of count = {} &create_expire=threshold_series_restart_interval &default=0;
 
 event bro_init() &priority=5
 	{
@@ -278,23 +271,35 @@ function add_filter(id: string, filter: Filter)
 		print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
 		return;
 		}
-	if ( filter?$threshold_series && ( filter?$default_threshold || filter?$custom_thresholds ) )
+	if ( !filter?$threshold_series &&  !filter?$default_threshold )
 		{
-		print "INVALID Metric filter: Cannot define $custom_thresholds and $default_threshold with $threshold_series";
+		print "INVALID Metric filter: Must define one of $default_threshold and $threshold_series";
 		return;
 		}
-	if ( !filter?$default_threshold && !filter?$custom_thresholds && !filter?$threshold_series )
+	if ( filter?$threshold_series &&  filter?$custom_check_threshold )
 		{
-		print "INVALID Metric filter: Must define one of $default_threshold, $custom_thresholds and threshold_series";
+		print "INVALID Metric filter: Cannot define $custom_check_threshold with $threshold_series";
 		return;
 		}
-
-	#Bro throws error anyway when a non-optional record field is missing
-	#if ( !filter?$threshold_crossed )
-	#	{
-	#	print "INVALID Metric filter: Must define the function $threshold_crossed";
-	#	return;
-	#	}
+	if ( filter?$threshold_series &&  filter?$default_threshold )
+		{
+		print "INVALID Metric filter: Cannot define both $default_threshold and $threshold_series";
+		return;
+		}
+	if ( filter?$custom_check_threshold &&  !filter?$default_threshold )
+		{
+		print "INVALID Metric filter: Must define $default_threshold with $custom_check_threshold";
+		return;
+		}
+	if ( !filter?$trigger_custom_check_threshold &&  filter?$custom_check_threshold )
+		{
+		print "INVALID Metric filter: You defined $trigger_custom_check_threshold but $custom_check_threshold is missing";
+		return;
+		}
+	if ( !filter?$trigger_custom_check_threshold &&  filter?$custom_check_threshold )
+		{
+		print "WARNING Metric filter: You did not define $trigger_custom_check_threshold (default will be used)";
+		}
 	
 	if ( ! filter?$id )
 		filter$id = id;
@@ -384,91 +389,38 @@ function add_unique(id: string, index: Index, data: string)
 	
 function check_threshold(filter: Filter, index: Index, val: count): bool
 	{
-	## It's possible for a user to skip defining either default_threshold or custom_thresholds.
-	## Therefore must check which one is missing, so we can craft and send a dummy value in the function
-
-	local cust_thresh: table[Index] of count;
 	local def_thresh = 0;
 
-	if ( filter?$custom_thresholds )
-		cust_thresh = filter$custom_thresholds;
-
 	if ( filter?$default_threshold )
 		def_thresh = filter$default_threshold;
  
 	if ( filter?$custom_check_threshold && ( val%filter$trigger_custom_check_threshold == 0 ) )
-		return filter$custom_check_threshold( index, def_thresh, cust_thresh, val );
+		return filter$custom_check_threshold( index, def_thresh, val );
 
 	# No custom check threshold defined
 	else if ( !filter?$custom_check_threshold )
 		{
-		if ( index in cust_thresh )
-			{
-			if ( val > cust_thresh[index] )
-				return T;
-			}
-		else if ( filter?$default_threshold )
+		if ( filter?$default_threshold )
 			{
 			if ( val > def_thresh)
-				{
-				if ( index$str == "80/tcp")
-					print fmt("default threshold--val is %d for index %s",val,index);
 				return T;
-				}
 			}
 
 		else if ( filter?$threshold_series )
 			{
-			#print threshold_series_index[filter$id, filter$name, index];
 			if ( |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] &&
 			     val >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] )
-				{
-				if ( index$str == "80/tcp")
-					print fmt("series threshold--val is %d for index %s",val,index);
 				return T;
-				}
 			}
-	
-		return F;
 		}
-	#else if ( !filter?$custom_check_threshold )
-	#	{
-	#	if ( index in cust_thresh )
-	#		{
-	#		if ( val > cust_thresh[index] )
-	#			return T;
-	#		}
-	#	else if ( val > def_thresh)
-	#		return T;
-	#	}
-
 	return F;
 	}
 		
 function threshold_crossed_alert(filter: Filter, index: Index, val: count)
 	{
-	# We include $peer_descr here because the a manager count have actually 
-	# generated the notice even though the current remote peer for the event 
-	# calling this could be a worker if this is running as a cluster.
-	#local n: Notice::Info = [$note=filter$note, 
-	#                        $n=val, 
-	#                         $metric_index=index, 
-	#                        $peer_descr=peer_description];
-	#n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
-	#if ( index?$str )
-	#	n$sub = index$str;
-	#if ( index?$host )
-	#	n$src = index$host;
-	# TODO: not sure where to put the network yet.
-	
-	#NOTICE(n);
-
 	if ( filter?$threshold_crossed )
 		filter$threshold_crossed( index, val );
 
-	# Resetting unique values
-	#local metric_tbl = store[filter$id, filter$name];
-	#metric_tbl[index]$unique_vals = set();
 	# If I don't reset here, the value just keeps
 	# retriggering once the threshold has been exceeded
 	if ( !filter?$threshold_series )
diff --git a/scripts/base/frameworks/metrics/non-cluster.bro b/scripts/base/frameworks/metrics/non-cluster.bro
index dc0daea2be..ccd0414b2b 100644
--- a/scripts/base/frameworks/metrics/non-cluster.bro
+++ b/scripts/base/frameworks/metrics/non-cluster.bro
@@ -18,5 +18,4 @@ function data_added(filter: Filter, index: Index, val: count)
 	{
 	if ( check_threshold(filter, index, val) )
 		threshold_crossed_alert( filter, index, val );
-		#do_notice(filter, index, val);
 	}

From d5cf730b516eb9ef32986345642f141c154b1b4c Mon Sep 17 00:00:00 2001
From: Sheharbano Khattak <sheharbano.k@gmail.com>
Date: Thu, 18 Oct 2012 13:45:31 +0500
Subject: [PATCH 09/70] Added str to pred function

---
 scripts/base/frameworks/metrics/main.bro | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index ebe5170062..174b30a09a 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -81,7 +81,7 @@ export {
 		id:                string                  &optional;
 		## A predicate so that you can decide per index if you would like
 		## to accept the data being inserted.
-		pred:              function(index: Index): bool &optional;
+		pred:              function(index: Index, str: string): bool &optional;
 		## A function to normalize the index.  This can be used to normalize
 		## any field in the index and is likely most useful to normalize
 		## the $str field.
@@ -328,7 +328,7 @@ function add_it(id: string, index: Index, integer_value: bool, num: count, str:
 		
 		# If this filter has a predicate, run the predicate and skip this
 		# index if the predicate return false.
-		if ( filter?$pred && ! filter$pred(index) )
+		if ( filter?$pred && ! filter$pred(index,str) )
 			next;
 		
 		if ( index?$host )

From d9195076b17332420b9131272d66a246409efe50 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 16 Nov 2012 02:37:52 -0500
Subject: [PATCH 10/70] Metrics framework checkpoint.

 - New scan.bro merged in and reworked a bit.

 - Updated metrics API.  Now possible to calculate much more.
---
 scripts/base/frameworks/metrics/cluster.bro   | 105 ++--
 scripts/base/frameworks/metrics/main.bro      | 520 +++++++++++-------
 .../base/frameworks/metrics/non-cluster.bro   |  10 +-
 .../protocols/conn/conn-stats-per-host.bro    |  22 +
 scripts/policy/protocols/conn/scan.bro        | 320 +++++++++++
 scripts/policy/protocols/http/detect-sqli.bro |  35 +-
 .../protocols/ssh/detect-bruteforcing.bro     |  15 +-
 scripts/site/local.bro                        |   3 +
 .../manager-1..stdout                         |   1 +
 .../.stdout                                   |  10 +
 .../base/frameworks/metrics/basic-cluster.bro |  40 +-
 .../scripts/base/frameworks/metrics/basic.bro |  14 +-
 .../metrics/cluster-intermediate-update.bro   |  45 +-
 .../base/frameworks/metrics/notice.bro        |  20 -
 .../base/frameworks/metrics/thresholding.bro  |  47 ++
 15 files changed, 851 insertions(+), 356 deletions(-)
 create mode 100644 scripts/policy/protocols/conn/conn-stats-per-host.bro
 create mode 100644 scripts/policy/protocols/conn/scan.bro
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
 delete mode 100644 testing/btest/scripts/base/frameworks/metrics/notice.bro
 create mode 100644 testing/btest/scripts/base/frameworks/metrics/thresholding.bro

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 19f431460f..9650b80554 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -3,9 +3,6 @@
 ##! and will be depending on if the cluster framework has been enabled.
 ##! The goal of this script is to make metric calculation completely and
 ##! transparently automated when running on a cluster.
-##! 
-##! Events defined here are not exported deliberately because they are meant
-##! to be an internal implementation detail.
 
 @load base/frameworks/cluster
 @load ./main
@@ -24,36 +21,34 @@ export {
 	## since it may opt not to if it requested a global view for the index
 	## recently.
 	const cluster_request_global_view_percent = 0.1 &redef;
-	
-	## Event sent by the manager in a cluster to initiate the 
-	## collection of metrics values for a filter.
+
+	# Event sent by the manager in a cluster to initiate the 
+	# collection of metrics values for a filter.
 	global cluster_filter_request: event(uid: string, id: string, filter_name: string);
 
-	## Event sent by nodes that are collecting metrics after receiving
-	## a request for the metric filter from the manager.
+	# Event sent by nodes that are collecting metrics after receiving
+	# a request for the metric filter from the manager.
 	global cluster_filter_response: event(uid: string, id: string, filter_name: string, data: MetricTable, done: bool);
 
-	## This event is sent by the manager in a cluster to initiate the
-	## collection of a single index value from a filter.  It's typically
-	## used to get intermediate updates before the break interval triggers
-	## to speed detection of a value crossing a threshold.
+	# This event is sent by the manager in a cluster to initiate the
+	# collection of a single index value from a filter.  It's typically
+	# used to get intermediate updates before the break interval triggers
+	# to speed detection of a value crossing a threshold.
 	global cluster_index_request: event(uid: string, id: string, filter_name: string, index: Index);
 
-	## This event is sent by nodes in response to a 
-	## :bro:id:`Metrics::cluster_index_request` event.
-	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, data: DataPoint);
+	# This event is sent by nodes in response to a 
+	# :bro:id:`Metrics::cluster_index_request` event.
+	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, val: ResultVal);
 
-	## This is sent by workers to indicate that they crossed the percent of the 
-	## current threshold by the percentage defined globally in 
-	## :bro:id:`Metrics::cluster_request_global_view_percent`
-	global cluster_index_intermediate_response: event(id: string, filter_name: string, index: Metrics::Index, val: count);
+	# This is sent by workers to indicate that they crossed the percent of the 
+	# current threshold by the percentage defined globally in 
+	# :bro:id:`Metrics::cluster_request_global_view_percent`
+	global cluster_index_intermediate_response: event(id: string, filter_name: string, index: Metrics::Index);
 
-	## This event is scheduled internally on workers to send result chunks.
+	# This event is scheduled internally on workers to send result chunks.
 	global send_data: event(uid: string, id: string, filter_name: string, data: MetricTable);
-	
 }
 
-
 # This is maintained by managers so they can know what data they requested and
 # when they requested it.
 global requested_results: table[string] of time = table() &create_expire=5mins;
@@ -76,7 +71,7 @@ global done_with: table[string] of count &create_expire=5mins &default=0;
 
 # This variable is maintained by managers to track intermediate responses as 
 # they are getting a global view for a certain index.
-global index_requests: table[string, string, string, Index] of DataPoint &create_expire=5mins &default=[];
+global index_requests: table[string, string, string, Index] of ResultVal &create_expire=5mins &default=[];
 
 # This variable is maintained by all hosts for different purposes. Non-managers
 # maintain it to know what indexes they have recently sent as intermediate
@@ -95,23 +90,20 @@ redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # This is done on all non-manager node types in the event that a metric is 
 # being collected somewhere other than a worker.
-function data_added(filter: Filter, index: Index, val: count)
+function data_added(filter: Filter, index: Index, val: ResultVal)
 	{
 	# If an intermediate update for this value was sent recently, don't send
 	# it again.
 	if ( [filter$id, filter$name, index] in recent_global_view_indexes )
 		return;
-		
+
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an 
 	# intermediate update.
-	local pct_val = double_to_count(val / cluster_request_global_view_percent);
-	
-	if ( check_threshold(filter, index, pct_val) ) 
+	if ( check_thresholds(filter, index, val, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val);
-		
+		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index);
 		++recent_global_view_indexes[filter$id, filter$name, index];
 		}
 	}
@@ -137,9 +129,6 @@ event Metrics::send_data(uid: string, id: string, filter_name: string, data: Met
 	# If data is empty, this metric is done.
 	if ( |data| == 0 )
 		done = T;
-
-	#print "Here is local_data";
-	#print local_data;
 	
 	event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
 	if ( ! done )
@@ -148,7 +137,7 @@ event Metrics::send_data(uid: string, id: string, filter_name: string, data: Met
 
 event Metrics::cluster_filter_request(uid: string, id: string, filter_name: string)
 	{
-	#print fmt("WORKER %s: received the cluster_filter_request event.", Cluster::node);
+	#print fmt("WORKER %s: received the cluster_filter_request event for %s.", Cluster::node, id);
 	
 	# Initiate sending all of the data for the requested filter.
 	event Metrics::send_data(uid, id, filter_name, store[id, filter_name]);
@@ -160,12 +149,12 @@ event Metrics::cluster_filter_request(uid: string, id: string, filter_name: stri
 	
 event Metrics::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
 	{
-	local data: DataPoint;
-	if ( index in store[id, filter_name] )
-		data = store[id, filter_name][index];
-	
-	# fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val);
-	event Metrics::cluster_index_response(uid, id, filter_name, index, data);
+	if ( [id, filter_name] in store && index in store[id, filter_name] )
+		{
+		local data = store[id, filter_name][index];
+		#print fmt("WORKER %s: received the cluster_index_request event for %s=%s.", Cluster::node, index2str(index), data);
+		event Metrics::cluster_index_response(uid, id, filter_name, index, data);
+		}
 	}
 
 @endif
@@ -177,7 +166,6 @@ event Metrics::cluster_index_request(uid: string, id: string, filter_name: strin
 event Metrics::log_it(filter: Filter)
 	{
 	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
-	
 	local uid = unique_id("");
 	
 	# Set some tracking variables.
@@ -187,39 +175,44 @@ event Metrics::log_it(filter: Filter)
 	# Request data from peers.
 	event Metrics::cluster_filter_request(uid, filter$id, filter$name);
 	# Schedule the log_it event for the next break period.
-	schedule filter$break_interval { Metrics::log_it(filter) };
+	schedule filter$every { Metrics::log_it(filter) };
 	}
 
 # This is unlikely to be called often, but it's here in case there are metrics
 # being collected by managers.
-function data_added(filter: Filter, index: Index, val: count)
+function data_added(filter: Filter, index: Index, val: ResultVal)
 	{
-	if ( check_threshold(filter, index, val) )
-		threshold_crossed_alert( filter, index, val );
-		#do_notice(filter, index, val);
+	if ( check_thresholds(filter, index, val, 1.0) )
+		threshold_crossed(filter, index, val);
 	}
 	
-event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint)
+event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: ResultVal)
 	{
-	#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
+	#print fmt("%0.6f MANAGER: receiving index data from %s - %s=%s", network_time(), get_event_peer()$descr, index2str(index), val);
 	
-	index_requests[uid, id, filter_name, index] = merge_data_points(index_requests[uid, id, filter_name, index], data);
+	local merged_val = merge_result_vals(index_requests[uid, id, filter_name, index], val);
+	index_requests[uid, id, filter_name, index] = merged_val;
 	local ir = index_requests[uid, id, filter_name, index];
 	
+	# Mark that this worker is done.
 	++done_with[uid];
+
+	#print ir;
+	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
+
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		local size = ir?$num ? ir$num : |ir$unique_vals|;
-		if ( check_threshold(filter_store[id, filter_name], index, size) )
-			threshold_crossed_alert( filter_store[id, filter_name], index, size );
-			#do_notice(filter_store[id, filter_name], index, size);
+		if ( check_thresholds(filter_store[id, filter_name], index, ir, 1.0) )
+			{
+			threshold_crossed(filter_store[id, filter_name], index, ir);
+			}
 		delete done_with[uid];
 		delete index_requests[uid, id, filter_name, index];
 		}
 	}
 
 # Managers handle intermediate updates here.
-event Metrics::cluster_index_intermediate_response(id: string, filter_name: string, index: Index, val: count)
+event Metrics::cluster_index_intermediate_response(id: string, filter_name: string, index: Index)
 	{
 	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting index data for %s", index2str(index));
@@ -237,12 +230,12 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 	for ( index in data )
 		{
 		if ( index in local_data )
-			local_data[index] = merge_data_points(local_data[index], data[index]);
+			local_data[index] = merge_result_vals(local_data[index], data[index]);
 		else
 			local_data[index] = data[index];
 		}
 	
-		# Mark another worker as being "done" for this uid.
+	# Mark another worker as being "done" for this uid.
 	if ( done )
 		++done_with[uid];
 	
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 174b30a09a..8e40e76e02 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -1,7 +1,5 @@
 ##! The metrics framework provides a way to count and measure data.  
 
-@load base/frameworks/notice
-
 module Metrics;
 
 export {
@@ -11,15 +9,21 @@ export {
 	## The default interval used for "breaking" metrics and writing the 
 	## current value to the logging stream.
 	const default_break_interval = 15mins &redef;
-
-	## The default number of metric items which trigger 
-	## filter$custom_check_threshold 
-	const default_trigger_custom_check_threshold = 10 &redef;
 	
 	## This is the interval for how often threshold based notices will happen 
 	## after they have already fired.
 	const threshold_series_restart_interval = 1hr &redef;
 	
+	type Calculation: enum {
+		SUM,
+		MIN,
+		MAX,
+		VARIANCE,
+		STD_DEV,
+		AVG,
+		UNIQUE,
+	};
+
 	## Represents a thing which is having metrics collected for it.  An instance
 	## of this record type and an id together represent a single measurement.
 	type Index: record {
@@ -40,12 +44,47 @@ export {
 		network:      subnet &optional;
 	} &log;
 	
-	## Represents data being added for a single metric data point.  Used internally.
+	## Represents data being added for a single metric data point.
 	type DataPoint: record {
-		num:        count       &optional;
-		unique_vals: set[string] &optional;
+		num:       count       &optional;
+		dbl:       double      &optional;
+		str:       string      &optional;
 	};
-	
+
+	## Value supplied when a metric is finished.  It contains all
+	## of the measurements collected for the metric.
+	type ResultVal: record {
+		## The number of measurements received.
+		num:     count        &log &default=0;
+
+		## For numeric data, this tracks the sum of all values.
+		sum:     double       &log &optional;
+
+		## For numeric data, this tracks the minimum value given.
+		min:     double       &log &optional;
+
+		## For numeric data, this tracks the maximum value given.
+		max:     double       &log &optional;
+
+		## For numeric data, this calculates the average of all values.
+		avg:     double       &log &optional;
+
+		## For numeric data, this calculates the variance.
+		variance: double      &log &optional;
+
+		## For numeric data, this calculates the standard deviation.
+		std_dev:  double      &log &optional;
+
+		## If cardinality is being tracked, the number of unique
+		## items is tracked here.
+		unique: count         &log &optional;
+
+		## A sample of something being measured.  This is helpful in 
+		## some cases for collecting information to do further detection
+		## or better logging for forensic purposes.
+		sample: set[DataPoint]   &optional;
+	};
+
 	## The record type that is used for logging metrics.
 	type Info: record {
 		## Timestamp at which the metric was "broken".
@@ -57,67 +96,58 @@ export {
 		## the data so this is necessary to understand the value.
 		filter_name:  string   &log;
 		## What measurement the metric represents.
-		metric_id:    string   &log;
+		metric:       string   &log;
 		## What the metric value applies to.
 		index:        Index    &log;
 		## The simple numeric value of the metric.
-		value:        count    &log;
+		result:        ResultVal    &log;
 	};
 	
-    # TODO: configure a metrics filter logging stream to log the current
-	#       metrics configuration in case someone is looking through
-	#       old logs and the configuration has changed since then.
-	
 	## Filters define how the data from a metric is aggregated and handled.  
-	## Filters can be used to set how often the measurements are cut or "broken"
+	## Filters can be used to set how often the measurements are cut 
 	## and logged or how the data within them is aggregated.  It's also 
-	## possible to disable logging and use filters for thresholding.
+	## possible to disable logging and use filters solely for thresholding.
 	type Filter: record {
 		## The name for this filter so that multiple filters can be
 		## applied to a single metrics to get a different view of the same
 		## metric data being collected (different aggregation, break, etc).
-		name:              string                  &default="default";
-		## The :bro:type:`Metrics::ID` that this filter applies to.
-		id:                string                  &optional;
+		name:              string                   &default="default";
+		## The metric that this filter applies to.
+		id:                string                   &optional;
+		## The measurements to perform on the data.
+		measure:           set[Calculation]         &optional;
 		## A predicate so that you can decide per index if you would like
 		## to accept the data being inserted.
-		pred:              function(index: Index, str: string): bool &optional;
-		## A function to normalize the index.  This can be used to normalize
-		## any field in the index and is likely most useful to normalize
-		## the $str field.
-		normalize_func:    function(index: Index): Index &optional;
-		## Global mask by which you'd like to aggregate traffic.
-		aggregation_mask:  count                   &optional;
-		## This is essentially a mapping table between addresses and subnets.
-		aggregation_table: table[subnet] of subnet &optional;
+		pred:              function(index: Metrics::Index, data: DataPoint): bool &optional;
+		## A function to normalize the index.  This can be used to aggregate or
+		## normalize the entire index.
+		normalize_func:    function(index: Metrics::Index): Index &optional;
+		## Global mask by to aggregate traffic measuring an attribute of hosts.
+		## This is a special case of the normalize_func.
+		aggregation_mask:  count                    &optional;
 		## The interval at which this filter should be "broken" and written
 		## to the logging stream.  The counters are also reset to zero at 
 		## this time so any threshold based detection needs to be set to a 
 		## number that should be expected to happen within this period.
-		break_interval:    interval                &default=default_break_interval;
+		every:             interval                 &default=default_break_interval;
 		## This determines if the result of this filter is sent to the metrics
 		## logging stream.  One use for the logging framework is as an internal
 		## thresholding and statistics gathering utility that is meant to
 		## never log but rather to generate notices and derive data.
-		log:               bool                    &default=T;
-		## A straight threshold for generating a notice.
-		default_threshold:  count                   &optional;
+		log:               bool                     &default=T;
+		## A direct threshold for calling the $threshold_crossed function when 
+		## the SUM is greater than or equal to this value.
+		threshold:         count                    &optional;
+		## A series of thresholds for calling the $threshold_crossed function.
+		threshold_series:  vector of count          &optional;
 		## A predicate so that you can decide when to flexibly declare when 
-		## a threshold crossed, and do extra stuff
-		custom_check_threshold:     function(index: Index, default_thresh: count, 
-						     val: count ): bool &optional;
-		## Even if custom_check_threshold has been defined, we don't want 
-		## to call it every time because of function call overhead.
-		## Metrics::Filter$trigger_custom_check_threshold describes how often
-		## custom_check_threshold will be called
-		## e.g. call custom_check_threshold for every 10 items seen by the metrics fw
-		trigger_custom_check_threshold: count   &default=default_trigger_custom_check_threshold;
-		## A predicate that is called whenever a threshold is crossed
-		## ToDo: Also have a parameter here that is a sample of the
-		## observed trackable items 
-		threshold_crossed:     function(index: Index, val: count );
-		## A series of thresholds at which to generate notices.
-		threshold_series: vector of count         &optional;
+		## a threshold crossed, and do extra work.
+		threshold_func:    function(index: Metrics::Index, val: Metrics::ResultVal): bool &optional;
+		## A function callback that is called when a threshold is crossed.
+		threshold_crossed: function(index: Metrics::Index, val: Metrics::ResultVal) &optional;
+		## A number of sample DataPoints to collect for the threshold 
+		## crossing callback.
+		samples:           count                    &optional;
 	};
 	
 	## Function to associate a metric filter with a metric ID.
@@ -125,70 +155,72 @@ export {
 	## id: The metric ID that the filter should be associated with.
 	##
 	## filter: The record representing the filter configuration.
-	global add_filter: function(id: string, filter: Filter);
+	global add_filter: function(id: string, filter: Metrics::Filter);
 	
-	## Add data into a :bro:type:`Metrics::ID`.  This should be called when
+	## Add data into a metric.  This should be called when
 	## a script has measured some point value and is ready to increment the
 	## counters.
 	##
-	## id: The metric ID that the data represents.
+	## id: The metric identifier that the data represents.
 	##
 	## index: The metric index that the value is to be added to.
 	##
 	## increment: How much to increment the counter by.
-	global add_data: function(id: string, index: Index, increment: count);
-
-	# This function does the following:
-	# If index (src,) doesn't exist, it creates an entry for this index. It
-	# adds data (c$id$orig_h) to a set associated with this index. If the number
-	# of unique data values for an index exceeds threshold, a notice is generated.
-	# So the threshold applies to the number of unique data values associated with
-	# an index.
-	
-	global add_unique: function(id: string, index: Index, data: string);
+	global add_data: function(id: string, index: Metrics::Index, data: Metrics::DataPoint);
 	
 	## Helper function to represent a :bro:type:`Metrics::Index` value as 
-	## a simple string
+	## a simple string.
 	## 
 	## index: The metric index that is to be converted into a string.
 	##
 	## Returns: A string reprentation of the metric index.
-	global index2str: function(index: Index): string;
-	
-	## Event that is used to "finish" metrics and adapt the metrics
-	## framework for clustered or non-clustered usage.
-	##
-	## ..note: This is primarily intended for internal use.
-	global log_it: event(filter: Filter);
-	
+	global index2str: function(index: Metrics::Index): string;
+		
 	## Event to access metrics records as they are passed to the logging framework.
-	global log_metrics: event(rec: Info);
+	global log_metrics: event(rec: Metrics::Info);
 	
-	## Type to store a table of metrics values.  Interal use only!
-	type MetricTable: table[Index] of DataPoint;
 }
 
-redef record Notice::Info += {
-	metric_index: Index &log &optional;
+redef record ResultVal += {
+	# Internal use only.  Used for incrementally calculating variance.
+	prev_avg:      double      &optional;
+
+	# Internal use only.  For calculating variance.
+	var_s:         double      &optional;
+
+	# Internal use only.  This is not meant to be publically available 
+	# because we don't want to trust that we can inspect the values 
+	# since we will like move to a probalistic data structure in the future.
+	# TODO: in the future this will optionally be a hyperloglog structure
+	unique_vals:  set[DataPoint] &optional;
 };
 
+# Type to store a table of metrics values.
+type MetricTable: table[Index] of ResultVal;
+
+# Store the filters indexed on the metric identifier.
 global metric_filters: table[string] of vector of Filter = table();
+
+# Store the filters indexed on the metric identifier and filter name.
 global filter_store: table[string, string] of Filter = table();
 
-# This is indexed by metric ID and stream filter name.
+# This is indexed by metric id and filter name.
 global store: table[string, string] of MetricTable = table() &default=table();
 
-# This function checks if a threshold has been crossed and generates a 
-# notice if it has.  It is also used as a method to implement 
-# mid-break-interval threshold crossing detection for cluster deployments.
-global check_threshold: function(filter: Filter, index: Index, val: count): bool;
-# This is hook for watching thresholds being crossed.  It is called whenever
-# index values are updated and the new val is given as the `val` argument.
-global data_added: function(filter: Filter, index: Index, val: count);
-
 # This stores the current threshold index for filters using $threshold_series.
 global threshold_series_index: table[string, string, Index] of count = {} &create_expire=threshold_series_restart_interval &default=0;
 
+# This is hook for watching thresholds being crossed.  It is called whenever
+# index values are updated and the new val is given as the `val` argument.
+# It's only prototyped here because cluster and non-cluster has separate 
+# implementations.
+global data_added: function(filter: Filter, index: Index, val: ResultVal);
+
+## Event that is used to "finish" metrics and adapt the metrics
+## framework for clustered or non-clustered usage.
+global log_it: event(filter: Metrics::Filter);
+
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Metrics::LOG, [$columns=Info, $ev=log_metrics]);
@@ -206,29 +238,91 @@ function index2str(index: Index): string
 	return fmt("metric_index(%s)", out);
 	}
 	
-function merge_data_points(dp1: DataPoint, dp2: DataPoint): DataPoint
+function do_calculated_fields(val: ResultVal)
 	{
-	local result: DataPoint;
-	if ( dp1?$num || dp2?$num )
+	if ( val?$unique_vals )
+		val$unique = |val$unique_vals|;
+	if ( val?$var_s )
+		val$variance = (val$num > 1) ? val$var_s/val$num : 0.0;
+	if ( val?$variance )
+		val$std_dev = sqrt(val$variance);
+	}
+
+function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
+	{
+	local result: ResultVal;
+	
+	# Merge $num
+	result$num = rv1$num + rv2$num;
+
+	# Merge $sum
+	if ( rv1?$sum || rv2?$sum )
 		{
-		result$num = 0;
-		if ( dp1?$num )
-			result$num += dp1$num;
-		if ( dp2?$num )
-			result$num += dp2$num;
+		result$sum = 0;
+		if ( rv1?$sum )
+			result$sum += rv1$sum;
+		if ( rv2?$sum )
+			result$sum += rv2$sum;
 		}
-		
-	if ( dp1?$unique_vals || dp2?$unique_vals )
+	
+	# Merge $max
+	if ( rv1?$max && rv2?$max )
+		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
+	else if ( rv1?$max )
+		result$max = rv1$max;
+	else if ( rv2?$max )
+		result$max = rv2$max;
+
+	# Merge $min
+	if ( rv1?$min && rv2?$min )
+		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
+	else if ( rv1?$min )
+		result$min = rv1$min;
+	else if ( rv2?$min )
+		result$min = rv2$min;
+
+	# Merge $avg
+	if ( rv1?$avg && rv2?$avg )
+		result$avg = ((rv1$avg*rv1$num) + (rv2$avg*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$avg )
+		result$avg = rv1$avg;
+	else if ( rv2?$avg )
+		result$avg = rv2$avg;
+
+	# Merge $prev_avg
+	if ( rv1?$prev_avg && rv2?$prev_avg )
+		result$prev_avg = ((rv1$prev_avg*rv1$num) + (rv2$prev_avg*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$prev_avg )
+		result$prev_avg = rv1$prev_avg;
+	else if ( rv2?$prev_avg )
+		result$prev_avg = rv2$prev_avg;
+
+	# Merge $var_s
+	if ( rv1?$var_s && rv2?$var_s )
+		{
+		local rv1_avg_sq = (rv1$avg - result$avg);
+		rv1_avg_sq = rv1_avg_sq*rv1_avg_sq;
+		local rv2_avg_sq = (rv2$avg - result$avg);
+		rv2_avg_sq = rv2_avg_sq*rv2_avg_sq;
+		result$var_s = rv1$num*(rv1$var_s/rv1$num + rv1_avg_sq) + rv2$num*(rv2$var_s/rv2$num + rv2_avg_sq);
+		}
+	else if ( rv1?$var_s )
+		result$var_s = rv1$var_s;
+	else if ( rv2?$var_s )
+		result$var_s = rv2$var_s;
+
+	if ( rv1?$unique_vals || rv2?$unique_vals )
 		{
 		result$unique_vals = set();
-		if ( dp1?$unique_vals )
-			for ( val1 in dp1$unique_vals )
+		if ( rv1?$unique_vals )
+			for ( val1 in rv1$unique_vals )
 				add result$unique_vals[val1];
-		if ( dp2?$unique_vals )
-			for ( val2 in dp2$unique_vals )
+		if ( rv2?$unique_vals )
+			for ( val2 in rv2$unique_vals )
 				add result$unique_vals[val2];
-			}
-			
+		}
+	
+	do_calculated_fields(result);
 	return result;
 	}
 	
@@ -236,24 +330,18 @@ function write_log(ts: time, filter: Filter, data: MetricTable)
 	{
 	for ( index in data )
 		{
-		local val = 0;
-		if ( data[index]?$unique_vals )
-			val = |data[index]$unique_vals|;
-		else
-			val = data[index]$num;
 		local m: Info = [$ts=ts,
-		                 $ts_delta=filter$break_interval,
-		                 $metric_id=filter$id,
+		                 $ts_delta=filter$every,
+		                 $metric=filter$id,
 		                 $filter_name=filter$name,
 		                 $index=index,
-		                 $value=val];
+		                 $result=data[index]];
 		
 		if ( filter$log )
 			Log::write(Metrics::LOG, m);
 		}
 	}
 
-
 function reset(filter: Filter)
 	{
 	store[filter$id, filter$name] = table();
@@ -261,45 +349,16 @@ function reset(filter: Filter)
 
 function add_filter(id: string, filter: Filter)
 	{
-	if ( filter?$aggregation_table && filter?$aggregation_mask )
+	if ( filter?$normalize_func && filter?$aggregation_mask )
 		{
-		print "INVALID Metric filter: Defined $aggregation_table and $aggregation_mask.";
+		Reporter::warning(fmt("invalid Metric filter (%s): Defined $normalize_func and $aggregation_mask.", filter$name));
 		return;
 		}
 	if ( [id, filter$name] in store )
 		{
-		print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
+		Reporter::warning(fmt("invalid Metric filter (%s): Filter with same name already exists.", filter$name));
 		return;
 		}
-	if ( !filter?$threshold_series &&  !filter?$default_threshold )
-		{
-		print "INVALID Metric filter: Must define one of $default_threshold and $threshold_series";
-		return;
-		}
-	if ( filter?$threshold_series &&  filter?$custom_check_threshold )
-		{
-		print "INVALID Metric filter: Cannot define $custom_check_threshold with $threshold_series";
-		return;
-		}
-	if ( filter?$threshold_series &&  filter?$default_threshold )
-		{
-		print "INVALID Metric filter: Cannot define both $default_threshold and $threshold_series";
-		return;
-		}
-	if ( filter?$custom_check_threshold &&  !filter?$default_threshold )
-		{
-		print "INVALID Metric filter: Must define $default_threshold with $custom_check_threshold";
-		return;
-		}
-	if ( !filter?$trigger_custom_check_threshold &&  filter?$custom_check_threshold )
-		{
-		print "INVALID Metric filter: You defined $trigger_custom_check_threshold but $custom_check_threshold is missing";
-		return;
-		}
-	if ( !filter?$trigger_custom_check_threshold &&  filter?$custom_check_threshold )
-		{
-		print "WARNING Metric filter: You did not define $trigger_custom_check_threshold (default will be used)";
-		}
 	
 	if ( ! filter?$id )
 		filter$id = id;
@@ -311,10 +370,10 @@ function add_filter(id: string, filter: Filter)
 	filter_store[id, filter$name] = filter;
 	store[id, filter$name] = table();
 	
-	schedule filter$break_interval { Metrics::log_it(filter) };
+	schedule filter$every { Metrics::log_it(filter) };
 	}
 
-function add_it(id: string, index: Index, integer_value: bool, num: count, str: string)
+function add_data(id: string, index: Index, data: DataPoint)
 	{
 	if ( id !in metric_filters )
 		return;
@@ -328,103 +387,140 @@ function add_it(id: string, index: Index, integer_value: bool, num: count, str:
 		
 		# If this filter has a predicate, run the predicate and skip this
 		# index if the predicate return false.
-		if ( filter?$pred && ! filter$pred(index,str) )
+		if ( filter?$pred && ! filter$pred(index, data) )
 			next;
 		
-		if ( index?$host )
+		if ( filter?$normalize_func )
+			index = filter$normalize_func(copy(index));
+
+		if ( index?$host && filter?$aggregation_mask )
 			{
-			if ( filter?$normalize_func )
-				{
-				index = filter$normalize_func(copy(index));
-				}
-			
-			if ( filter?$aggregation_mask )
-				{
-				index$network = mask_addr(index$host, filter$aggregation_mask);
-				delete index$host;
-				}
-			else if ( filter?$aggregation_table )
-				{
-				# Don't add the data if the aggregation table doesn't include 
-				# the given host address.
-				if ( index$host !in filter$aggregation_table )
-					return;
-				index$network = filter$aggregation_table[index$host];
-				delete index$host;
-				}
+			index$network = mask_addr(index$host, filter$aggregation_mask);
+			delete index$host;
 			}
 		
 		local metric_tbl = store[id, filter$name];
-		if ( integer_value )
+		if ( index !in metric_tbl )
+			metric_tbl[index] = [];
+
+		local result = metric_tbl[index];
+
+		# If a string was given, fall back to 1.0 as the value.
+		local val = 1.0;
+		if ( data?$num || data?$dbl )
+			val = data?$dbl ? data$dbl : data$num;
+
+		++result$num;
+
+		if ( SUM in filter$measure )
 			{
-			if ( index !in metric_tbl )
-				metric_tbl[index] = [$num=0];
-			metric_tbl[index]$num += num;
-			data_added(filter, index, metric_tbl[index]$num);
+			if ( ! result?$sum ) result$sum = 0;
+			result$sum += val;
 			}
-		else
+
+		if ( MIN in filter$measure )
 			{
-			if ( index !in metric_tbl )
+			if ( ! result?$min ) 
+				result$min = val;
+			else if (val < result$min) 
+				result$min = val;
+			}
+
+		if ( MAX in filter$measure )
+			{
+			if ( ! result?$max ) 
+				result$max = val;
+			else if (val > result$max) 
+				result$max = val;
+			}
+	
+		if ( AVG in filter$measure || VARIANCE in filter$measure )
+			{
+			if ( ! result?$avg ) 
 				{
-				local empty_ss: set[string] = set();
-				metric_tbl[index] = [$unique_vals=empty_ss];
+				result$avg = val;
+				result$prev_avg = val;
+				}
+			else
+				{
+				result$prev_avg = result$avg;
+				result$avg += (val - result$avg) / result$num;
 				}
-			add metric_tbl[index]$unique_vals[str];
-			#print metric_tbl[index]$unique_vals;
-			#print "-------------------------------------";
-			data_added(filter, index, |metric_tbl[index]$unique_vals|);
 			}
+
+		if ( VARIANCE in filter$measure )
+			{
+			if ( ! result?$var_s ) result$var_s = 0.0;
+			result$var_s += (val - result$prev_avg)*(val - result$avg);
+			}
+
+		if ( STD_DEV in filter$measure )
+			{
+			#if ( result?$variance )
+			#	result$std_dev = sqrt(result$variance);
+			}
+
+		if ( UNIQUE in filter$measure )
+			{
+			if ( ! result?$unique_vals ) result$unique_vals=set();
+			add result$unique_vals[data];
+			}
+
+		do_calculated_fields(result);
+		data_added(filter, index, result);
 		}
 	}
 
-function add_data(id: string, index: Index, increment: count)
+# This function checks if a threshold has been crossed and generates a 
+# notice if it has.  It is also used as a method to implement 
+# mid-break-interval threshold crossing detection for cluster deployments.
+function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_pct: double): bool
 	{
-	add_it(id, index, T, increment, "");
-	}
-	
-function add_unique(id: string, index: Index, data: string)
-	{
-	add_it(id, index, F, 0, data);
-	}
-	
-function check_threshold(filter: Filter, index: Index, val: count): bool
-	{
-	local def_thresh = 0;
+	local watch = 0.0;
+	if ( val?$unique )
+		watch = val$unique;
+	else if ( val?$sum )
+		watch = val$sum;
 
-	if ( filter?$default_threshold )
-		def_thresh = filter$default_threshold;
- 
-	if ( filter?$custom_check_threshold && ( val%filter$trigger_custom_check_threshold == 0 ) )
-		return filter$custom_check_threshold( index, def_thresh, val );
+	if ( modify_pct < 1.0 && modify_pct > 0.0 )
+		watch = watch/modify_pct;
 
-	# No custom check threshold defined
-	else if ( !filter?$custom_check_threshold )
+	if ( filter?$threshold && watch >= filter$threshold )
 		{
-		if ( filter?$default_threshold )
-			{
-			if ( val > def_thresh)
-				return T;
-			}
-
-		else if ( filter?$threshold_series )
-			{
-			if ( |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] &&
-			     val >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] )
-				return T;
-			}
+		# A default threshold was given and the value crossed it.
+		return T;
 		}
+
+	if ( filter?$threshold_series &&
+	     |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] &&
+	     watch >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] )
+		{
+		# A threshold series was given and the value crossed the next 
+		# value in the series.
+		return T;
+		}
+
+	if ( filter?$threshold_func &&
+	     filter$threshold_func(index, val) )
+		{
+		# The threshold function indicated it was crossed.
+		return T;
+		}
+
 	return F;
 	}
 		
-function threshold_crossed_alert(filter: Filter, index: Index, val: count)
+function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
 	{
 	if ( filter?$threshold_crossed )
-		filter$threshold_crossed( index, val );
+		filter$threshold_crossed(index, val);
 
 	# If I don't reset here, the value just keeps
-	# retriggering once the threshold has been exceeded
+	# retriggering once the threshold has been exceeded.
 	if ( !filter?$threshold_series )
+		{
 		reset(filter);
+		}
 	else
 		{
 		# This just needs set to some value so that it doesn't refire the 
diff --git a/scripts/base/frameworks/metrics/non-cluster.bro b/scripts/base/frameworks/metrics/non-cluster.bro
index ccd0414b2b..a94370d776 100644
--- a/scripts/base/frameworks/metrics/non-cluster.bro
+++ b/scripts/base/frameworks/metrics/non-cluster.bro
@@ -6,16 +6,16 @@ event Metrics::log_it(filter: Filter)
 	{
 	local id = filter$id;
 	local name = filter$name;
-	
+
 	write_log(network_time(), filter, store[id, name]);
 	reset(filter);
 	
-	schedule filter$break_interval { Metrics::log_it(filter) };
+	schedule filter$every { Metrics::log_it(filter) };
 	}
 	
 	
-function data_added(filter: Filter, index: Index, val: count)
+function data_added(filter: Filter, index: Index, val: ResultVal)
 	{
-	if ( check_threshold(filter, index, val) )
-		threshold_crossed_alert( filter, index, val );
+	if ( check_thresholds(filter, index, val, 1.0) )
+		threshold_crossed(filter, index, val);
 	}
diff --git a/scripts/policy/protocols/conn/conn-stats-per-host.bro b/scripts/policy/protocols/conn/conn-stats-per-host.bro
new file mode 100644
index 0000000000..9e532b8590
--- /dev/null
+++ b/scripts/policy/protocols/conn/conn-stats-per-host.bro
@@ -0,0 +1,22 @@
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter("conn.orig.data", 
+	                    [$every=5mins,
+	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
+	Metrics::add_filter("conn.resp.data", 
+	                    [$every=5mins,
+	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
+	}
+
+
+event connection_state_remove(c: connection)
+	{
+	if ( ! (c$conn$conn_state == "SF" && c$conn$proto == tcp) )
+		return;
+
+	if ( Site::is_local_addr(c$id$orig_h) )
+		Metrics::add_data("conn.orig.data", [$host=c$id$orig_h], [$num=c$orig$size]);
+	if ( Site::is_local_addr(c$id$resp_h) )
+		Metrics::add_data("conn.resp.data", [$host=c$id$resp_h], [$num=c$resp$size]);
+	}
\ No newline at end of file
diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro
new file mode 100644
index 0000000000..4df2697092
--- /dev/null
+++ b/scripts/policy/protocols/conn/scan.bro
@@ -0,0 +1,320 @@
+##! Scan detection
+##!
+##! ..Authors: Sheharbano Kattack
+##!            Seth Hall
+##!            All the authors of the old scan.bro
+
+module Scan;
+
+export {
+
+	redef enum Notice::Type += {
+		AddressScan,
+		PortScan,
+		};
+
+	const analyze_addr_scan = T &redef;
+	const analyze_port_scan = T &redef;
+
+	## Interval at which to watch for the
+	## :bro:id:`Scan::conn_failed_(port|addr)_threshold` variable to be crossed.
+	## At the end of each interval the counter is reset.
+	const conn_failed_addr_interval = 5min &redef;
+	const conn_failed_port_interval = 5min &redef;
+
+	const default_addr_scan_threshold = 25 &redef;
+	const default_port_scan_threshold = 15 &redef;
+
+	# For address scan
+	const suppress_UDP_scan_checks  = T &redef;
+	const suppress_TCP_scan_checks  = F &redef;
+	const suppress_ICMP_scan_checks = T &redef;
+	
+	global addr_scan_thresh_series: vector of count = vector(100, 200, 300);
+	global port_scan_thresh_series: vector of count = vector(10, 20, 30);
+
+	# Custom threholds based on service for address scan
+	const addr_scan_custom_thresholds: table[port] of count &redef;
+}
+
+function is_failed_conn(c: connection): bool
+	{
+	# Sr || ( (hR || ShR) && (data not sent in any direction) ) 
+	if ( (c$orig$state == TCP_SYN_SENT && c$resp$state == TCP_RESET) ||
+	     (
+	      ((c$orig$state == TCP_RESET && c$resp$state == TCP_SYN_ACK_SENT) ||
+	       (c$orig$state == TCP_RESET && c$resp$state == TCP_ESTABLISHED && "S" in c$history )
+	      ) &&
+	      !("D" in c$history || "d" in c$history)
+	     ) )
+		return T;
+	return F;
+	}
+
+function is_reverse_failed_conn(c: connection): bool
+	{
+	# reverse scan i.e. conn dest is the scanner
+	# sR || ( (Hr || sHr) && (data not sent in any direction) ) 
+	if ( (c$resp$state == TCP_SYN_SENT && c$orig$state == TCP_RESET) ||
+	     (
+	      ((c$resp$state == TCP_RESET && c$orig$state == TCP_SYN_ACK_SENT) ||
+	       (c$resp$state == TCP_RESET && c$orig$state == TCP_ESTABLISHED && "s" in c$history )
+	      ) &&
+	      !("D" in c$history || "d" in c$history)
+	     ) )
+		return T;
+	return F;
+	}
+
+function addr_scan_predicate(index: Metrics::Index, data: Metrics::DataPoint): bool
+	{
+	local service = to_port(index$str);
+	local host = index$host;
+
+	local transport_layer_proto = get_port_transport_proto(service);
+	if ( suppress_UDP_scan_checks && (transport_layer_proto == udp) )
+		return F;
+	else if ( suppress_TCP_scan_checks && (transport_layer_proto == tcp) )
+		return F;
+	else if ( suppress_ICMP_scan_checks && (transport_layer_proto == icmp) )
+		return F;
+
+	# TODO: all of this whitelist/blacklist will be done 
+	#       through the upcoming hook mechanism
+	# Blacklisting/whitelisting services
+	#if ( |analyze_services| > 0 )
+	#	{
+	#	if ( service !in analyze_services )
+	#		return F;
+	#	}
+	#else if ( service in skip_services )
+	#	return F;
+	#
+	## Blacklisting/whitelisting subnets
+	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
+	#	return F;
+
+	return T;
+	}
+
+function port_scan_predicate(index: Metrics::Index, data: Metrics::DataPoint): bool
+	{
+	local service = to_port(data$str);
+	local host = index$host;
+
+	local transport_layer_proto = get_port_transport_proto(service);
+	if ( suppress_UDP_scan_checks && (transport_layer_proto == udp) )
+		return F;
+	else if ( suppress_TCP_scan_checks && (transport_layer_proto == tcp) )
+		return F;
+	else if ( suppress_ICMP_scan_checks && (transport_layer_proto == icmp) )
+		return F;
+
+	# TODO: all of this whitelist/blacklist will be done 
+	#       through the upcoming hook mechanism
+	# Blacklisting/whitelisting services
+	#if ( |analyze_services| > 0 )
+	#	{
+	#	if ( service !in analyze_services )
+	#		return F;
+	#	}
+	#else if ( service in skip_services )
+	#	return F;
+	#
+	## Blacklisting/whitelisting subnets
+	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
+	#	return F;
+
+	return T;
+	}
+
+function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
+	{
+	local service = to_port(index$str);
+
+	return ( service in addr_scan_custom_thresholds &&
+	         val$sum > addr_scan_custom_thresholds[service] );
+	}
+
+function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
+	{
+	local direction = Site::is_local_addr(index$host) ? "OutboundScan" : "InboundScan";
+	local message=fmt("%s scanned %d unique hosts on port %s", index$host, val$unique, index$str);
+
+	NOTICE([$note=AddressScan,
+	        $src=index$host,
+	        $p=to_port(index$str),
+	        $sub=direction,
+	        $msg=message,
+	        $identifier=message]);
+	}
+
+function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
+	{
+	local direction = Site::is_local_addr(index$host) ? "OutboundScan" : "InboundScan";
+	local message = fmt("%s scanned %d unique ports of host %s", index$host, val$unique, index$str);
+
+	NOTICE([$note=PortScan, 
+	        $src=index$host,
+	        $dst=to_addr(index$str),
+	        $sub=direction,
+	        $msg=message,
+	        $identifier=message]);
+	}
+
+event bro_init() &priority=5
+	{
+	# Add local networks here to determine scan direction
+	# i.e. inbound scan / outbound scan
+	#add Site::local_nets[0.0.0.0/16];
+
+	if ( analyze_addr_scan )
+		{
+		# note=> Addr scan: table [src_ip, port] of set(dst);	
+		# Add filters to the metrics so that the metrics framework knows how to
+		# determine when it looks like an actual attack and how to respond when
+		# thresholds are crossed.
+		Metrics::add_filter("scan.addr.fail", [$log=F,
+		                                       $every=conn_failed_addr_interval,
+		                                       $measure=set(Metrics::UNIQUE),
+		                                       $pred=addr_scan_predicate,
+		                                       $threshold_func=check_addr_scan_threshold,
+		                                       $threshold=default_addr_scan_threshold,
+		                                       $threshold_crossed=addr_scan_threshold_crossed]); 
+		}
+
+	if ( analyze_port_scan )
+		{
+		# note=> Port Sweep: table[src_ip, dst_ip] of set(port);
+		# Add filters to the metrics so that the metrics framework knows how to
+		# determine when it looks like an actual attack and how to respond when
+		# thresholds are crossed.
+		Metrics::add_filter("scan.port.fail", [$log=F,
+		                                       $every=conn_failed_port_interval,
+		                                       $measure=set(Metrics::UNIQUE),
+		                                       $pred=port_scan_predicate,
+		                                       $threshold=default_port_scan_threshold,
+		                                       $threshold_crossed=port_scan_threshold_crossed]); 
+		}
+	}
+
+## Generated when a SYN-ACK packet is seen in response to a SYN 
+## packet during a TCP handshake. The final ACK of the handshake 
+## in response to SYN-ACK may or may not occur later, one way to 
+## tell is to check the history field of connection to see if the 
+## originator sent an ACK, indicated by ‘A’ in the history string.
+#event connection_established(c: connection)
+#	{
+	# Not useful for scan (too early)
+#	}
+
+## Generated when one endpoint of a TCP connection attempted 
+## to gracefully close the connection, but the other endpoint 
+## is in the TCP_INACTIVE state. This can happen due to split 
+## routing, in which Bro only sees one side of a connection.
+#event connection_half_finished(c: connection)
+#	{
+	# Half connections never were "established", so do scan-checking here.
+	# I am not taking *f cases of c$history into account. Ask Seth if I should
+#	}
+
+function add_metrics(id: conn_id, reverse: bool)
+	{
+	local scanner:      addr;
+	local victim:       string;
+	local scanned_port: string;
+
+	if ( reverse )
+		{
+		scanner      = id$resp_h;
+		victim       = cat(id$orig_h);
+		scanned_port = fmt("%s", id$orig_p);
+		}
+	else
+		{
+		scanner      = id$orig_h;
+		victim       = cat(id$resp_h);
+		scanned_port = fmt("%s", id$resp_p);
+		}
+
+	if ( analyze_addr_scan )
+		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=scanned_port], [$str=victim]);
+	if ( analyze_port_scan )
+		Metrics::add_data("scan.port.fail", [$host=scanner, $str=victim],       [$str=scanned_port]);
+	}
+
+## Generated for an unsuccessful connection attempt. This 
+## event is raised when an originator unsuccessfully attempted 
+## to establish a connection. “Unsuccessful” is defined as at least 
+## tcp_attempt_delay seconds having elapsed since the originator 
+## first sent a connection establishment packet to the destination 
+## without seeing a reply.
+event connection_attempt(c: connection)
+	{
+	local is_reverse_scan = F;
+	if ( "H" in c$history )
+		is_reverse_scan = T;
+	
+	add_metrics(c$id, is_reverse_scan);
+	}
+
+## Generated for a rejected TCP connection. This event 
+## is raised when an originator attempted to setup a TCP 
+## connection but the responder replied with a RST packet 
+## denying it.
+event connection_rejected(c: connection)
+	{
+	local is_reverse_scan = F;
+	if ( "s" in c$history )
+		is_reverse_scan = T;
+	
+	add_metrics(c$id, is_reverse_scan);
+	}
+
+## Generated when an endpoint aborted a TCP connection. 
+## The event is raised when one endpoint of an *established* 
+## TCP connection aborted by sending a RST packet.
+event connection_reset(c: connection)
+	{
+	local is_reverse_scan = F;
+	local is_scan = F;
+
+	if ( is_failed_conn(c) )
+		{
+		is_scan = T;
+		is_reverse_scan = F;
+		}
+	else if ( is_reverse_failed_conn(c) )
+		{
+		is_scan = T;
+		is_reverse_scan = T;
+		}
+
+	if ( is_scan )
+		{
+		add_metrics(c$id, is_reverse_scan);
+		}
+	}
+
+## Generated for each still-open connection when Bro terminates.
+event connection_pending(c: connection)
+	{
+	local is_reverse_scan = F;
+	local is_scan = F;
+
+	if ( is_failed_conn(c) )
+		{
+		is_scan = T;
+		is_reverse_scan = F;
+		}
+	else if ( is_reverse_failed_conn(c) )
+		{
+		is_scan = T;
+		is_reverse_scan = T;
+		}
+
+	if ( is_scan )
+		{
+		add_metrics(c$id, is_reverse_scan);
+		}
+	}
\ No newline at end of file
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 193a4b9614..9dab73c43e 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -51,14 +51,29 @@ event bro_init() &priority=3
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
 	
-	Metrics::add_filter("http.sqli.attacker", [$log=F,
-	                                           $notice_threshold=sqli_requests_threshold,
-	                                           $break_interval=sqli_requests_interval,
-	                                           $note=SQL_Injection_Attacker]);
-	Metrics::add_filter("http.sqli.victim", [$log=F,
-	                                         $notice_threshold=sqli_requests_threshold,
-	                                         $break_interval=sqli_requests_interval,
-	                                         $note=SQL_Injection_Victim]);
+	Metrics::add_filter("http.sqli.attacker", 
+	                    [$every=sqli_requests_interval,
+	                     $measure=set(Metrics::SUM),
+	                     $threshold=sqli_requests_threshold,
+	                     $samples=10,
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	NOTICE([$note=SQL_Injection_Attacker, 
+	                     	        $msg="An SQL injection attacker was discovered!",
+	                     	        $src=index$host,
+	                     	        $identifier=cat(index$host)]);
+	                     }, $log=F]);
+
+	Metrics::add_filter("http.sqli.victim",
+	                    [$every=sqli_requests_interval,
+	                     $measure=set(Metrics::SUM),
+	                     $threshold=sqli_requests_threshold,
+	                     $samples=10,
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	NOTICE([$note=SQL_Injection_Victim, 
+	                     	        $msg="An SQL injection victim was discovered!",
+	                     	        $src=index$host,
+	                     	        $identifier=cat(index$host)]);
+	                     }, $log=F]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -68,7 +83,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data("http.sqli.attacker", [$host=c$id$orig_h], 1);
-		Metrics::add_data("http.sqli.victim", [$host=c$id$resp_h], 1);
+		Metrics::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
+		Metrics::add_data("http.sqli.victim",   [$host=c$id$resp_h], [$str=original_URI]);
 		}
 	}
diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index 7939f00c72..d0f1b63d70 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -42,10 +42,15 @@ export {
 event bro_init()
 	{
 	Metrics::add_filter("ssh.login.failure", [$name="detect-bruteforcing", $log=F,
-	                                          $note=Password_Guessing,
-	                                          $notice_threshold=password_guesses_limit,
-	                                          $notice_freq=1hr,
-	                                          $break_interval=guessing_timeout]);
+	                                          $every=guessing_timeout,
+	                                          $measure=set(Metrics::SUM),
+	                                          $threshold=password_guesses_limit,
+	                                          $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                                          	NOTICE([$note=Password_Guessing, 
+	                                          	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %.0f connections).", index$host, val$sum),
+	                                          	        $src=index$host,
+	                                          	        $identifier=cat(index$host)]);
+	                                          }]);
 	}
 
 event SSH::heuristic_successful_login(c: connection)
@@ -70,5 +75,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		Metrics::add_data("ssh.login.failure", [$host=id$orig_h], 1);
+		Metrics::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
 	}
diff --git a/scripts/site/local.bro b/scripts/site/local.bro
index db1a786839..acbef96721 100644
--- a/scripts/site/local.bro
+++ b/scripts/site/local.bro
@@ -8,6 +8,9 @@
 # Apply the default tuning scripts for common tuning settings.
 @load tuning/defaults
 
+# Load the scan detection script.
+@load protocols/conn/scan
+
 # Generate notices when vulnerable versions of software are discovered.
 # The default is to only monitor software found in the address space defined
 # as "local".  Refer to the software framework's documentation for more 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout
new file mode 100644
index 0000000000..2d0750ca18
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout
@@ -0,0 +1 @@
+A test metric threshold was crossed!
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
new file mode 100644
index 0000000000..fc881ba68e
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
@@ -0,0 +1,10 @@
+THRESHOLD_SERIES: hit a threshold series value at 3 for metric_index(host=1.2.3.4)
+THRESHOLD_FUNC: hit a threshold function value at 3 for metric_index(host=1.2.3.4)
+THRESHOLD_FUNC: hit a threshold function value at 2 for metric_index(host=6.5.4.3)
+THRESHOLD_FUNC: hit a threshold function value at 1 for metric_index(host=7.2.1.5)
+THRESHOLD: hit a threshold value at 6 for metric_index(host=1.2.3.4)
+THRESHOLD_SERIES: hit a threshold series value at 6 for metric_index(host=1.2.3.4)
+THRESHOLD_FUNC: hit a threshold function value at 3 for metric_index(host=1.2.3.4)
+THRESHOLD: hit a threshold value at 1000 for metric_index(host=7.2.1.5)
+THRESHOLD_SERIES: hit a threshold series value at 1001 for metric_index(host=7.2.1.5)
+THRESHOLD_FUNC: hit a threshold function value at 1000 for metric_index(host=7.2.1.5)
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index acd57f037e..41ef9b57dc 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -5,15 +5,15 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
 	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
-	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
-	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
+	["proxy-1"]   = [$node_type=Cluster::PROXY,   $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
 @TEST-END-FILE
 
@@ -22,8 +22,8 @@ redef Log::default_rotation_interval = 0secs;
 event bro_init() &priority=5
 	{
 	Metrics::add_filter("test.metric", 
-		[$name="foo-bar",
-		 $break_interval=3secs]);
+	                    [$every=3secs,
+	                     $measure=set(Metrics::SUM, Metrics::MIN, Metrics::MAX, Metrics::AVG, Metrics::STD_DEV, Metrics::VARIANCE)]);
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -39,9 +39,25 @@ redef Cluster::manager2worker_events += /ready_for_data/;
 
 event ready_for_data()
 	{
-	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	if ( Cluster::node == "worker-1" )
+		{
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=34]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
+		Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=1]);
+		Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=54]);
+		}
+	if ( Cluster::node == "worker-2" )
+		{
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=75]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=57]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=52]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=61]);
+		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=95]);
+		Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=5]);
+		Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=91]);
+		}
 	}
 
 @endif
@@ -53,7 +69,7 @@ global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
-	n = n + 1;
+	++n;
 	if ( n == 3 )
 		{
 		terminate_communication();
@@ -64,11 +80,9 @@ event Metrics::log_metrics(rec: Metrics::Info)
 event remote_connection_handshake_done(p: event_peer)
 	{
 	print p;
-	peer_count = peer_count + 1;
+	++peer_count;
 	if ( peer_count == 3 )
-		{
 		event ready_for_data();
-		}
 	}
 
 @endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic.bro b/testing/btest/scripts/base/frameworks/metrics/basic.bro
index 23a79d2bd3..12163ed689 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic.bro
@@ -5,8 +5,14 @@ event bro_init() &priority=5
 	{
 	Metrics::add_filter("test.metric", 
 		[$name="foo-bar",
-		 $break_interval=3secs]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
-	Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
-	Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
+		 $every=3secs,
+		 $measure=set(Metrics::SUM, Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=5]);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=22]);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=94]);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
+
+	Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
+	Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
 	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
index eacebaa50e..3341fa1887 100644
--- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
@@ -1,36 +1,33 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 3
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 20
-# @TEST-EXEC: btest-diff manager-1/notice.log
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-diff manager-1/.stdout
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
 	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
-	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
-	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
 };
 @TEST-END-FILE
 
 redef Log::default_rotation_interval = 0secs;
 
-redef enum Notice::Type += {
-	Test_Notice,
-};
-
 event bro_init() &priority=5
 	{
 	Metrics::add_filter("test.metric",
-		[$name="foo-bar",
-		 $break_interval=1hr,
-		 $note=Test_Notice,
-		 $notice_threshold=100,
-		 $log=T]);
+	                    [$every=1hr,
+	                     $measure=set(Metrics::SUM),
+	                     $threshold=100,
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	print "A test metric threshold was crossed!";
+	                     	terminate();
+	                     }
+	                     ]);
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -38,24 +35,12 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
-@if ( Cluster::local_node_type() == Cluster::MANAGER )
-
-event Notice::log_notice(rec: Notice::Info)
-	{
-	terminate_communication();
-	terminate();
-	}
-
-@endif
-
-@if ( Cluster::local_node_type() == Cluster::WORKER )
-
 event do_metrics(i: count)
 	{
 	# Worker-1 will trigger an intermediate update and then if everything
 	# works correctly, the data from worker-2 will hit the threshold and
 	# should trigger the notice.
-	Metrics::add_data("test.metric", [$host=1.2.3.4], i);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=i]);
 	}
 
 event bro_init()
@@ -65,5 +50,3 @@ event bro_init()
 	if ( Cluster::node == "worker-2" )
 		event do_metrics(1);
 	}
-
-@endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/notice.bro b/testing/btest/scripts/base/frameworks/metrics/notice.bro
deleted file mode 100644
index 1ed11a968c..0000000000
--- a/testing/btest/scripts/base/frameworks/metrics/notice.bro
+++ /dev/null
@@ -1,20 +0,0 @@
-# @TEST-EXEC: bro %INPUT
-# @TEST-EXEC: btest-diff notice.log
-
-
-redef enum Notice::Type += {
-	Test_Notice,
-};
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("test.metric", 
-		[$name="foo-bar",
-		 $break_interval=3secs,
-		 $note=Test_Notice,
-		 $notice_threshold=2,
-		 $log=F]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
-	Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
-	Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
-	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/thresholding.bro b/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
new file mode 100644
index 0000000000..bd0cd6faae
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
@@ -0,0 +1,47 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter("test.metric", 
+	                    [$name="foobar",
+	                     $every=3secs,
+	                     $measure=set(Metrics::SUM),
+	                     $threshold=5,
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	print fmt("THRESHOLD: hit a threshold value at %.0f for %s", val$sum, Metrics::index2str(index));
+	                     },
+	                     $log=F]);
+
+	Metrics::add_filter("test.metric", 
+	                    [$name="foobar2",
+	                     $every=3secs,
+	                     $measure=set(Metrics::SUM),
+	                     $threshold_series=vector(3,6,800),
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	print fmt("THRESHOLD_SERIES: hit a threshold series value at %.0f for %s", val$sum, Metrics::index2str(index));
+		                 },
+	                     $log=F]);
+	Metrics::add_filter("test.metric", 
+	                    [$every=3secs,
+	                     $measure=set(Metrics::SUM),
+	                     $threshold_func(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	# This causes any data added to be cross the threshold.
+	                     	return T;
+	                     },
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	print fmt("THRESHOLD_FUNC: hit a threshold function value at %.0f for %s", val$sum, Metrics::index2str(index));
+	                     },
+	                     $log=F]);
+
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
+	Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
+	Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
+	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
+	Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=1000]);
+	}

From 257b460b18a9364fa74a0c0594c17724875f35e4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 16 Nov 2012 03:05:43 -0500
Subject: [PATCH 11/70] Updated the app-metrics script to the new metrics api.

- Inconsequential change to scan.bro.
---
 scripts/policy/misc/app-metrics.bro    | 35 +++++++++++---------------
 scripts/policy/protocols/conn/scan.bro |  2 --
 2 files changed, 14 insertions(+), 23 deletions(-)

diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index 5cb108ea73..a89d0d8eb3 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -12,48 +12,41 @@ export {
 
 event bro_init() &priority=3
 	{
-	Metrics::add_filter("apps.bytes", [$break_interval=break_interval]);
-	Metrics::add_filter("apps.views", [$break_interval=break_interval]);
-	Metrics::add_filter("apps.users", [$break_interval=break_interval]);
+	Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM)]);
+	Metrics::add_filter("apps.hits",  [$every=break_interval, $measure=set(Metrics::SUM, Metrics::UNIQUE)]);
 	}
 
 function do_metric(id: conn_id, hostname: string, size: count)
 	{
 	if ( /youtube/ in hostname && size > 512*1024 ) 
 		{
-		Metrics::add_data("apps.bytes", [$str="youtube"], size);
-		Metrics::add_data("apps.views", [$str="youtube"], 1);
-		Metrics::add_unique("apps.users", [$str="youtube"], cat(id$orig_h));
+		Metrics::add_data("apps.bytes", [$str="youtube"], [$num=size]);
+		Metrics::add_data("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /facebook.com|fbcdn.net/ in hostname && size > 20 )
 		{
-		Metrics::add_data("apps.bytes", [$str="facebook"], size);
-		Metrics::add_data("apps.views", [$str="facebook"], 1);
-		Metrics::add_unique("apps.users", [$str="facebook"], cat(id$orig_h));
+		Metrics::add_data("apps.bytes", [$str="facebook"], [$num=size]);
+		Metrics::add_data("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /google.com/ in hostname && size > 20 ) 
 		{
-		Metrics::add_data("apps.bytes", [$str="google"], size);
-		Metrics::add_data("apps.views", [$str="google"], 1);
-		Metrics::add_unique("apps.users", [$str="google"], cat(id$orig_h));
+		Metrics::add_data("apps.bytes", [$str="google"], [$num=size]);
+		Metrics::add_data("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /nflximg.com/ in hostname && size > 200*1024 ) 
 		{
-		Metrics::add_data("apps.bytes", [$str="netflix"], size);
-		Metrics::add_data("apps.views", [$str="netflix"], 1);
-		Metrics::add_unique("apps.users", [$str="netflix"], cat(id$orig_h));
+		Metrics::add_data("apps.bytes", [$str="netflix"], [$num=size]);
+		Metrics::add_data("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /pandora.com/ in hostname && size > 512*1024 )
 		{
-		Metrics::add_data("apps.bytes", [$str="pandora"], size);
-		Metrics::add_data("apps.views", [$str="pandora"], 1);
-		Metrics::add_unique("apps.users", [$str="pandora"], cat(id$orig_h));
+		Metrics::add_data("apps.bytes", [$str="pandora"], [$num=size]);
+		Metrics::add_data("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /gmail.com/ in hostname && size > 20 )
 		{
-		Metrics::add_data("apps.bytes", [$str="gmail"], size);
-		Metrics::add_data("apps.views", [$str="gmail"], 1);
-		Metrics::add_unique("apps.users", [$str="gmail"], cat(id$orig_h));
+		Metrics::add_data("apps.bytes", [$str="gmail"], [$num=size]);
+		Metrics::add_data("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
 		}
 }
 
diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro
index 4df2697092..8795cfda06 100644
--- a/scripts/policy/protocols/conn/scan.bro
+++ b/scripts/policy/protocols/conn/scan.bro
@@ -282,7 +282,6 @@ event connection_reset(c: connection)
 	if ( is_failed_conn(c) )
 		{
 		is_scan = T;
-		is_reverse_scan = F;
 		}
 	else if ( is_reverse_failed_conn(c) )
 		{
@@ -305,7 +304,6 @@ event connection_pending(c: connection)
 	if ( is_failed_conn(c) )
 		{
 		is_scan = T;
-		is_reverse_scan = F;
 		}
 	else if ( is_reverse_failed_conn(c) )
 		{

From 5b81cfe7e2a8768667ddd19e524819a250d34505 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 19 Nov 2012 23:42:19 -0500
Subject: [PATCH 12/70] Implemented a nearly generic Queue in scriptland.

---
 scripts/base/init-default.bro                 |   1 +
 scripts/base/utils/queue.bro                  | 177 ++++++++++++++++++
 .../Baseline/scripts.base.utils.queue/output  |  11 ++
 testing/btest/scripts/base/utils/queue.test   |  35 ++++
 4 files changed, 224 insertions(+)
 create mode 100644 scripts/base/utils/queue.bro
 create mode 100644 testing/btest/Baseline/scripts.base.utils.queue/output
 create mode 100644 testing/btest/scripts/base/utils/queue.test

diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro
index 8b36899f10..563f8af0bc 100644
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@@ -12,6 +12,7 @@
 @load base/utils/numbers
 @load base/utils/paths
 @load base/utils/patterns
+@load base/utils/queue
 @load base/utils/strings
 @load base/utils/thresholds
 @load base/utils/urls
diff --git a/scripts/base/utils/queue.bro b/scripts/base/utils/queue.bro
new file mode 100644
index 0000000000..c5e3bcf906
--- /dev/null
+++ b/scripts/base/utils/queue.bro
@@ -0,0 +1,177 @@
+##! A FIFO string queue.
+
+module Queue;
+
+export {
+	## Settings for initializing the queue.
+	type Settings: record {
+		## If a maximum length is set for the queue
+		## it will maintain itself at that 
+		## maximum length automatically.
+		max_len: count &optional;
+	};
+
+	## The internal data structure for the queue.
+	type Queue: record {};
+
+	## Initialize a queue record structure.
+	## 
+	## s: A :bro:record:`Settings` record configuring the queue.
+	##
+	## Returns: An opaque queue record.
+	global init:       function(s: Settings): Queue;
+
+	## Push a string onto the top of a queue.
+	## 
+	## q: The queue to push the string into.
+	## 
+	## val: The string to push 
+	global push:       function(q: Queue, val: any);
+
+	## Pop a string from the bottom of a queue.
+	##
+	## q: The queue to pop the string from.
+	##
+	## Returns: The string popped from the queue.
+	global pop:        function(q: Queue): any;
+
+	## Merge two queue's together.  If any settings are applied 
+	## to the queues, the settings from q1 are used for the new
+	## merged queue.
+	## 
+	## q1: The first queue.  Settings are taken from here.
+	##
+	## q2: The second queue.
+	## 
+	## Returns: A new queue from merging the other two together.
+	global merge:      function(q1: Queue, q2: Queue): Queue;
+
+	## Get the number of items in a queue.
+	## 
+	## q: The queue.
+	##
+	## Returns: The length of the queue.
+	global len:     function(q: Queue): count;
+	
+	## Get the contents of the queue as a string vector.
+	## 
+	## q: The queue.
+	##
+	## Returns: A :bro:type:`vector of string` containing the 
+	##          current contents of q.
+	global get_str_vector: function(q: Queue): vector of string;
+
+	## Get the contents of the queue as a count vector.  Use care
+	## with this function.  If the data put into the queue wasn't 
+	## integers you will get conversion errors.
+	## 
+	## q: The queue.
+	##
+	## Returns: A :bro:type:`vector of count` containing the 
+	##          current contents of q.
+	global get_cnt_vector: function(q: Queue): vector of count;
+}
+
+redef record Queue += {
+	# Indicator for if the queue was appropriately initialized.
+	initialized: bool                   &default=F;
+	# The values are stored here.
+	vals:        table[count] of any &optional;
+	# Settings for the queue.
+	settings:    Settings               &optional;
+	# The top value in the vals table.
+	top:         count                  &default=0;
+	# The bottom value in the vals table.
+	bottom:      count                  &default=0;
+	# The number of bytes in the queue.
+	size:        count                  &default=0;
+};
+
+function init(s: Settings): Queue
+	{
+	local q: Queue;
+	q$vals=table();
+	q$settings = copy(s);
+	q$initialized=T;
+	return q;
+	}
+
+function push(q: Queue, val: any)
+	{
+	if ( q$settings?$max_len && len(q) >= q$settings$max_len )
+		pop(q);
+	q$vals[q$top] = val;
+	++q$top;
+	}
+
+function pop(q: Queue): any
+	{
+	local ret = q$vals[q$bottom];
+	delete q$vals[q$bottom];
+	++q$bottom;
+	return ret;
+	}
+
+function merge(q1: Queue, q2: Queue): Queue
+	{
+	local ret = init(q1$settings);
+	local i = q1$bottom;
+	local j = q2$bottom;
+	for ( ignored_val in q1$vals )
+		{
+		if ( i in q1$vals )
+			push(ret, q1$vals[i]);
+		if ( j in q2$vals )
+			push(ret, q2$vals[j]);
+		++i;
+		++j;
+		}
+	}
+
+function len(q: Queue): count
+	{
+	return |q$vals|;
+	}
+
+function get_str_vector(q: Queue): vector of string
+	{
+	local ret: vector of string;
+	local i = q$bottom;
+	local j = 0;
+	# Really dumb hack, this is only to provide
+	# the iteration for the correct number of 
+	# values in q$vals.
+	for ( ignored_val in q$vals )
+		{
+		if ( i >= q$top )
+			break;
+
+		ret[j] = cat(q$vals[i]);
+		++j; ++i;
+		}
+	return ret;
+	}
+
+function get_cnt_vector(q: Queue): vector of count
+	{
+	local ret: vector of count;
+	local i = q$bottom;
+	local j = 0;
+	# Really dumb hack, this is only to provide
+	# the iteration for the correct number of 
+	# values in q$vals.
+	for ( ignored_val in q$vals )
+		{
+		if ( i >= q$top )
+			break;
+
+		# TODO: this is terrible and should be replaced by 
+		#       a more generic version of the various 
+		#       functions to get vectors of values.
+		#       (the way "any" works right now makes this impossible though)
+		ret[j] = to_count(cat(q$vals[i]));
+		++j; ++i;
+		}
+	return ret;
+	}
+
diff --git a/testing/btest/Baseline/scripts.base.utils.queue/output b/testing/btest/Baseline/scripts.base.utils.queue/output
new file mode 100644
index 0000000000..b878006310
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.utils.queue/output
@@ -0,0 +1,11 @@
+This is a get_cnt_vector test: 3
+This is a get_cnt_vector test: 4
+This is a get_str_vector test: 3
+This is a get_str_vector test: 4
+Testing pop: 3
+Length after pop: 1
+Size of q2: 4
+String queue value: test 1
+String queue value: test 2
+String queue value: test 2
+String queue value: test 1
diff --git a/testing/btest/scripts/base/utils/queue.test b/testing/btest/scripts/base/utils/queue.test
new file mode 100644
index 0000000000..50f541a25f
--- /dev/null
+++ b/testing/btest/scripts/base/utils/queue.test
@@ -0,0 +1,35 @@
+# @TEST-EXEC: bro -b %INPUT > output
+# @TEST-EXEC: btest-diff output
+
+# This is loaded by default
+@load base/utils/queue
+
+event bro_init()
+	{
+	local q = Queue::init([$max_len=2]);
+	Queue::push(q, 1);
+	Queue::push(q, 2);
+	Queue::push(q, 3);
+	Queue::push(q, 4);
+	local test1 =  Queue::get_cnt_vector(q);
+	for ( i in test1 )
+		print fmt("This is a get_cnt_vector test: %d", test1[i]);
+
+	local test2 =  Queue::get_str_vector(q);
+	for ( i in test2 )
+		print fmt("This is a get_str_vector test: %s", test2[i]);
+
+	local test_val = Queue::pop(q);
+	print fmt("Testing pop: %s", test_val);
+	print fmt("Length after pop: %d", Queue::len(q));
+
+	local q2 = Queue::init([]);
+	Queue::push(q2, "test 1");
+	Queue::push(q2, "test 2");
+	Queue::push(q2, "test 2");
+	Queue::push(q2, "test 1");
+	print fmt("Size of q2: %d", Queue::len(q2));
+	local test3: vector of string = Queue::get_str_vector(q2);
+	for ( i in test3 )
+		print fmt("String queue value: %s", test3[i]);
+	}
\ No newline at end of file

From 95b12262e44da0f5ccd3bb66872b0c818d8c01de Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 19 Nov 2012 23:43:15 -0500
Subject: [PATCH 13/70] More cleanup and fixed to the metrics framework.

---
 scripts/base/frameworks/metrics/main.bro      | 113 ++++++++++++------
 .../.stdout                                   |   4 +-
 2 files changed, 79 insertions(+), 38 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 8e40e76e02..ffb3a18354 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -1,5 +1,7 @@
 ##! The metrics framework provides a way to count and measure data.  
 
+@load base/utils/queue
+
 module Metrics;
 
 export {
@@ -12,15 +14,23 @@ export {
 	
 	## This is the interval for how often threshold based notices will happen 
 	## after they have already fired.
-	const threshold_series_restart_interval = 1hr &redef;
+	const threshold_crossed_restart_interval = 1hr &redef;
 	
 	type Calculation: enum {
+		## Sums the values given.  For string values,
+		## this will be the number of strings given.
 		SUM,
+		## Find the minimum value.
 		MIN,
+		## Find the maximum value.
 		MAX,
+		## Find the variance of the values.
 		VARIANCE,
+		## Find the standard deviation of the values.
 		STD_DEV,
+		## Calculate the average of the values.
 		AVG,
+		## Calculate the number of unique values.
 		UNIQUE,
 	};
 
@@ -45,9 +55,13 @@ export {
 	} &log;
 	
 	## Represents data being added for a single metric data point.
+	## Only supply a single value here at a time.
 	type DataPoint: record {
+		## Count value.
 		num:       count       &optional;
+		## Double value.
 		dbl:       double      &optional;
+		## String value.
 		str:       string      &optional;
 	};
 
@@ -55,34 +69,34 @@ export {
 	## of the measurements collected for the metric.
 	type ResultVal: record {
 		## The number of measurements received.
-		num:     count        &log &default=0;
+		num:      count         &log &default=0;
 
 		## For numeric data, this tracks the sum of all values.
-		sum:     double       &log &optional;
+		sum:      double        &log &optional;
 
 		## For numeric data, this tracks the minimum value given.
-		min:     double       &log &optional;
+		min:      double        &log &optional;
 
 		## For numeric data, this tracks the maximum value given.
-		max:     double       &log &optional;
+		max:      double        &log &optional;
 
 		## For numeric data, this calculates the average of all values.
-		avg:     double       &log &optional;
+		avg:      double        &log &optional;
 
 		## For numeric data, this calculates the variance.
-		variance: double      &log &optional;
+		variance: double        &log &optional;
 
 		## For numeric data, this calculates the standard deviation.
-		std_dev:  double      &log &optional;
+		std_dev:  double        &log &optional;
 
 		## If cardinality is being tracked, the number of unique
 		## items is tracked here.
-		unique: count         &log &optional;
+		unique:   count         &log &optional;
 
 		## A sample of something being measured.  This is helpful in 
 		## some cases for collecting information to do further detection
 		## or better logging for forensic purposes.
-		sample: set[DataPoint]   &optional;
+		samples:  vector of string   &optional;
 	};
 
 	## The record type that is used for logging metrics.
@@ -145,7 +159,7 @@ export {
 		threshold_func:    function(index: Metrics::Index, val: Metrics::ResultVal): bool &optional;
 		## A function callback that is called when a threshold is crossed.
 		threshold_crossed: function(index: Metrics::Index, val: Metrics::ResultVal) &optional;
-		## A number of sample DataPoints to collect for the threshold 
+		## A number of sample DataPoint strings to collect for the threshold 
 		## crossing callback.
 		samples:           count                    &optional;
 	};
@@ -193,6 +207,17 @@ redef record ResultVal += {
 	# since we will like move to a probalistic data structure in the future.
 	# TODO: in the future this will optionally be a hyperloglog structure
 	unique_vals:  set[DataPoint] &optional;
+
+	# Internal use only.  This is the queue where samples
+	# are maintained since the queue is self managing for
+	# the number of samples requested.
+	sample_queue: Queue::Queue &optional;
+
+	# Internal use only.  Indicates if a simple threshold was already crossed.
+	is_threshold_crossed: bool &default=F;
+
+	# Internal use only.  Current index for threshold series.
+	threshold_series_index: count &default=0;
 };
 
 # Type to store a table of metrics values.
@@ -207,9 +232,6 @@ global filter_store: table[string, string] of Filter = table();
 # This is indexed by metric id and filter name.
 global store: table[string, string] of MetricTable = table() &default=table();
 
-# This stores the current threshold index for filters using $threshold_series.
-global threshold_series_index: table[string, string, Index] of count = {} &create_expire=threshold_series_restart_interval &default=0;
-
 # This is hook for watching thresholds being crossed.  It is called whenever
 # index values are updated and the new val is given as the `val` argument.
 # It's only prototyped here because cluster and non-cluster has separate 
@@ -311,6 +333,7 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	else if ( rv2?$var_s )
 		result$var_s = rv2$var_s;
 
+	# Merge $unique_vals
 	if ( rv1?$unique_vals || rv2?$unique_vals )
 		{
 		result$unique_vals = set();
@@ -321,7 +344,22 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 			for ( val2 in rv2$unique_vals )
 				add result$unique_vals[val2];
 		}
+
+	# Merge $sample_queue
+	if ( rv1?$sample_queue && rv2?$sample_queue )
+		result$sample_queue = Queue::merge(rv1$sample_queue, rv2$sample_queue);
+	else if ( rv1?$sample_queue )
+		result$sample_queue = rv1$sample_queue;
+	else if ( rv2?$sample_queue )
+		result$sample_queue = rv2$sample_queue;
 	
+	# Merge $threshold_series_index
+	result$threshold_series_index = (rv1$threshold_series_index > rv2$threshold_series_index) ? rv1$threshold_series_index : rv2$threshold_series_index;
+
+	# Merge $is_threshold_crossed
+	if ( rv1$is_threshold_crossed || rv2$is_threshold_crossed )
+		result$is_threshold_crossed = T;
+
 	do_calculated_fields(result);
 	return result;
 	}
@@ -412,6 +450,13 @@ function add_data(id: string, index: Index, data: DataPoint)
 
 		++result$num;
 
+		if ( filter?$samples && data?$str )
+			{
+			if ( ! result?$sample_queue )
+				result$sample_queue = Queue::init([$max_len=filter$samples]);
+			Queue::push(result$sample_queue, data$str);
+			}
+
 		if ( SUM in filter$measure )
 			{
 			if ( ! result?$sum ) result$sum = 0;
@@ -422,7 +467,7 @@ function add_data(id: string, index: Index, data: DataPoint)
 			{
 			if ( ! result?$min ) 
 				result$min = val;
-			else if (val < result$min) 
+			else if ( val < result$min )
 				result$min = val;
 			}
 
@@ -430,7 +475,7 @@ function add_data(id: string, index: Index, data: DataPoint)
 			{
 			if ( ! result?$max ) 
 				result$max = val;
-			else if (val > result$max) 
+			else if ( val > result$max )
 				result$max = val;
 			}
 	
@@ -485,22 +530,24 @@ function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_p
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
 		watch = watch/modify_pct;
 
-	if ( filter?$threshold && watch >= filter$threshold )
+	if ( ! val$is_threshold_crossed &&
+	     filter?$threshold && watch >= filter$threshold )
 		{
 		# A default threshold was given and the value crossed it.
 		return T;
 		}
 
 	if ( filter?$threshold_series &&
-	     |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] &&
-	     watch >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] )
+	     |filter$threshold_series| >= val$threshold_series_index &&
+	     watch >= filter$threshold_series[val$threshold_series_index] )
 		{
 		# A threshold series was given and the value crossed the next 
 		# value in the series.
 		return T;
 		}
 
-	if ( filter?$threshold_func &&
+	if ( ! val$is_threshold_crossed &&
+	     filter?$threshold_func &&
 	     filter$threshold_func(index, val) )
 		{
 		# The threshold function indicated it was crossed.
@@ -512,20 +559,16 @@ function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_p
 		
 function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
 	{
-	if ( filter?$threshold_crossed )
-		filter$threshold_crossed(index, val);
+	if ( ! filter?$threshold_crossed )
+		return;
 
-	# If I don't reset here, the value just keeps
-	# retriggering once the threshold has been exceeded.
-	if ( !filter?$threshold_series )
-		{
-		reset(filter);
-		}
-	else
-		{
-		# This just needs set to some value so that it doesn't refire the 
-		# notice until it expires from the table or it crosses the next 
-		# threshold in the case of vectors of thresholds.
-		++threshold_series_index[filter$id, filter$name, index];
-		}
+	if ( val?$sample_queue )
+		val$samples = Queue::get_str_vector(val$sample_queue);
+
+	filter$threshold_crossed(index, val);
+	val$is_threshold_crossed = T;
+
+	# Bump up to the next threshold series index if a threshold series is being used.
+	if ( filter?$threshold_series )
+		++val$threshold_series_index;
 	}
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
index fc881ba68e..da692f2fe2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
@@ -4,7 +4,5 @@ THRESHOLD_FUNC: hit a threshold function value at 2 for metric_index(host=6.5.4.
 THRESHOLD_FUNC: hit a threshold function value at 1 for metric_index(host=7.2.1.5)
 THRESHOLD: hit a threshold value at 6 for metric_index(host=1.2.3.4)
 THRESHOLD_SERIES: hit a threshold series value at 6 for metric_index(host=1.2.3.4)
-THRESHOLD_FUNC: hit a threshold function value at 3 for metric_index(host=1.2.3.4)
-THRESHOLD: hit a threshold value at 1000 for metric_index(host=7.2.1.5)
+THRESHOLD: hit a threshold value at 1001 for metric_index(host=7.2.1.5)
 THRESHOLD_SERIES: hit a threshold series value at 1001 for metric_index(host=7.2.1.5)
-THRESHOLD_FUNC: hit a threshold function value at 1000 for metric_index(host=7.2.1.5)

From 47f5d256d80427aec26fb0ba8fe16a26bb44d200 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 20 Nov 2012 01:01:37 -0500
Subject: [PATCH 14/70] Added a script module for detecting hosts doing
 traceroutes.

---
 .../misc/detect-traceroute/__load__.bro       |  1 +
 .../detect-traceroute/detect-low-ttls.sig     |  9 ++
 .../policy/misc/detect-traceroute/main.bro    | 87 +++++++++++++++++++
 3 files changed, 97 insertions(+)
 create mode 100644 scripts/policy/misc/detect-traceroute/__load__.bro
 create mode 100644 scripts/policy/misc/detect-traceroute/detect-low-ttls.sig
 create mode 100644 scripts/policy/misc/detect-traceroute/main.bro

diff --git a/scripts/policy/misc/detect-traceroute/__load__.bro b/scripts/policy/misc/detect-traceroute/__load__.bro
new file mode 100644
index 0000000000..d551be57d3
--- /dev/null
+++ b/scripts/policy/misc/detect-traceroute/__load__.bro
@@ -0,0 +1 @@
+@load ./main
\ No newline at end of file
diff --git a/scripts/policy/misc/detect-traceroute/detect-low-ttls.sig b/scripts/policy/misc/detect-traceroute/detect-low-ttls.sig
new file mode 100644
index 0000000000..c04a8905f7
--- /dev/null
+++ b/scripts/policy/misc/detect-traceroute/detect-low-ttls.sig
@@ -0,0 +1,9 @@
+signature traceroute-detector-ipv4 {
+	header ip[8] < 10
+	event "match"
+}
+
+signature traceroute-detector-ipv6 {
+	header ip6[7] < 10
+	event "match"
+}
diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
new file mode 100644
index 0000000000..0709834cea
--- /dev/null
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -0,0 +1,87 @@
+##! This script detects large number of ICMP Time Exceeded messages heading
+##! toward hosts that have sent low TTL packets.
+##! It generates a notice when the number of ICMP Time Exceeded 
+##! messages for a source-destination pair exceeds threshold
+@load base/frameworks/metrics
+@load base/frameworks/signatures
+@load-sigs ./detect-low-ttls.sig
+
+redef Signatures::ignored_ids += /traceroute-detector.*/;
+
+module Traceroute;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	redef enum Notice::Type += {
+		## Indicates that a host was seen running traceroutes.  For more
+		## detail about specific traceroutes that we run, refer to the
+		## traceroute.log.
+		Detected
+	};
+
+	## By default this script requires that any host detected running traceroutes
+	## first send low TTL packets (TTL < 10) to the traceroute destination host.
+	## Changing this this setting to `F` will relax the detection a bit by 
+	## solely relying on ICMP time-exceeded messages to detect traceroute.
+	const require_low_ttl_packets = T &redef;
+	
+	## Defines the threshold for ICMP Time Exceeded messages for a src-dst pair.
+	## This threshold only comes into play after a host is found to be
+	## sending low ttl packets.
+	const icmp_time_exceeded_threshold = 2 &redef;
+
+	## Interval at which to watch for the
+	## :bro:id:`ICMPTimeExceeded::icmp_time_exceeded_threshold` variable to be crossed.
+	## At the end of each interval the counter is reset.
+	const icmp_time_exceeded_interval = 1min &redef;
+
+	## The log record for the traceroute log.
+	type Info: record {
+		## Timestamp
+		ts:   time &log;
+		## Address initiaing the traceroute.
+		src:  addr &log;
+		## Destination address of the traceroute.
+		dst:  addr &log;
+	};
+
+	global log_traceroute: event(rec: Traceroute::Info);
+}
+
+# Track hosts that have sent low TTL packets.
+global low_ttlers: set[addr, addr] = {} &create_expire=2min &synchronized;
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
+
+	Metrics::add_filter("traceroute.time_exceeded", 
+	                    [$log=F,
+	                     $every=icmp_time_exceeded_interval,
+	                     $measure=set(Metrics::UNIQUE),
+	                     $threshold=icmp_time_exceeded_threshold,
+	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     	local parts = split1(index$str, /-/);
+	                     	local src = to_addr(parts[1]);
+	                     	local dst = to_addr(parts[2]);
+	                     	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
+	                     	NOTICE([$note=Traceroute::Detected,
+	                     	        $msg=fmt("%s seems to be running traceroute", src),
+	                     	        $src=src, $dst=dst,
+	                     	        $identifier=parts[1]]);
+	                     }]);
+	}
+
+# Low TTL packets are detected with a signature.
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	if ( state$sig_id == /traceroute-detector.*/ )
+		add low_ttlers[state$conn$id$orig_h, state$conn$id$resp_h];
+	}
+
+event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
+	{
+	if ( ! require_low_ttl_packets || [context$id$orig_h, context$id$resp_h] in low_ttlers )
+		Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
+	}

From 20fdd36a44a3b0d10c3da342af1f771b3098897a Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 20 Nov 2012 01:02:23 -0500
Subject: [PATCH 15/70] Updated the SQL injection detection script to make it
 include samples in notice emails.

---
 scripts/base/frameworks/metrics/main.bro      |  7 +++--
 scripts/policy/protocols/http/detect-sqli.bro | 26 ++++++++++++++-----
 2 files changed, 25 insertions(+), 8 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index ffb3a18354..6b587a0939 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -68,6 +68,9 @@ export {
 	## Value supplied when a metric is finished.  It contains all
 	## of the measurements collected for the metric.
 	type ResultVal: record {
+		## The time when this result was first started.
+		begin:    time          &log;
+
 		## The number of measurements received.
 		num:      count         &log &default=0;
 
@@ -439,7 +442,7 @@ function add_data(id: string, index: Index, data: DataPoint)
 		
 		local metric_tbl = store[id, filter$name];
 		if ( index !in metric_tbl )
-			metric_tbl[index] = [];
+			metric_tbl[index] = [$begin=network_time()];
 
 		local result = metric_tbl[index];
 
@@ -450,7 +453,7 @@ function add_data(id: string, index: Index, data: DataPoint)
 
 		++result$num;
 
-		if ( filter?$samples && data?$str )
+		if ( filter?$samples && filter$samples > 0 && data?$str )
 			{
 			if ( ! result?$sample_queue )
 				result$sample_queue = Queue::init([$max_len=filter$samples]);
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 9dab73c43e..06f14219d1 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -35,6 +35,11 @@ export {
 	## At the end of each interval the counter is reset.
 	const sqli_requests_interval = 5min &redef;
 
+	## Collecting samples will add extra data to notice emails
+	## by collecting some sample SQL injection url paths.  Disable
+	## sample collection by setting this value to 0.
+	const collect_SQLi_samples = 5 &redef;
+
 	## Regular expression is used to match URI based SQL injections.
 	const match_sql_injection_uri = 
 		  /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/
@@ -45,20 +50,28 @@ export {
 		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
+function format_sqli_samples(samples: vector of string): string
+	{
+	local ret = "SQL Injection samples\n---------------------";
+	for ( i in samples )
+		ret += "\n" + samples[i];
+	return ret;
+	}
+
 event bro_init() &priority=3
 	{
 	# Add filters to the metrics so that the metrics framework knows how to 
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
-	
-	Metrics::add_filter("http.sqli.attacker", 
+	Metrics::add_filter("http.sqli.attacker",
 	                    [$every=sqli_requests_interval,
 	                     $measure=set(Metrics::SUM),
 	                     $threshold=sqli_requests_threshold,
-	                     $samples=10,
+	                     $samples=collect_SQLi_samples,
 	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	NOTICE([$note=SQL_Injection_Attacker, 
+	                     	NOTICE([$note=SQL_Injection_Attacker,
 	                     	        $msg="An SQL injection attacker was discovered!",
+	                     	        $email_body_sections=vector(format_sqli_samples(val$samples)),
 	                     	        $src=index$host,
 	                     	        $identifier=cat(index$host)]);
 	                     }, $log=F]);
@@ -67,10 +80,11 @@ event bro_init() &priority=3
 	                    [$every=sqli_requests_interval,
 	                     $measure=set(Metrics::SUM),
 	                     $threshold=sqli_requests_threshold,
-	                     $samples=10,
+	                     $samples=collect_SQLi_samples,
 	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	NOTICE([$note=SQL_Injection_Victim, 
+	                     	NOTICE([$note=SQL_Injection_Victim,
 	                     	        $msg="An SQL injection victim was discovered!",
+	                     	        $email_body_sections=vector(format_sqli_samples(val$samples)),
 	                     	        $src=index$host,
 	                     	        $identifier=cat(index$host)]);
 	                     }, $log=F]);

From 08538211e178846262251824c0f9c35917257c1b Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 20 Nov 2012 02:08:49 -0500
Subject: [PATCH 16/70] Some test updates.

---
 scripts/base/frameworks/metrics/main.bro      |  2 +-
 .../frameworks/metrics/conn-example.bro       |  4 +--
 .../frameworks/metrics/http-example.bro       | 17 +++---------
 .../policy/frameworks/metrics/ssl-example.bro | 10 +++----
 .../protocols/conn/conn-stats-per-host.bro    |  3 +++
 scripts/policy/protocols/conn/metrics.bro     |  9 ++++---
 scripts/policy/protocols/conn/scan.bro        |  4 ++-
 scripts/policy/protocols/smtp/metrics.bro     | 27 +++++++++++--------
 .../canonified_loaded_scripts.log             |  5 ++--
 .../manager-1.metrics.log                     | 14 +++++-----
 .../manager-1.notice.log                      | 10 +++----
 .../manager-1.notice.log                      | 10 +++----
 .../notice.log                                | 10 +++----
 13 files changed, 62 insertions(+), 63 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 6b587a0939..48f11ef7f7 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -135,7 +135,7 @@ export {
 		measure:           set[Calculation]         &optional;
 		## A predicate so that you can decide per index if you would like
 		## to accept the data being inserted.
-		pred:              function(index: Metrics::Index, data: DataPoint): bool &optional;
+		pred:              function(index: Metrics::Index, data: Metrics::DataPoint): bool &optional;
 		## A function to normalize the index.  This can be used to aggregate or
 		## normalize the entire index.
 		normalize_func:    function(index: Metrics::Index): Index &optional;
diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro
index 00c82f840d..e5c604a5b2 100644
--- a/scripts/policy/frameworks/metrics/conn-example.bro
+++ b/scripts/policy/frameworks/metrics/conn-example.bro
@@ -16,6 +16,6 @@ event bro_init()
 
 event connection_established(c: connection)
 	{
-	Metrics::add_data("conns.originated", [$host=c$id$orig_h], 1);
-	Metrics::add_data("conns.responded",  [$host=c$id$resp_h], 1);
+	Metrics::add_data("conns.originated", [$host=c$id$orig_h], [$num=1]);
+	Metrics::add_data("conns.responded",  [$host=c$id$resp_h], [$num=1]);
 	}
diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro
index 58ca4e6614..3c60f3c931 100644
--- a/scripts/policy/frameworks/metrics/http-example.bro
+++ b/scripts/policy/frameworks/metrics/http-example.bro
@@ -6,15 +6,6 @@
 @load base/protocols/http
 @load base/utils/site
 
-redef enum Metrics::ID += {
-	## Measures HTTP requests indexed on both the request host and the response
-	## code from the server.
-	HTTP_REQUESTS_BY_STATUS_CODE,
-
-	## Currently unfinished and not working.
-	HTTP_REQUESTS_BY_HOST_HEADER,
-};
-
 event bro_init()
 	{
 	# TODO: these are waiting on a fix with table vals + records before they will work.
@@ -24,14 +15,14 @@ event bro_init()
 	#                     $break_interval=1min]);
 	
 	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table,
-	                                                   $break_interval=1min]);
+	Metrics::add_filter("http.request.by_status_code", [$aggregation_table=Site::local_nets_table,
+	                                                    $break_interval=1min]);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if ( rec?$host )
-		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host], 1);
+		Metrics::add_data("http.request.by_host_header", [$str=rec$host], [$num=1]);
 	if ( rec?$status_code )
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], 1);
+		Metrics::add_data("http.request.by_status_code", [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], [$num=1]);
 	}
diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro
index 5ec675779a..64e63bc215 100644
--- a/scripts/policy/frameworks/metrics/ssl-example.bro
+++ b/scripts/policy/frameworks/metrics/ssl-example.bro
@@ -6,15 +6,11 @@
 @load base/frameworks/metrics
 @load base/protocols/ssl
 
-redef enum Metrics::ID += {
-	SSL_SERVERNAME,
-};
-
 event bro_init()
 	{
-	Metrics::add_filter(SSL_SERVERNAME, 
+	Metrics::add_filter("ssl.by_servername", 
 		[$name="no-google-ssl-servers",
-		 $pred(index: Metrics::Index) = { 
+		 $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
 		    return (/google\.com$/ !in index$str); 
 		 },
 		 $break_interval=10secs
@@ -24,5 +20,5 @@ event bro_init()
 event SSL::log_ssl(rec: SSL::Info)
 	{
 	if ( rec?$server_name )
-		Metrics::add_data(SSL_SERVERNAME, [$str=rec$server_name], 1);
+		Metrics::add_data("ssl.by_servername", [$str=rec$server_name], [$num=1]);
 	}
diff --git a/scripts/policy/protocols/conn/conn-stats-per-host.bro b/scripts/policy/protocols/conn/conn-stats-per-host.bro
index 9e532b8590..df58081163 100644
--- a/scripts/policy/protocols/conn/conn-stats-per-host.bro
+++ b/scripts/policy/protocols/conn/conn-stats-per-host.bro
@@ -1,4 +1,7 @@
 
+@load base/protocols/conn
+@load base/frameworks/metrics
+
 event bro_init() &priority=5
 	{
 	Metrics::add_filter("conn.orig.data", 
diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro
index 910ae4aa6e..0fb5fa2134 100644
--- a/scripts/policy/protocols/conn/metrics.bro
+++ b/scripts/policy/protocols/conn/metrics.bro
@@ -1,9 +1,10 @@
 @load base/frameworks/metrics
+@load base/utils/site
 
 event bro_init() &priority=3
 	{
-	Metrics::add_filter("conns.country", [$break_interval=1hr]);
-	Metrics::add_filter("hosts.active", [$break_interval=1hr]);
+	Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM)]);
+	Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM)]);
 	}
 
 event connection_established(c: connection) &priority=3
@@ -12,10 +13,10 @@ event connection_established(c: connection) &priority=3
 		{
 		local loc = lookup_location(c$id$resp_h);
 		if ( loc?$country_code )
-			Metrics::add_data("conns.country", [$str=loc$country_code], 1);
+			Metrics::add_data("conns.country", [$str=loc$country_code], [$num=1]);
 		}
 		
 	local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h;
 	# There is no index for this.
-	Metrics::add_unique("hosts.active", [], cat(the_host));
+	Metrics::add_data("hosts.active", [], [$str=cat(the_host)]);
 	}
diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro
index 8795cfda06..503b8c34b4 100644
--- a/scripts/policy/protocols/conn/scan.bro
+++ b/scripts/policy/protocols/conn/scan.bro
@@ -4,10 +4,12 @@
 ##!            Seth Hall
 ##!            All the authors of the old scan.bro
 
+@load base/frameworks/notice
+@load base/frameworks/metrics
+
 module Scan;
 
 export {
-
 	redef enum Notice::Type += {
 		AddressScan,
 		PortScan,
diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro
index c3d1bb1e20..ac803ac621 100644
--- a/scripts/policy/protocols/smtp/metrics.bro
+++ b/scripts/policy/protocols/smtp/metrics.bro
@@ -2,31 +2,36 @@
 ##!   "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?"
 ##!   "How much mail is being sent from each local mail server per hour?"
 
+@load base/protocols/smtp
 @load base/frameworks/metrics
+@load base/utils/site
+@load base/utils/directions-and-hosts
 
 module SMTPMetrics;
 
 export {
 	## Define the break intervals for all of the metrics collected and logged by this script.
-	const breaks = 1hr &redef;
+	const breaks=1hr &redef;
 }
 
 event bro_init() &priority=5
 	{
-	Metrics::add_filter("smtp.mailfrom", [$pred(index: Metrics::Index) = { 
-	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); }, 
-	                                      $break_interval=breaks]);
-	Metrics::add_filter("smtp.messages", [$pred(index: Metrics::Index) = { 
-	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); }, 
-	                                      $break_interval=breaks]);
+	Metrics::add_filter("smtp.mailfrom", [$every=breaks,
+	                                      $measure=set(Metrics::SUM),
+	                                      $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
+	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
+	                                      }]);
+	Metrics::add_filter("smtp.messages", [$every=breaks,
+	                                      $measure=set(Metrics::SUM),
+	                                      $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
+	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
+	                                      }]);
 	}
 
 event SMTP::log_smtp(rec: SMTP::Info)
 	{
-	Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], 1);
+	Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], [$num=1]);
 	
 	if ( rec?$mailfrom )
-		Metrics::add_unique("smtp.mailfrom", [$host=rec$id$orig_h], rec$mailfrom);
+		Metrics::add_data("smtp.mailfrom", [$host=rec$id$orig_h], [$str=rec$mailfrom]);
 	}
-
-
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 7fd3a1bdc8..02b7e51030 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2012-11-05-23-29-45
+#open	2012-11-20-06-11-08
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -38,6 +38,7 @@ scripts/base/init-default.bro
   scripts/base/utils/files.bro
   scripts/base/utils/numbers.bro
   scripts/base/utils/paths.bro
+  scripts/base/utils/queue.bro
   scripts/base/utils/strings.bro
   scripts/base/utils/thresholds.bro
   scripts/base/utils/urls.bro
@@ -118,4 +119,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/./main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2012-11-05-23-29-45
+#close	2012-11-20-06-11-08
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index cb1bd5af01..e6c33719aa 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#open	2012-07-20-01-50-41
-#fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
-#types	time	enum	string	addr	string	subnet	count
-1342749041.601712	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1342749041.601712	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
-1342749041.601712	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
-#close	2012-07-20-01-50-49
+#open	2012-11-20-06-46-51
+#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
+#types	time	interval	string	string	string	addr	subnet	time	count	double	double	double	double	double	double	count
+1353394011.192622	3.000000	default	test.metric	-	6.5.4.3	-	-	2	6.0	1.0	5.0	3.0	4.0	2.0	-
+1353394011.192622	3.000000	default	test.metric	-	1.2.3.4	-	-	9	437.0	3.0	95.0	48.555556	674.469136	25.970544	-
+1353394011.192622	3.000000	default	test.metric	-	7.2.1.5	-	-	2	145.0	54.0	91.0	72.5	342.25	18.5	-
+#close	2012-11-20-06-46-51
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
index 6c93cb875e..a5e28de7f9 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2012-07-20-01-51-18
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1342749078.270791	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2012-07-20-01-51-27
+#open	2012-11-20-06-46-22
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double
+1353393982.260495	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-
+#close	2012-11-20-06-46-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
index 88f25b066f..d657cf0ce8 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2012-07-20-01-51-36
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1342749096.545663	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2012-07-20-01-51-45
+#open	2012-11-20-06-45-52
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double
+1353393952.489496	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-
+#close	2012-11-20-06-45-56
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
index f9292344a8..92206c35ce 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2012-10-05-21-45-15
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2012-10-05-21-45-15
+#open	2012-11-20-06-09-07
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double
+1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-
+#close	2012-11-20-06-09-07

From 5921a68e910787c9a12ea95d4038e5b0e555a93f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 20 Nov 2012 11:18:55 -0500
Subject: [PATCH 17/70] More test updates.

---
 doc/scripts/DocSourcesList.cmake                   |  7 +++++++
 .../metrics.log                                    | 14 +++++++-------
 2 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index b95464b6b3..88e5f51025 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -102,6 +102,7 @@ rest_target(${psd} base/utils/files.bro)
 rest_target(${psd} base/utils/numbers.bro)
 rest_target(${psd} base/utils/paths.bro)
 rest_target(${psd} base/utils/patterns.bro)
+rest_target(${psd} base/utils/queue.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
@@ -129,13 +130,18 @@ rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/integration/collective-intel/main.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
+rest_target(${psd} policy/misc/app-metrics.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
+rest_target(${psd} policy/misc/detect-traceroute/main.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
+rest_target(${psd} policy/protocols/conn/conn-stats-per-host.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
+rest_target(${psd} policy/protocols/conn/metrics.bro)
+rest_target(${psd} policy/protocols/conn/scan.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
@@ -153,6 +159,7 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
 rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
+rest_target(${psd} policy/protocols/smtp/metrics.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
 rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index fb6476ee88..784d6d7920 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#open	2012-07-20-01-49-22
-#fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
-#types	time	enum	string	addr	string	subnet	count
-1342748962.841548	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
-1342748962.841548	TEST_METRIC	foo-bar	7.2.1.5	-	-	1
-1342748962.841548	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
-#close	2012-07-20-01-49-22
+#open	2012-11-20-15-05-07
+#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
+#types	time	interval	string	string	string	addr	subnet	time	count	double	double	double	double	double	double	count
+1353423907.236002	3.000000	foo-bar	test.metric	-	6.5.4.3	-	1353423907.236002	1	2.0	2.0	2.0	2.0	0.0	0.0	-
+1353423907.236002	3.000000	foo-bar	test.metric	-	1.2.3.4	-	1353423907.236002	5	221.0	5.0	94.0	44.2	915.36	30.254917	-
+1353423907.236002	3.000000	foo-bar	test.metric	-	7.2.1.5	-	1353423907.236002	1	1.0	1.0	1.0	1.0	0.0	0.0	-
+#close	2012-11-20-15-05-07

From ebacb80d1ca104715c30e95db22cf700acd605d6 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 21 Nov 2012 11:56:39 -0500
Subject: [PATCH 18/70] Add intel detection for apparently successful logins.

---
 .../protocols/ssh/detect-bruteforcing.bro     | 25 ++++++++++---------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index d0f1b63d70..edf6379bec 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -19,6 +19,11 @@ export {
 		## currently implemented.
 		Login_By_Password_Guesser,
 	};
+
+	redef enum Intel::Where += {
+		## An indicator of the login for the intel framework.
+		SSH::SUCCESSFUL_LOGIN,
+	};
 	
 	## The number of failed SSH connections before a host is designated as
 	## guessing passwords.
@@ -33,10 +38,6 @@ export {
 	## heuristic fails and this acts as the whitelist.  The index represents 
 	## client subnets and the yield value represents server subnets.
 	const ignore_guessers: table[subnet] of subnet &redef;
-
-	## Tracks hosts identified as guessing passwords.
-	global password_guessers: set[addr] 
-		&read_expire=guessing_timeout+1hr &synchronized &redef;
 }
 
 event bro_init()
@@ -46,10 +47,15 @@ event bro_init()
 	                                          $measure=set(Metrics::SUM),
 	                                          $threshold=password_guesses_limit,
 	                                          $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                                          	# Generate the notice.
 	                                          	NOTICE([$note=Password_Guessing, 
 	                                          	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %.0f connections).", index$host, val$sum),
 	                                          	        $src=index$host,
 	                                          	        $identifier=cat(index$host)]);
+	                                          	# Insert the guesser into the intel framework.
+	                                          	Intel::insert([$host=index$host,
+	                                          		           $meta=[$source="local", 
+	                                          		                  $desc=fmt("Bro observed %0.f apparently failed SSH connections.", val$sum)]]);
 	                                          }]);
 	}
 
@@ -57,14 +63,9 @@ event SSH::heuristic_successful_login(c: connection)
 	{
 	local id = c$id;
 	
-	# TODO: This is out for the moment pending some more additions to the 
-	#       metrics framework.
-	#if ( id$orig_h in password_guessers )
-	#	{
-	#	NOTICE([$note=Login_By_Password_Guesser,
-	#	        $conn=c,
-	#	        $msg=fmt("Successful SSH login by password guesser %s", id$orig_h)]);
-	#	}
+	Intel::seen([$host=id$orig_h,
+	             $conn=c,
+	             $where=SSH::SUCCESSFUL_LOGIN]);
 	}
 
 event SSH::heuristic_failed_login(c: connection)

From 6bdcdcecf909439814eaa6d07fd400df4a14c744 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 26 Nov 2012 16:17:35 -0500
Subject: [PATCH 19/70] Fixed a problem with metrics aggregation on clusters
 (thanks Jon!).

---
 scripts/base/frameworks/metrics/cluster.bro | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 9650b80554..d20dd733b8 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -189,15 +189,16 @@ function data_added(filter: Filter, index: Index, val: ResultVal)
 event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: ResultVal)
 	{
 	#print fmt("%0.6f MANAGER: receiving index data from %s - %s=%s", network_time(), get_event_peer()$descr, index2str(index), val);
-	
-	local merged_val = merge_result_vals(index_requests[uid, id, filter_name, index], val);
-	index_requests[uid, id, filter_name, index] = merged_val;
+	if ( [uid, id, filter_name, index] in index_requests )
+		index_requests[uid, id, filter_name, index] = merge_result_vals(index_requests[uid, id, filter_name, index], val);
+	else
+		index_requests[uid, id, filter_name, index] = val;
+
 	local ir = index_requests[uid, id, filter_name, index];
 	
 	# Mark that this worker is done.
 	++done_with[uid];
 
-	#print ir;
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
 
 	if ( Cluster::worker_count == done_with[uid] )

From 2add60b4b140f99d246c03c5e344a1eb2ba8e1fb Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Nov 2012 15:22:45 -0500
Subject: [PATCH 20/70] A function wasn't returning a value like it should be.

---
 scripts/base/utils/queue.bro | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/base/utils/queue.bro b/scripts/base/utils/queue.bro
index c5e3bcf906..438529f579 100644
--- a/scripts/base/utils/queue.bro
+++ b/scripts/base/utils/queue.bro
@@ -126,6 +126,7 @@ function merge(q1: Queue, q2: Queue): Queue
 		++i;
 		++j;
 		}
+	return ret;
 	}
 
 function len(q: Queue): count

From 92285a97114aa6dfb830598bbb05687ab2f2b762 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Nov 2012 15:52:41 -0500
Subject: [PATCH 21/70] Fix a race condition when multiple workers report
 intermediate indexes simultaneously.

---
 scripts/base/frameworks/metrics/cluster.bro | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index d20dd733b8..4f2de5577e 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -81,7 +81,7 @@ global index_requests: table[string, string, string, Index] of ResultVal &create
 # an intermediate result has been received. The manager may optionally request
 # the index again before data expires from here if too many workers are crossing
 # the percentage threshold (not implemented yet!).
-global recent_global_view_indexes: table[string, string, Index] of count &create_expire=5mins &default=0;
+global recent_global_view_indexes: table[string, string, Index] of count &create_expire=1min &default=0;
 
 # Add events to the cluster framework to make this work.
 redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
@@ -217,10 +217,12 @@ event Metrics::cluster_index_intermediate_response(id: string, filter_name: stri
 	{
 	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting index data for %s", index2str(index));
-	
+	++recent_global_view_indexes[id, filter_name, index];
+	if ( [id, filter_name, index] in recent_global_view_indexes )
+		return;
+
 	local uid = unique_id("");
 	event Metrics::cluster_index_request(uid, id, filter_name, index);
-	++recent_global_view_indexes[id, filter_name, index];
 	}
 
 event Metrics::cluster_filter_response(uid: string, id: string, filter_name: string, data: MetricTable, done: bool)

From f1b7ca62eef5deb03a043227b374ac8424824b12 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Nov 2012 15:58:29 -0500
Subject: [PATCH 22/70] Actually fix the problem I just tried to fix a minute
 ago.

---
 scripts/base/frameworks/metrics/cluster.bro | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 4f2de5577e..64c7a2d7ee 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -217,10 +217,13 @@ event Metrics::cluster_index_intermediate_response(id: string, filter_name: stri
 	{
 	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting index data for %s", index2str(index));
-	++recent_global_view_indexes[id, filter_name, index];
+
+	# If a worker recently sent this as an intermediate update, don't request it.
 	if ( [id, filter_name, index] in recent_global_view_indexes )
 		return;
 
+	++recent_global_view_indexes[id, filter_name, index];
+
 	local uid = unique_id("");
 	event Metrics::cluster_index_request(uid, id, filter_name, index);
 	}

From 2b72275d7e321b356b0a7b5f7b13944428543f78 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 28 Nov 2012 17:07:30 -0500
Subject: [PATCH 23/70] More updates to clean up scan.bro

---
 scripts/policy/protocols/conn/scan.bro | 326 ++++++++++---------------
 1 file changed, 128 insertions(+), 198 deletions(-)

diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro
index 503b8c34b4..c35e768912 100644
--- a/scripts/policy/protocols/conn/scan.bro
+++ b/scripts/policy/protocols/conn/scan.bro
@@ -27,18 +27,79 @@ export {
 	const default_addr_scan_threshold = 25 &redef;
 	const default_port_scan_threshold = 15 &redef;
 
-	# For address scan
-	const suppress_UDP_scan_checks  = T &redef;
-	const suppress_TCP_scan_checks  = F &redef;
-	const suppress_ICMP_scan_checks = T &redef;
-	
-	global addr_scan_thresh_series: vector of count = vector(100, 200, 300);
-	global port_scan_thresh_series: vector of count = vector(10, 20, 30);
-
 	# Custom threholds based on service for address scan
 	const addr_scan_custom_thresholds: table[port] of count &redef;
 }
 
+
+function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
+	{
+	local service = to_port(index$str);
+
+	return ( service in addr_scan_custom_thresholds &&
+	         val$sum > addr_scan_custom_thresholds[service] );
+	}
+
+function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
+	{
+	local side = Site::is_local_addr(index$host) ? "local" : "remote";
+	local message=fmt("%s scanned %d unique hosts on port %s", index$host, val$unique, index$str);
+
+	NOTICE([$note=AddressScan,
+	        $src=index$host,
+	        $p=to_port(index$str),
+	        $sub=side,
+	        $msg=message,
+	        $identifier=message]);
+	}
+
+function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
+	{
+	local side = Site::is_local_addr(index$host) ? "local" : "remote";
+	local message = fmt("%s scanned %d unique ports of host %s", index$host, val$unique, index$str);
+
+	NOTICE([$note=PortScan, 
+	        $src=index$host,
+	        $dst=to_addr(index$str),
+	        $sub=side,
+	        $msg=message,
+	        $identifier=message]);
+	}
+
+event bro_init() &priority=5
+	{
+	# Add local networks here to determine scan direction
+	# i.e. inbound scan / outbound scan
+	#add Site::local_nets[0.0.0.0/16];
+
+	if ( analyze_addr_scan )
+		{
+		# note=> Addr scan: table [src_ip, port] of set(dst);	
+		# Add filters to the metrics so that the metrics framework knows how to
+		# determine when it looks like an actual attack and how to respond when
+		# thresholds are crossed.
+		Metrics::add_filter("scan.addr.fail", [$log=F,
+		                                       $every=conn_failed_addr_interval,
+		                                       $measure=set(Metrics::UNIQUE),
+		                                       $threshold_func=check_addr_scan_threshold,
+		                                       $threshold=default_addr_scan_threshold,
+		                                       $threshold_crossed=addr_scan_threshold_crossed]); 
+		}
+
+	if ( analyze_port_scan )
+		{
+		# note=> Port Sweep: table[src_ip, dst_ip] of set(port);
+		# Add filters to the metrics so that the metrics framework knows how to
+		# determine when it looks like an actual attack and how to respond when
+		# thresholds are crossed.
+		Metrics::add_filter("scan.port.fail", [$log=F,
+		                                       $every=conn_failed_port_interval,
+		                                       $measure=set(Metrics::UNIQUE),
+		                                       $threshold=default_port_scan_threshold,
+		                                       $threshold_crossed=port_scan_threshold_crossed]); 
+		}
+	}
+
 function is_failed_conn(c: connection): bool
 	{
 	# Sr || ( (hR || ShR) && (data not sent in any direction) ) 
@@ -68,181 +129,56 @@ function is_reverse_failed_conn(c: connection): bool
 	return F;
 	}
 
-function addr_scan_predicate(index: Metrics::Index, data: Metrics::DataPoint): bool
-	{
-	local service = to_port(index$str);
-	local host = index$host;
-
-	local transport_layer_proto = get_port_transport_proto(service);
-	if ( suppress_UDP_scan_checks && (transport_layer_proto == udp) )
-		return F;
-	else if ( suppress_TCP_scan_checks && (transport_layer_proto == tcp) )
-		return F;
-	else if ( suppress_ICMP_scan_checks && (transport_layer_proto == icmp) )
-		return F;
-
-	# TODO: all of this whitelist/blacklist will be done 
-	#       through the upcoming hook mechanism
-	# Blacklisting/whitelisting services
-	#if ( |analyze_services| > 0 )
-	#	{
-	#	if ( service !in analyze_services )
-	#		return F;
-	#	}
-	#else if ( service in skip_services )
-	#	return F;
-	#
-	## Blacklisting/whitelisting subnets
-	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
-	#	return F;
-
-	return T;
-	}
-
-function port_scan_predicate(index: Metrics::Index, data: Metrics::DataPoint): bool
-	{
-	local service = to_port(data$str);
-	local host = index$host;
-
-	local transport_layer_proto = get_port_transport_proto(service);
-	if ( suppress_UDP_scan_checks && (transport_layer_proto == udp) )
-		return F;
-	else if ( suppress_TCP_scan_checks && (transport_layer_proto == tcp) )
-		return F;
-	else if ( suppress_ICMP_scan_checks && (transport_layer_proto == icmp) )
-		return F;
-
-	# TODO: all of this whitelist/blacklist will be done 
-	#       through the upcoming hook mechanism
-	# Blacklisting/whitelisting services
-	#if ( |analyze_services| > 0 )
-	#	{
-	#	if ( service !in analyze_services )
-	#		return F;
-	#	}
-	#else if ( service in skip_services )
-	#	return F;
-	#
-	## Blacklisting/whitelisting subnets
-	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
-	#	return F;
-
-	return T;
-	}
-
-function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
-	{
-	local service = to_port(index$str);
-
-	return ( service in addr_scan_custom_thresholds &&
-	         val$sum > addr_scan_custom_thresholds[service] );
-	}
-
-function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
-	{
-	local direction = Site::is_local_addr(index$host) ? "OutboundScan" : "InboundScan";
-	local message=fmt("%s scanned %d unique hosts on port %s", index$host, val$unique, index$str);
-
-	NOTICE([$note=AddressScan,
-	        $src=index$host,
-	        $p=to_port(index$str),
-	        $sub=direction,
-	        $msg=message,
-	        $identifier=message]);
-	}
-
-function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
-	{
-	local direction = Site::is_local_addr(index$host) ? "OutboundScan" : "InboundScan";
-	local message = fmt("%s scanned %d unique ports of host %s", index$host, val$unique, index$str);
-
-	NOTICE([$note=PortScan, 
-	        $src=index$host,
-	        $dst=to_addr(index$str),
-	        $sub=direction,
-	        $msg=message,
-	        $identifier=message]);
-	}
-
-event bro_init() &priority=5
-	{
-	# Add local networks here to determine scan direction
-	# i.e. inbound scan / outbound scan
-	#add Site::local_nets[0.0.0.0/16];
-
-	if ( analyze_addr_scan )
-		{
-		# note=> Addr scan: table [src_ip, port] of set(dst);	
-		# Add filters to the metrics so that the metrics framework knows how to
-		# determine when it looks like an actual attack and how to respond when
-		# thresholds are crossed.
-		Metrics::add_filter("scan.addr.fail", [$log=F,
-		                                       $every=conn_failed_addr_interval,
-		                                       $measure=set(Metrics::UNIQUE),
-		                                       $pred=addr_scan_predicate,
-		                                       $threshold_func=check_addr_scan_threshold,
-		                                       $threshold=default_addr_scan_threshold,
-		                                       $threshold_crossed=addr_scan_threshold_crossed]); 
-		}
-
-	if ( analyze_port_scan )
-		{
-		# note=> Port Sweep: table[src_ip, dst_ip] of set(port);
-		# Add filters to the metrics so that the metrics framework knows how to
-		# determine when it looks like an actual attack and how to respond when
-		# thresholds are crossed.
-		Metrics::add_filter("scan.port.fail", [$log=F,
-		                                       $every=conn_failed_port_interval,
-		                                       $measure=set(Metrics::UNIQUE),
-		                                       $pred=port_scan_predicate,
-		                                       $threshold=default_port_scan_threshold,
-		                                       $threshold_crossed=port_scan_threshold_crossed]); 
-		}
-	}
-
-## Generated when a SYN-ACK packet is seen in response to a SYN 
-## packet during a TCP handshake. The final ACK of the handshake 
-## in response to SYN-ACK may or may not occur later, one way to 
-## tell is to check the history field of connection to see if the 
-## originator sent an ACK, indicated by ‘A’ in the history string.
-#event connection_established(c: connection)
-#	{
-	# Not useful for scan (too early)
-#	}
-
-## Generated when one endpoint of a TCP connection attempted 
-## to gracefully close the connection, but the other endpoint 
-## is in the TCP_INACTIVE state. This can happen due to split 
-## routing, in which Bro only sees one side of a connection.
-#event connection_half_finished(c: connection)
-#	{
-	# Half connections never were "established", so do scan-checking here.
-	# I am not taking *f cases of c$history into account. Ask Seth if I should
-#	}
-
 function add_metrics(id: conn_id, reverse: bool)
 	{
 	local scanner:      addr;
-	local victim:       string;
-	local scanned_port: string;
+	local victim:       addr;
+	local scanned_port: port;
 
 	if ( reverse )
 		{
 		scanner      = id$resp_h;
-		victim       = cat(id$orig_h);
-		scanned_port = fmt("%s", id$orig_p);
+		victim       = id$orig_h;
+		scanned_port = id$orig_p;
 		}
 	else
 		{
 		scanner      = id$orig_h;
-		victim       = cat(id$resp_h);
-		scanned_port = fmt("%s", id$resp_p);
+		victim       = id$resp_h;
+		scanned_port = id$resp_p;
 		}
 
+	# Defaults to be implemented with a hook...
+	#local transport_layer_proto = get_port_transport_proto(service);
+	#if ( suppress_UDP_scan_checks && (transport_layer_proto == udp) )
+	#	return F;
+	#else if ( suppress_TCP_scan_checks && (transport_layer_proto == tcp) )
+	#	return F;
+	#else if ( suppress_ICMP_scan_checks && (transport_layer_proto == icmp) )
+	#	return F;
+
+	# TODO: all of this whitelist/blacklist will be done 
+	#       through the upcoming hook mechanism
+	# Blacklisting/whitelisting services
+	#if ( |analyze_services| > 0 )
+	#	{
+	#	if ( service !in analyze_services )
+	#		return F;
+	#	}
+	#else if ( service in skip_services )
+	#	return F;
+	#
+	## Blacklisting/whitelisting subnets
+	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
+	#	return F;
+
+	# Probably do a hook point here?
 	if ( analyze_addr_scan )
-		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=scanned_port], [$str=victim]);
+		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
+
+	# Probably do a hook point here?
 	if ( analyze_port_scan )
-		Metrics::add_data("scan.port.fail", [$host=scanner, $str=victim],       [$str=scanned_port]);
+		Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 
 ## Generated for an unsuccessful connection attempt. This 
@@ -278,43 +214,37 @@ event connection_rejected(c: connection)
 ## TCP connection aborted by sending a RST packet.
 event connection_reset(c: connection)
 	{
-	local is_reverse_scan = F;
-	local is_scan = F;
-
 	if ( is_failed_conn(c) )
-		{
-		is_scan = T;
-		}
+		add_metrics(c$id, F);
 	else if ( is_reverse_failed_conn(c) )
-		{
-		is_scan = T;
-		is_reverse_scan = T;
-		}
-
-	if ( is_scan )
-		{
-		add_metrics(c$id, is_reverse_scan);
-		}
+		add_metrics(c$id, T);
 	}
 
 ## Generated for each still-open connection when Bro terminates.
 event connection_pending(c: connection)
 	{
-	local is_reverse_scan = F;
-	local is_scan = F;
-
 	if ( is_failed_conn(c) )
-		{
-		is_scan = T;
-		}
+		add_metrics(c$id, F);
 	else if ( is_reverse_failed_conn(c) )
-		{
-		is_scan = T;
-		is_reverse_scan = T;
-		}
+		add_metrics(c$id, T);
+	}
 
-	if ( is_scan )
-		{
-		add_metrics(c$id, is_reverse_scan);
-		}
-	}
\ No newline at end of file
+## Generated when a SYN-ACK packet is seen in response to a SYN 
+## packet during a TCP handshake. The final ACK of the handshake 
+## in response to SYN-ACK may or may not occur later, one way to 
+## tell is to check the history field of connection to see if the 
+## originator sent an ACK, indicated by ‘A’ in the history string.
+#event connection_established(c: connection)
+#	{
+	# Not useful for scan (too early)
+#	}
+
+## Generated when one endpoint of a TCP connection attempted 
+## to gracefully close the connection, but the other endpoint 
+## is in the TCP_INACTIVE state. This can happen due to split 
+## routing, in which Bro only sees one side of a connection.
+#event connection_half_finished(c: connection)
+#	{
+	# Half connections never were "established", so do scan-checking here.
+	# I am not taking *f cases of c$history into account. Ask Seth if I should
+#	}

From 2484295db3ad1a93879f19019111cb40db27b3e5 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 30 Nov 2012 09:48:52 -0500
Subject: [PATCH 24/70] scan.bro updates.

---
 scripts/policy/protocols/conn/scan.bro | 132 +++++++++++--------------
 1 file changed, 59 insertions(+), 73 deletions(-)

diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro
index c35e768912..dde96bf939 100644
--- a/scripts/policy/protocols/conn/scan.bro
+++ b/scripts/policy/protocols/conn/scan.bro
@@ -11,31 +11,39 @@ module Scan;
 
 export {
 	redef enum Notice::Type += {
-		AddressScan,
-		PortScan,
+		## Address scans detect that a host appears to be scanning
+		## some number of other hosts on a single port.
+		Address_Scan,
+		## Port scans detect that a host appears to be scanning a
+		## single other host on numerous ports.
+		Port_Scan,
 		};
 
-	const analyze_addr_scan = T &redef;
-	const analyze_port_scan = T &redef;
+	## Interval at which to watch for an address scan detection threshold to be crossed.
+	const addr_scan_interval = 5min &redef;
+	## Interval at which to watch for a port scan detection threshold to be crossed.
+	const port_scan_interval = 5min &redef;
 
-	## Interval at which to watch for the
-	## :bro:id:`Scan::conn_failed_(port|addr)_threshold` variable to be crossed.
-	## At the end of each interval the counter is reset.
-	const conn_failed_addr_interval = 5min &redef;
-	const conn_failed_port_interval = 5min &redef;
+	## The threshold of a unique number of hosts a scanning host has to have failed 
+	## connections with on a single port.
+	const addr_scan_threshold = 25 &redef;
+	## The threshold of a number of unique ports a scanning host has to have failed
+	## connections with on a single victim host.
+	const port_scan_threshold = 15 &redef;
 
-	const default_addr_scan_threshold = 25 &redef;
-	const default_port_scan_threshold = 15 &redef;
-
-	# Custom threholds based on service for address scan
+	## Custom threholds based on service for address scan.  This is primarily 
+	## useful for setting reduced thresholds for specific ports.
 	const addr_scan_custom_thresholds: table[port] of count &redef;
 }
 
 
 function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
 	{
-	local service = to_port(index$str);
+	# We don't need to do this if no custom thresholds are defined.
+	if ( |addr_scan_custom_thresholds| == 0 )
+		return F;
 
+	local service = to_port(index$str);
 	return ( service in addr_scan_custom_thresholds &&
 	         val$sum > addr_scan_custom_thresholds[service] );
 	}
@@ -45,12 +53,12 @@ function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
 	local message=fmt("%s scanned %d unique hosts on port %s", index$host, val$unique, index$str);
 
-	NOTICE([$note=AddressScan,
+	NOTICE([$note=Address_Scan,
 	        $src=index$host,
 	        $p=to_port(index$str),
 	        $sub=side,
 	        $msg=message,
-	        $identifier=message]);
+	        $identifier=cat(index)]);
 	}
 
 function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
@@ -58,58 +66,46 @@ function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
 	local message = fmt("%s scanned %d unique ports of host %s", index$host, val$unique, index$str);
 
-	NOTICE([$note=PortScan, 
+	NOTICE([$note=Port_Scan, 
 	        $src=index$host,
 	        $dst=to_addr(index$str),
 	        $sub=side,
 	        $msg=message,
-	        $identifier=message]);
+	        $identifier=cat(index)]);
 	}
 
 event bro_init() &priority=5
 	{
-	# Add local networks here to determine scan direction
-	# i.e. inbound scan / outbound scan
-	#add Site::local_nets[0.0.0.0/16];
+	# note=> Addr scan: table [src_ip, port] of set(dst);	
+	# Add filters to the metrics so that the metrics framework knows how to
+	# determine when it looks like an actual attack and how to respond when
+	# thresholds are crossed.
+	Metrics::add_filter("scan.addr.fail", [$log=F,
+	                                       $every=addr_scan_interval,
+	                                       $measure=set(Metrics::UNIQUE),
+	                                       $threshold_func=check_addr_scan_threshold,
+	                                       $threshold=addr_scan_threshold,
+	                                       $threshold_crossed=addr_scan_threshold_crossed]); 
 
-	if ( analyze_addr_scan )
-		{
-		# note=> Addr scan: table [src_ip, port] of set(dst);	
-		# Add filters to the metrics so that the metrics framework knows how to
-		# determine when it looks like an actual attack and how to respond when
-		# thresholds are crossed.
-		Metrics::add_filter("scan.addr.fail", [$log=F,
-		                                       $every=conn_failed_addr_interval,
-		                                       $measure=set(Metrics::UNIQUE),
-		                                       $threshold_func=check_addr_scan_threshold,
-		                                       $threshold=default_addr_scan_threshold,
-		                                       $threshold_crossed=addr_scan_threshold_crossed]); 
-		}
-
-	if ( analyze_port_scan )
-		{
-		# note=> Port Sweep: table[src_ip, dst_ip] of set(port);
-		# Add filters to the metrics so that the metrics framework knows how to
-		# determine when it looks like an actual attack and how to respond when
-		# thresholds are crossed.
-		Metrics::add_filter("scan.port.fail", [$log=F,
-		                                       $every=conn_failed_port_interval,
-		                                       $measure=set(Metrics::UNIQUE),
-		                                       $threshold=default_port_scan_threshold,
-		                                       $threshold_crossed=port_scan_threshold_crossed]); 
-		}
+	# note=> Port Sweep: table[src_ip, dst_ip] of set(port);
+	# Add filters to the metrics so that the metrics framework knows how to
+	# determine when it looks like an actual attack and how to respond when
+	# thresholds are crossed.
+	Metrics::add_filter("scan.port.fail", [$log=F,
+	                                       $every=port_scan_interval,
+	                                       $measure=set(Metrics::UNIQUE),
+	                                       $threshold=port_scan_threshold,
+	                                       $threshold_crossed=port_scan_threshold_crossed]); 
 	}
 
 function is_failed_conn(c: connection): bool
 	{
 	# Sr || ( (hR || ShR) && (data not sent in any direction) ) 
 	if ( (c$orig$state == TCP_SYN_SENT && c$resp$state == TCP_RESET) ||
-	     (
-	      ((c$orig$state == TCP_RESET && c$resp$state == TCP_SYN_ACK_SENT) ||
+	     (((c$orig$state == TCP_RESET && c$resp$state == TCP_SYN_ACK_SENT) ||
 	       (c$orig$state == TCP_RESET && c$resp$state == TCP_ESTABLISHED && "S" in c$history )
-	      ) &&
-	      !("D" in c$history || "d" in c$history)
-	     ) )
+	      ) && /[Dd]/ !in c$history )
+	   )
 		return T;
 	return F;
 	}
@@ -119,21 +115,19 @@ function is_reverse_failed_conn(c: connection): bool
 	# reverse scan i.e. conn dest is the scanner
 	# sR || ( (Hr || sHr) && (data not sent in any direction) ) 
 	if ( (c$resp$state == TCP_SYN_SENT && c$orig$state == TCP_RESET) ||
-	     (
-	      ((c$resp$state == TCP_RESET && c$orig$state == TCP_SYN_ACK_SENT) ||
+	     (((c$resp$state == TCP_RESET && c$orig$state == TCP_SYN_ACK_SENT) ||
 	       (c$resp$state == TCP_RESET && c$orig$state == TCP_ESTABLISHED && "s" in c$history )
-	      ) &&
-	      !("D" in c$history || "d" in c$history)
-	     ) )
+	      ) && /[Dd]/ !in c$history )
+	   )
 		return T;
 	return F;
 	}
 
 function add_metrics(id: conn_id, reverse: bool)
 	{
-	local scanner:      addr;
-	local victim:       addr;
-	local scanned_port: port;
+	local scanner      = id$orig_h;
+	local victim       = id$resp_h;
+	local scanned_port = id$resp_p;
 
 	if ( reverse )
 		{
@@ -141,12 +135,6 @@ function add_metrics(id: conn_id, reverse: bool)
 		victim       = id$orig_h;
 		scanned_port = id$orig_p;
 		}
-	else
-		{
-		scanner      = id$orig_h;
-		victim       = id$resp_h;
-		scanned_port = id$resp_p;
-		}
 
 	# Defaults to be implemented with a hook...
 	#local transport_layer_proto = get_port_transport_proto(service);
@@ -173,12 +161,10 @@ function add_metrics(id: conn_id, reverse: bool)
 	#	return F;
 
 	# Probably do a hook point here?
-	if ( analyze_addr_scan )
-		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
+	Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
 
 	# Probably do a hook point here?
-	if ( analyze_port_scan )
-		Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
+	Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 
 ## Generated for an unsuccessful connection attempt. This 
@@ -236,7 +222,7 @@ event connection_pending(c: connection)
 ## originator sent an ACK, indicated by ‘A’ in the history string.
 #event connection_established(c: connection)
 #	{
-	# Not useful for scan (too early)
+#	# Not useful for scan (too early)
 #	}
 
 ## Generated when one endpoint of a TCP connection attempted 
@@ -245,6 +231,6 @@ event connection_pending(c: connection)
 ## routing, in which Bro only sees one side of a connection.
 #event connection_half_finished(c: connection)
 #	{
-	# Half connections never were "established", so do scan-checking here.
-	# I am not taking *f cases of c$history into account. Ask Seth if I should
+#	# Half connections never were "established", so do scan-checking here.
+#	# I am not taking *f cases of c$history into account. Ask Seth if I should
 #	}

From 96f850ca4e922a665d9d67414d65a6f7a76ea1f6 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 30 Nov 2012 09:49:16 -0500
Subject: [PATCH 25/70] Moving scan.bro to a more appropriate place.

---
 scripts/policy/{protocols/conn => misc}/scan.bro | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename scripts/policy/{protocols/conn => misc}/scan.bro (100%)

diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/misc/scan.bro
similarity index 100%
rename from scripts/policy/protocols/conn/scan.bro
rename to scripts/policy/misc/scan.bro

From bb7db648417983f4288eaf86e2ae8e45cec29ef8 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 30 Nov 2012 09:51:20 -0500
Subject: [PATCH 26/70] Fixed Sheharbano's name.

---
 scripts/policy/misc/scan.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index dde96bf939..dbafb51bea 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -1,6 +1,6 @@
 ##! Scan detection
 ##!
-##! ..Authors: Sheharbano Kattack
+##! ..Authors: Sheharbano Khattak
 ##!            Seth Hall
 ##!            All the authors of the old scan.bro
 

From 1542b3696ee3ceb0054861ae0aacad4bc3842336 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 30 Nov 2012 11:27:09 -0500
Subject: [PATCH 27/70] Changed how traceroute detection works by having it
 check for low ttl packets after detecting time exceeded messages.

---
 .../policy/misc/detect-traceroute/main.bro    | 34 +++++++++++++------
 1 file changed, 24 insertions(+), 10 deletions(-)

diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index 0709834cea..fd19e7fef1 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -29,7 +29,7 @@ export {
 	## Defines the threshold for ICMP Time Exceeded messages for a src-dst pair.
 	## This threshold only comes into play after a host is found to be
 	## sending low ttl packets.
-	const icmp_time_exceeded_threshold = 2 &redef;
+	const icmp_time_exceeded_threshold = 3 &redef;
 
 	## Interval at which to watch for the
 	## :bro:id:`ICMPTimeExceeded::icmp_time_exceeded_threshold` variable to be crossed.
@@ -49,10 +49,21 @@ export {
 	global log_traceroute: event(rec: Traceroute::Info);
 }
 
-# Track hosts that have sent low TTL packets.
+# Track hosts that have sent low TTL packets and which hosts they 
+# sent them to.
 global low_ttlers: set[addr, addr] = {} &create_expire=2min &synchronized;
 
-event bro_init() &priority=3
+function traceroute_detected(src: addr, dst: addr)
+	{
+	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
+	NOTICE([$note=Traceroute::Detected,
+	        $msg=fmt("%s seems to be running traceroute", src),
+	        $src=src, $dst=dst,
+	        $identifier=cat(src)]);
+	}
+
+
+event bro_init() &priority=5
 	{
 	Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
 
@@ -65,11 +76,15 @@ event bro_init() &priority=3
 	                     	local parts = split1(index$str, /-/);
 	                     	local src = to_addr(parts[1]);
 	                     	local dst = to_addr(parts[2]);
-	                     	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
-	                     	NOTICE([$note=Traceroute::Detected,
-	                     	        $msg=fmt("%s seems to be running traceroute", src),
-	                     	        $src=src, $dst=dst,
-	                     	        $identifier=parts[1]]);
+	                     	if ( require_low_ttl_packets )
+	                     		{
+	                     		when ( [src, dst] in low_ttlers )
+	                     			{
+	                     			traceroute_detected(src, dst);
+		                     		}
+		                     	}
+                     		else
+                     			traceroute_detected(src, dst);
 	                     }]);
 	}
 
@@ -82,6 +97,5 @@ event signature_match(state: signature_state, msg: string, data: string)
 
 event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
 	{
-	if ( ! require_low_ttl_packets || [context$id$orig_h, context$id$resp_h] in low_ttlers )
-		Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
+	Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
 	}

From 4bb8babb4556b9d0bd30538e91c60d592845fcfa Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 3 Dec 2012 14:58:11 -0500
Subject: [PATCH 28/70] Small change to load the correct scan file in
 local.bro.

---
 scripts/policy/misc/scan.bro | 50 ++++++++++++++++++------------------
 scripts/site/local.bro       |  2 +-
 2 files changed, 26 insertions(+), 26 deletions(-)

diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index dbafb51bea..a8ed6e8359 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -98,31 +98,6 @@ event bro_init() &priority=5
 	                                       $threshold_crossed=port_scan_threshold_crossed]); 
 	}
 
-function is_failed_conn(c: connection): bool
-	{
-	# Sr || ( (hR || ShR) && (data not sent in any direction) ) 
-	if ( (c$orig$state == TCP_SYN_SENT && c$resp$state == TCP_RESET) ||
-	     (((c$orig$state == TCP_RESET && c$resp$state == TCP_SYN_ACK_SENT) ||
-	       (c$orig$state == TCP_RESET && c$resp$state == TCP_ESTABLISHED && "S" in c$history )
-	      ) && /[Dd]/ !in c$history )
-	   )
-		return T;
-	return F;
-	}
-
-function is_reverse_failed_conn(c: connection): bool
-	{
-	# reverse scan i.e. conn dest is the scanner
-	# sR || ( (Hr || sHr) && (data not sent in any direction) ) 
-	if ( (c$resp$state == TCP_SYN_SENT && c$orig$state == TCP_RESET) ||
-	     (((c$resp$state == TCP_RESET && c$orig$state == TCP_SYN_ACK_SENT) ||
-	       (c$resp$state == TCP_RESET && c$orig$state == TCP_ESTABLISHED && "s" in c$history )
-	      ) && /[Dd]/ !in c$history )
-	   )
-		return T;
-	return F;
-	}
-
 function add_metrics(id: conn_id, reverse: bool)
 	{
 	local scanner      = id$orig_h;
@@ -167,6 +142,31 @@ function add_metrics(id: conn_id, reverse: bool)
 	Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 
+function is_failed_conn(c: connection): bool
+	{
+	# Sr || ( (hR || ShR) && (data not sent in any direction) ) 
+	if ( (c$orig$state == TCP_SYN_SENT && c$resp$state == TCP_RESET) ||
+	     (((c$orig$state == TCP_RESET && c$resp$state == TCP_SYN_ACK_SENT) ||
+	       (c$orig$state == TCP_RESET && c$resp$state == TCP_ESTABLISHED && "S" in c$history )
+	      ) && /[Dd]/ !in c$history )
+	   )
+		return T;
+	return F;
+	}
+
+function is_reverse_failed_conn(c: connection): bool
+	{
+	# reverse scan i.e. conn dest is the scanner
+	# sR || ( (Hr || sHr) && (data not sent in any direction) ) 
+	if ( (c$resp$state == TCP_SYN_SENT && c$orig$state == TCP_RESET) ||
+	     (((c$resp$state == TCP_RESET && c$orig$state == TCP_SYN_ACK_SENT) ||
+	       (c$resp$state == TCP_RESET && c$orig$state == TCP_ESTABLISHED && "s" in c$history )
+	      ) && /[Dd]/ !in c$history )
+	   )
+		return T;
+	return F;
+	}
+
 ## Generated for an unsuccessful connection attempt. This 
 ## event is raised when an originator unsuccessfully attempted 
 ## to establish a connection. “Unsuccessful” is defined as at least 
diff --git a/scripts/site/local.bro b/scripts/site/local.bro
index acbef96721..918bc0f462 100644
--- a/scripts/site/local.bro
+++ b/scripts/site/local.bro
@@ -9,7 +9,7 @@
 @load tuning/defaults
 
 # Load the scan detection script.
-@load protocols/conn/scan
+@load misc/scan
 
 # Generate notices when vulnerable versions of software are discovered.
 # The default is to only monitor software found in the address space defined

From 3ca0333294dcaafff68a10cc44ce17a005a0df69 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 4 Dec 2012 00:15:19 -0500
Subject: [PATCH 29/70] Fix to checking metrics thresholds at the end of the
 break interval ($every field).

---
 scripts/base/frameworks/metrics/cluster.bro | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 64c7a2d7ee..6a536efb85 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -143,7 +143,7 @@ event Metrics::cluster_filter_request(uid: string, id: string, filter_name: stri
 	event Metrics::send_data(uid, id, filter_name, store[id, filter_name]);
 	
 	# Lookup the actual filter and reset it, the reference to the data
-	# currently stored will be maintained interally by the send_data event.
+	# currently stored will be maintained internally by the send_data event.
 	reset(filter_store[id, filter_name]);
 	}
 	
@@ -232,6 +232,10 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 	
+	# Mark another worker as being "done" for this uid.
+	if ( done )
+		++done_with[uid];
+
 	local local_data = filter_results[uid, id, filter_name];
 	for ( index in data )
 		{
@@ -239,12 +243,19 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 			local_data[index] = merge_result_vals(local_data[index], data[index]);
 		else
 			local_data[index] = data[index];
+
+		# If a filter is done being collected, thresholds for each index
+		# need to checked so we're doing it here to avoid doubly iterating 
+		# over each index.
+		if ( Cluster::worker_count == done_with[uid] )
+			{
+			if ( check_thresholds(filter_store[id, filter_name], index, local_data[index], 1.0) )
+				{
+				threshold_crossed(filter_store[id, filter_name], index, local_data[index]);
+				}
+			}
 		}
 	
-	# Mark another worker as being "done" for this uid.
-	if ( done )
-		++done_with[uid];
-	
 	# If the data has been collected from all peers, we are done and ready to log.
 	if ( Cluster::worker_count == done_with[uid] )
 		{

From e769ab469f400c7f1d11a35709b52bc3ca66cb86 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 4 Dec 2012 00:15:49 -0500
Subject: [PATCH 30/70] Comment and indentation cleanup.

---
 .../policy/misc/detect-traceroute/main.bro    |  8 ++++----
 scripts/policy/misc/scan.bro                  | 20 ++-----------------
 2 files changed, 6 insertions(+), 22 deletions(-)

diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index fd19e7fef1..051d81c5c7 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -81,10 +81,10 @@ event bro_init() &priority=5
 	                     		when ( [src, dst] in low_ttlers )
 	                     			{
 	                     			traceroute_detected(src, dst);
-		                     		}
-		                     	}
-                     		else
-                     			traceroute_detected(src, dst);
+	                     			}
+	                     		}
+	                     	else
+	                     		traceroute_detected(src, dst);
 	                     }]);
 	}
 
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index a8ed6e8359..decc34c894 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -76,10 +76,7 @@ function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 
 event bro_init() &priority=5
 	{
-	# note=> Addr scan: table [src_ip, port] of set(dst);	
-	# Add filters to the metrics so that the metrics framework knows how to
-	# determine when it looks like an actual attack and how to respond when
-	# thresholds are crossed.
+	# Note: addr scans are trcked similar to:  table[src_ip, port] of set(dst);	
 	Metrics::add_filter("scan.addr.fail", [$log=F,
 	                                       $every=addr_scan_interval,
 	                                       $measure=set(Metrics::UNIQUE),
@@ -87,10 +84,7 @@ event bro_init() &priority=5
 	                                       $threshold=addr_scan_threshold,
 	                                       $threshold_crossed=addr_scan_threshold_crossed]); 
 
-	# note=> Port Sweep: table[src_ip, dst_ip] of set(port);
-	# Add filters to the metrics so that the metrics framework knows how to
-	# determine when it looks like an actual attack and how to respond when
-	# thresholds are crossed.
+	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
 	Metrics::add_filter("scan.port.fail", [$log=F,
 	                                       $every=port_scan_interval,
 	                                       $measure=set(Metrics::UNIQUE),
@@ -224,13 +218,3 @@ event connection_pending(c: connection)
 #	{
 #	# Not useful for scan (too early)
 #	}
-
-## Generated when one endpoint of a TCP connection attempted 
-## to gracefully close the connection, but the other endpoint 
-## is in the TCP_INACTIVE state. This can happen due to split 
-## routing, in which Bro only sees one side of a connection.
-#event connection_half_finished(c: connection)
-#	{
-#	# Half connections never were "established", so do scan-checking here.
-#	# I am not taking *f cases of c$history into account. Ask Seth if I should
-#	}

From 3af4517e2a92efc657f65121cdbf5e54b4e1b999 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 4 Dec 2012 11:04:01 -0500
Subject: [PATCH 31/70] Adding an $end time for result values to measure the
 length of time a measurement took.

---
 scripts/base/frameworks/metrics/main.bro           | 13 ++++++++++++-
 scripts/policy/misc/scan.bro                       |  4 ++--
 .../manager-1.metrics.log                          | 14 +++++++-------
 .../metrics.log                                    | 14 +++++++-------
 4 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 48f11ef7f7..8d7ea26bc7 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -71,6 +71,9 @@ export {
 		## The time when this result was first started.
 		begin:    time          &log;
 
+		## The time when the last value was added to this result.
+		end:      time          &log;
+
 		## The number of measurements received.
 		num:      count         &log &default=0;
 
@@ -277,6 +280,12 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	{
 	local result: ResultVal;
 	
+	# Merge $begin (take the earliest one)
+	result$begin = rv1$begin < rv2$begin ? rv1$begin : rv2$begin;
+
+	# Merge $end (take the latest one)
+	result$end = rv1$end > rv2$end ? rv1$end : rv2$end;
+
 	# Merge $num
 	result$num = rv1$num + rv2$num;
 
@@ -442,7 +451,7 @@ function add_data(id: string, index: Index, data: DataPoint)
 		
 		local metric_tbl = store[id, filter$name];
 		if ( index !in metric_tbl )
-			metric_tbl[index] = [$begin=network_time()];
+			metric_tbl[index] = [$begin=network_time(), $end=network_time()];
 
 		local result = metric_tbl[index];
 
@@ -452,6 +461,8 @@ function add_data(id: string, index: Index, data: DataPoint)
 			val = data?$dbl ? data$dbl : data$num;
 
 		++result$num;
+		# Continually update the $end field.
+		result$end=network_time();
 
 		if ( filter?$samples && filter$samples > 0 && data?$str )
 			{
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index decc34c894..42350bbe77 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -51,7 +51,7 @@ function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVa
 function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
-	local message=fmt("%s scanned %d unique hosts on port %s", index$host, val$unique, index$str);
+	local message=fmt("%s scanned %d unique hosts on port %s in %s", index$host, val$unique, index$str, val$end-val$begin);
 
 	NOTICE([$note=Address_Scan,
 	        $src=index$host,
@@ -64,7 +64,7 @@ function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
-	local message = fmt("%s scanned %d unique ports of host %s", index$host, val$unique, index$str);
+	local message = fmt("%s scanned %d unique ports of host %s in %s", index$host, val$unique, index$str, val$end-val$begin);
 
 	NOTICE([$note=Port_Scan, 
 	        $src=index$host,
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index e6c33719aa..98794673f1 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#open	2012-11-20-06-46-51
-#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
-#types	time	interval	string	string	string	addr	subnet	time	count	double	double	double	double	double	double	count
-1353394011.192622	3.000000	default	test.metric	-	6.5.4.3	-	-	2	6.0	1.0	5.0	3.0	4.0	2.0	-
-1353394011.192622	3.000000	default	test.metric	-	1.2.3.4	-	-	9	437.0	3.0	95.0	48.555556	674.469136	25.970544	-
-1353394011.192622	3.000000	default	test.metric	-	7.2.1.5	-	-	2	145.0	54.0	91.0	72.5	342.25	18.5	-
-#close	2012-11-20-06-46-51
+#open	2012-12-04-15-53-23
+#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
+#types	time	interval	string	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
+1354636403.682565	3.000000	default	test.metric	-	6.5.4.3	-	1354636401.774655	1354636401.782720	2	6.0	1.0	5.0	3.0	4.0	2.0	-
+1354636403.682565	3.000000	default	test.metric	-	1.2.3.4	-	1354636401.774655	1354636401.782720	9	437.0	3.0	95.0	48.555556	674.469136	25.970544	-
+1354636403.682565	3.000000	default	test.metric	-	7.2.1.5	-	1354636401.774655	1354636401.782720	2	145.0	54.0	91.0	72.5	342.25	18.5	-
+#close	2012-12-04-15-53-23
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index 784d6d7920..63bf7c95fb 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#open	2012-11-20-15-05-07
-#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
-#types	time	interval	string	string	string	addr	subnet	time	count	double	double	double	double	double	double	count
-1353423907.236002	3.000000	foo-bar	test.metric	-	6.5.4.3	-	1353423907.236002	1	2.0	2.0	2.0	2.0	0.0	0.0	-
-1353423907.236002	3.000000	foo-bar	test.metric	-	1.2.3.4	-	1353423907.236002	5	221.0	5.0	94.0	44.2	915.36	30.254917	-
-1353423907.236002	3.000000	foo-bar	test.metric	-	7.2.1.5	-	1353423907.236002	1	1.0	1.0	1.0	1.0	0.0	0.0	-
-#close	2012-11-20-15-05-07
+#open	2012-12-04-15-55-13
+#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
+#types	time	interval	string	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
+1354636513.492214	3.000000	foo-bar	test.metric	-	6.5.4.3	-	1354636513.492214	1354636513.492214	1	2.0	2.0	2.0	2.0	0.0	0.0	-
+1354636513.492214	3.000000	foo-bar	test.metric	-	1.2.3.4	-	1354636513.492214	1354636513.492214	5	221.0	5.0	94.0	44.2	915.36	30.254917	-
+1354636513.492214	3.000000	foo-bar	test.metric	-	7.2.1.5	-	1354636513.492214	1354636513.492214	1	1.0	1.0	1.0	1.0	0.0	0.0	-
+#close	2012-12-04-15-55-13

From d0e8a6eef3a8da10480fbe18ebd3d1622de3e93b Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 4 Dec 2012 11:54:39 -0500
Subject: [PATCH 32/70] Comment updates and revised scan detection duration
 logging.

- Detection duration tracking is now logged in notices as 2m43s and
  only goes down to seconds.  Previously is was proceeding to milli-
  and micro seconds which aren't particularly useful.

- Inline docu-comment updates from Vlad Grigorescu.
---
 scripts/policy/misc/scan.bro | 43 +++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 18 deletions(-)

diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 42350bbe77..5a8e3f7830 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -11,17 +11,26 @@ module Scan;
 
 export {
 	redef enum Notice::Type += {
-		## Address scans detect that a host appears to be scanning
-		## some number of other hosts on a single port.
+		## Address scans detect that a host appears to be scanning some number
+		## of hosts on a single port. This notice is generated when more than 
+		## :bro:id:`addr_scan_threshold` unique hosts are seen over the 
+		## previous :bro:id:`addr_scan_interval` time range.
 		Address_Scan,
-		## Port scans detect that a host appears to be scanning a
-		## single other host on numerous ports.
+		## Port scans detect that an attacking host appears to be scanning a 
+		## single victim host on several ports.  This notice is generated when 
+		## an attacking host attempts to connect to :bro:id:`port_scan_threshold` 
+		## unique ports on a single host over the previous 
+		## :bro:id:`port_scan_interval` time range.
 		Port_Scan,
 		};
 
-	## Interval at which to watch for an address scan detection threshold to be crossed.
+	## Failed connection attempts are tracked over this time interval for the address 
+	## scan detection.  A higher interval will detect slower scanners, but may 
+	## also yield more false positives.
 	const addr_scan_interval = 5min &redef;
-	## Interval at which to watch for a port scan detection threshold to be crossed.
+	## Failed connection attempts are tracked over this time interval for the port 
+	## scan detection.  A higher interval will detect slower scanners, but may 
+	## also yield more false positives.
 	const port_scan_interval = 5min &redef;
 
 	## The threshold of a unique number of hosts a scanning host has to have failed 
@@ -48,10 +57,17 @@ function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVa
 	         val$sum > addr_scan_custom_thresholds[service] );
 	}
 
+function duration_to_mins_secs(dur: interval): string
+	{
+	local dur_count = double_to_count(interval_to_double(dur));
+	return fmt("%dm%ds", dur_count/60, dur_count%60);
+	}
+
 function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
-	local message=fmt("%s scanned %d unique hosts on port %s in %s", index$host, val$unique, index$str, val$end-val$begin);
+	local dur = duration_to_mins_secs(val$end-val$begin);
+	local message=fmt("%s scanned %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
 
 	NOTICE([$note=Address_Scan,
 	        $src=index$host,
@@ -64,7 +80,8 @@ function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
-	local message = fmt("%s scanned %d unique ports of host %s in %s", index$host, val$unique, index$str, val$end-val$begin);
+	local dur = duration_to_mins_secs(val$end-val$begin);
+	local message = fmt("%s scanned %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
 
 	NOTICE([$note=Port_Scan, 
 	        $src=index$host,
@@ -208,13 +225,3 @@ event connection_pending(c: connection)
 	else if ( is_reverse_failed_conn(c) )
 		add_metrics(c$id, T);
 	}
-
-## Generated when a SYN-ACK packet is seen in response to a SYN 
-## packet during a TCP handshake. The final ACK of the handshake 
-## in response to SYN-ACK may or may not occur later, one way to 
-## tell is to check the history field of connection to see if the 
-## originator sent an ACK, indicated by ‘A’ in the history string.
-#event connection_established(c: connection)
-#	{
-#	# Not useful for scan (too early)
-#	}

From 69b7ce12d2085b343768705b6225abc68e3fd8e7 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 18 Dec 2012 01:08:59 -0500
Subject: [PATCH 33/70] API updates for metrics framework.

- Removed default logging.  Now a function is available for the new
  $period_finished filter field to get the same behavior for logging
  named Metrics::write_log.

- Added index rollups for getting multiple metrics result values
  as the same time.
---
 scripts/base/frameworks/metrics/cluster.bro   |  45 +++-
 scripts/base/frameworks/metrics/main.bro      | 210 ++++++++++++------
 .../base/frameworks/metrics/non-cluster.bro   |  26 ++-
 .../frameworks/metrics/conn-example.bro       |   9 +-
 .../frameworks/metrics/http-example.bro       |  15 +-
 .../policy/frameworks/metrics/ssl-example.bro |   5 +-
 scripts/policy/misc/app-metrics.bro           |  55 ++++-
 scripts/policy/misc/capture-loss.bro          |   1 -
 scripts/policy/misc/scan.bro                  |  18 +-
 .../protocols/conn/conn-stats-per-host.bro    |   6 +-
 scripts/policy/protocols/conn/metrics.bro     |   6 +-
 .../manager-1.metrics.log                     |  14 +-
 .../metrics.log                               |  14 +-
 .../base/frameworks/metrics/basic-cluster.bro |  26 +--
 .../scripts/base/frameworks/metrics/basic.bro |   4 +-
 .../metrics/cluster-intermediate-update.bro   |   3 +-
 .../base/frameworks/metrics/thresholding.bro  |   9 +-
 17 files changed, 304 insertions(+), 162 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 6a536efb85..60342b327f 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -60,18 +60,18 @@ global requested_results: table[string] of time = table() &create_expire=5mins;
 
 # This variable is maintained by manager nodes as they collect and aggregate 
 # results.
-global filter_results: table[string, string, string] of MetricTable &create_expire=5mins;
+global filter_results: table[string, string, string] of MetricTable &read_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid 
 # matches the number of peer nodes that results should be coming from, the 
 # result is written out and deleted from here.
 # TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &create_expire=5mins &default=0;
+global done_with: table[string] of count &read_expire=1min &default=0;
 
 # This variable is maintained by managers to track intermediate responses as 
 # they are getting a global view for a certain index.
-global index_requests: table[string, string, string, Index] of ResultVal &create_expire=5mins &default=[];
+global index_requests: table[string, string, string, Index] of ResultVal &read_expire=1min;
 
 # This variable is maintained by all hosts for different purposes. Non-managers
 # maintain it to know what indexes they have recently sent as intermediate
@@ -163,7 +163,7 @@ event Metrics::cluster_index_request(uid: string, id: string, filter_name: strin
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 # Manager's handle logging.
-event Metrics::log_it(filter: Filter)
+event Metrics::finish_period(filter: Filter)
 	{
 	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
 	local uid = unique_id("");
@@ -174,8 +174,8 @@ event Metrics::log_it(filter: Filter)
 	
 	# Request data from peers.
 	event Metrics::cluster_filter_request(uid, filter$id, filter$name);
-	# Schedule the log_it event for the next break period.
-	schedule filter$every { Metrics::log_it(filter) };
+	# Schedule the next finish_period event.
+	schedule filter$every { Metrics::finish_period(filter) };
 	}
 
 # This is unlikely to be called often, but it's here in case there are metrics
@@ -237,6 +237,8 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 		++done_with[uid];
 
 	local local_data = filter_results[uid, id, filter_name];
+	local filter = filter_store[id, filter_name];
+
 	for ( index in data )
 		{
 		if ( index in local_data )
@@ -245,18 +247,18 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 			local_data[index] = data[index];
 
 		# If a filter is done being collected, thresholds for each index
-		# need to checked so we're doing it here to avoid doubly iterating 
+		# need to be checked so we're doing it here to avoid doubly iterating 
 		# over each index.
 		if ( Cluster::worker_count == done_with[uid] )
 			{
-			if ( check_thresholds(filter_store[id, filter_name], index, local_data[index], 1.0) )
+			if ( check_thresholds(filter, index, local_data[index], 1.0) )
 				{
-				threshold_crossed(filter_store[id, filter_name], index, local_data[index]);
+				threshold_crossed(filter, index, local_data[index]);
 				}
 			}
 		}
 	
-	# If the data has been collected from all peers, we are done and ready to log.
+	# If the data has been collected from all peers, we are done and ready to finish.
 	if ( Cluster::worker_count == done_with[uid] )
 		{
 		local ts = network_time();
@@ -267,11 +269,30 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 			delete requested_results[uid];
 			}
 		
-		write_log(ts, filter_store[id, filter_name], local_data);
-		
+		if ( filter?$rollup )
+			{
+			for ( index in local_data )
+				{
+				if ( index !in rollup_store )
+					rollup_store[index] = table();
+				rollup_store[index][id, filter_name] = local_data[index];
+
+				# If all of the result vals are stored then the rollup callback can be executed.
+				if ( |rollup_store[index]| == |rollups[filter$rollup]$filters| )
+					{
+					rollups[filter$rollup]$callback(index, rollup_store[index]);
+					}
+				}
+			}
+
+		if ( filter?$period_finished )
+			filter$period_finished(ts, filter$id, filter$name, local_data);
+
 		# Clean up
 		delete filter_results[uid, id, filter_name];
 		delete done_with[uid];
+		# Not sure I need to reset the filter on the manager.
+		reset(filter);
 		}
 	}
 
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 8d7ea26bc7..534529e020 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -8,10 +8,6 @@ export {
 	## The metrics logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	## The default interval used for "breaking" metrics and writing the 
-	## current value to the logging stream.
-	const default_break_interval = 15mins &redef;
-	
 	## This is the interval for how often threshold based notices will happen 
 	## after they have already fired.
 	const threshold_crossed_restart_interval = 1hr &redef;
@@ -108,63 +104,74 @@ export {
 	## The record type that is used for logging metrics.
 	type Info: record {
 		## Timestamp at which the metric was "broken".
-		ts:           time     &log;
+		ts:           time       &log;
 		## Interval between logging of this filter and the last time it was logged.
-		ts_delta:     interval &log;
-		## The name of the filter being logged.  Values
-		## can have multiple filters which represent different perspectives on
-		## the data so this is necessary to understand the value.
-		filter_name:  string   &log;
+		ts_delta:     interval   &log;
 		## What measurement the metric represents.
-		metric:       string   &log;
+		metric:       string     &log;
 		## What the metric value applies to.
-		index:        Index    &log;
+		index:        Index      &log;
 		## The simple numeric value of the metric.
-		result:        ResultVal    &log;
+		result:       ResultVal  &log;
 	};
 	
+	## Type to store a table of metrics result values.
+	type MetricTable: table[Index] of ResultVal;
+
 	## Filters define how the data from a metric is aggregated and handled.  
 	## Filters can be used to set how often the measurements are cut 
 	## and logged or how the data within them is aggregated.  It's also 
 	## possible to disable logging and use filters solely for thresholding.
 	type Filter: record {
-		## The name for this filter so that multiple filters can be
-		## applied to a single metrics to get a different view of the same
-		## metric data being collected (different aggregation, break, etc).
+		## A name for the filter in case multiple filters are being
+		## applied to the same metric.  In most cases the default 
+		## filter name is fine and this field does not need to be set.
 		name:              string                   &default="default";
-		## The metric that this filter applies to.
-		id:                string                   &optional;
-		## The measurements to perform on the data.
-		measure:           set[Calculation]         &optional;
-		## A predicate so that you can decide per index if you would like
-		## to accept the data being inserted.
-		pred:              function(index: Metrics::Index, data: Metrics::DataPoint): bool &optional;
-		## A function to normalize the index.  This can be used to aggregate or
-		## normalize the entire index.
-		normalize_func:    function(index: Metrics::Index): Index &optional;
-		## Global mask by to aggregate traffic measuring an attribute of hosts.
-		## This is a special case of the normalize_func.
-		aggregation_mask:  count                    &optional;
+
 		## The interval at which this filter should be "broken" and written
 		## to the logging stream.  The counters are also reset to zero at 
 		## this time so any threshold based detection needs to be set to a 
 		## number that should be expected to happen within this period.
-		every:             interval                 &default=default_break_interval;
-		## This determines if the result of this filter is sent to the metrics
-		## logging stream.  One use for the logging framework is as an internal
-		## thresholding and statistics gathering utility that is meant to
-		## never log but rather to generate notices and derive data.
-		log:               bool                     &default=T;
+		every:             interval;
+		
+		## The measurements to perform on the data.
+		measure:           set[Calculation]         &optional;
+		
+		## A predicate so that you can decide per index if you would like
+		## to accept the data being inserted.
+		pred:              function(index: Metrics::Index, data: Metrics::DataPoint): bool &optional;
+		
+		## A function to normalize the index.  This can be used to aggregate or
+		## normalize the entire index.
+		normalize_func:    function(index: Metrics::Index): Index &optional;
+		
+		## Global mask by to aggregate traffic measuring an attribute of hosts.
+		## This is a special case of the normalize_func.
+		aggregation_mask:  count                    &optional;
+		
 		## A direct threshold for calling the $threshold_crossed function when 
 		## the SUM is greater than or equal to this value.
 		threshold:         count                    &optional;
+		
 		## A series of thresholds for calling the $threshold_crossed function.
 		threshold_series:  vector of count          &optional;
+		
 		## A predicate so that you can decide when to flexibly declare when 
 		## a threshold crossed, and do extra work.
 		threshold_func:    function(index: Metrics::Index, val: Metrics::ResultVal): bool &optional;
-		## A function callback that is called when a threshold is crossed.
+		
+		## A callback with the full collection of ResultVals for this filter.  This 
+		## is defined as a redef because the function includes a :bro:type:`Filter`
+		## record which is self referential before the Filter type has been fully 
+		## defined and doesn't work.
+		period_finished:   function(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable) &optional;
+
+		## A callback that is called when a threshold is crossed.
 		threshold_crossed: function(index: Metrics::Index, val: Metrics::ResultVal) &optional;
+
+		## A rollup to register this filter with.
+		rollup:            string &optional;
+
 		## A number of sample DataPoint strings to collect for the threshold 
 		## crossing callback.
 		samples:           count                    &optional;
@@ -187,7 +194,19 @@ export {
 	##
 	## increment: How much to increment the counter by.
 	global add_data: function(id: string, index: Metrics::Index, data: Metrics::DataPoint);
-	
+
+	## The callback definition for rollup functions.
+	type RollupCallback: function(index: Metrics::Index, vals: table[string, string] of Metrics::ResultVal);
+
+	## Add a rollup function for merging multiple filters with matching 
+	## indexes.  If the metrics filters being merged don't have equivalent times
+	## in the $every field, an error will be generated.
+	##
+	## name: An arbitrary name for this filter rollup.
+	##
+	## vals: Each ResultVal record indexed by the appropriate metric name and filter name.
+	global create_index_rollup: function(name: string, rollup: RollupCallback);
+
 	## Helper function to represent a :bro:type:`Metrics::Index` value as 
 	## a simple string.
 	## 
@@ -195,12 +214,23 @@ export {
 	##
 	## Returns: A string reprentation of the metric index.
 	global index2str: function(index: Metrics::Index): string;
-		
+	
+	## A helper function to use with the `period_finished` field in filters.  Using 
+	## this function is not recommended however since each metric likely has 
+	## different data and different semantics which would be better served by writing
+	## a custom function that logs in more domain specific fashion.
+	global write_log: function(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable);
+
 	## Event to access metrics records as they are passed to the logging framework.
 	global log_metrics: event(rec: Metrics::Info);
 	
 }
 
+redef record Filter += {
+	# The metric that this filter applies to.  The value is automatically set.
+	id: string &optional;
+};
+
 redef record ResultVal += {
 	# Internal use only.  Used for incrementally calculating variance.
 	prev_avg:      double      &optional;
@@ -226,9 +256,6 @@ redef record ResultVal += {
 	threshold_series_index: count &default=0;
 };
 
-# Type to store a table of metrics values.
-type MetricTable: table[Index] of ResultVal;
-
 # Store the filters indexed on the metric identifier.
 global metric_filters: table[string] of vector of Filter = table();
 
@@ -238,16 +265,23 @@ global filter_store: table[string, string] of Filter = table();
 # This is indexed by metric id and filter name.
 global store: table[string, string] of MetricTable = table() &default=table();
 
-# This is hook for watching thresholds being crossed.  It is called whenever
+# This is a hook for watching thresholds being crossed.  It is called whenever
 # index values are updated and the new val is given as the `val` argument.
-# It's only prototyped here because cluster and non-cluster has separate 
+# It's only prototyped here because cluster and non-cluster have separate 
 # implementations.
 global data_added: function(filter: Filter, index: Index, val: ResultVal);
 
+type Rollup: record {
+	callback: RollupCallback;
+	filters:  set[Filter] &optional;
+};
+global rollups: table[string] of Rollup;
+global rollup_store: table[Index] of table[string, string] of ResultVal = {};
+
+
 ## Event that is used to "finish" metrics and adapt the metrics
 ## framework for clustered or non-clustered usage.
-global log_it: event(filter: Metrics::Filter);
-
+global finish_period: event(filter: Metrics::Filter);
 
 event bro_init() &priority=5
 	{
@@ -279,22 +313,21 @@ function do_calculated_fields(val: ResultVal)
 function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	{
 	local result: ResultVal;
-	
+
 	# Merge $begin (take the earliest one)
-	result$begin = rv1$begin < rv2$begin ? rv1$begin : rv2$begin;
+	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
 
 	# Merge $end (take the latest one)
-	result$end = rv1$end > rv2$end ? rv1$end : rv2$end;
+	result$end = (rv1$end > rv2$end) ? rv1$end : rv2$end;
 
 	# Merge $num
 	result$num = rv1$num + rv2$num;
 
 	# Merge $sum
+	result$sum = rv1$sum + rv2$sum;
 	if ( rv1?$sum || rv2?$sum )
 		{
-		result$sum = 0;
-		if ( rv1?$sum )
-			result$sum += rv1$sum;
+		result$sum = rv1?$sum ? rv1$sum : 0;
 		if ( rv2?$sum )
 			result$sum += rv2$sum;
 		}
@@ -348,13 +381,15 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	# Merge $unique_vals
 	if ( rv1?$unique_vals || rv2?$unique_vals )
 		{
-		result$unique_vals = set();
 		if ( rv1?$unique_vals )
-			for ( val1 in rv1$unique_vals )
-				add result$unique_vals[val1];
+			result$unique_vals = rv1$unique_vals;
+		
 		if ( rv2?$unique_vals )
-			for ( val2 in rv2$unique_vals )
-				add result$unique_vals[val2];
+			if ( ! result?$unique_vals )
+				result$unique_vals = rv2$unique_vals;
+			else
+				for ( val2 in rv2$unique_vals )
+					add result$unique_vals[val2];
 		}
 
 	# Merge $sample_queue
@@ -376,8 +411,9 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	return result;
 	}
 	
-function write_log(ts: time, filter: Filter, data: MetricTable)
+function write_log(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable)
 	{
+	local filter = filter_store[metric_name, filter_name];
 	for ( index in data )
 		{
 		local m: Info = [$ts=ts,
@@ -386,9 +422,7 @@ function write_log(ts: time, filter: Filter, data: MetricTable)
 		                 $filter_name=filter$name,
 		                 $index=index,
 		                 $result=data[index]];
-		
-		if ( filter$log )
-			Log::write(Metrics::LOG, m);
+		Log::write(LOG, m);
 		}
 	}
 
@@ -401,7 +435,7 @@ function add_filter(id: string, filter: Filter)
 	{
 	if ( filter?$normalize_func && filter?$aggregation_mask )
 		{
-		Reporter::warning(fmt("invalid Metric filter (%s): Defined $normalize_func and $aggregation_mask.", filter$name));
+		Reporter::warning(fmt("invalid Metric filter (%s): Defined both $normalize_func and $aggregation_mask.", filter$name));
 		return;
 		}
 	if ( [id, filter$name] in store )
@@ -409,7 +443,33 @@ function add_filter(id: string, filter: Filter)
 		Reporter::warning(fmt("invalid Metric filter (%s): Filter with same name already exists.", filter$name));
 		return;
 		}
-	
+	if ( filter?$rollup )
+		{
+		if ( filter$rollup !in rollups )
+			{
+			Reporter::warning(fmt("invalid Metric filter (%s): %s rollup doesn't exist.", filter$name, filter$rollup));
+			return;
+			}
+		else
+			{
+			local every_field = 0secs;
+			for ( filt in rollups )
+				{
+				if ( [id, filt] !in filter_store )
+					next;
+				
+				if ( every_field == 0secs )
+					every_field = filter_store[id, filt]$every;
+				else if ( every_field == filter_store[id, filt]$every )
+					{
+					Reporter::warning(fmt("invalid Metric rollup for %s: Filters with differing $every fields applied to %s.", filter$name, filter$rollup));
+					return;
+					}
+				}
+			}
+			add rollups[filter$rollup]$filters[filter];
+		}
+
 	if ( ! filter?$id )
 		filter$id = id;
 	
@@ -419,8 +479,8 @@ function add_filter(id: string, filter: Filter)
 
 	filter_store[id, filter$name] = filter;
 	store[id, filter$name] = table();
-	
-	schedule filter$every { Metrics::log_it(filter) };
+
+	schedule filter$every { Metrics::finish_period(filter) };
 	}
 
 function add_data(id: string, index: Index, data: DataPoint)
@@ -513,11 +573,11 @@ function add_data(id: string, index: Index, data: DataPoint)
 			result$var_s += (val - result$prev_avg)*(val - result$avg);
 			}
 
-		if ( STD_DEV in filter$measure )
-			{
-			#if ( result?$variance )
-			#	result$std_dev = sqrt(result$variance);
-			}
+		#if ( STD_DEV in filter$measure )
+		#	{
+		#	#if ( result?$variance )
+		#	#	result$std_dev = sqrt(result$variance);
+		#	}
 
 		if ( UNIQUE in filter$measure )
 			{
@@ -530,8 +590,7 @@ function add_data(id: string, index: Index, data: DataPoint)
 		}
 	}
 
-# This function checks if a threshold has been crossed and generates a 
-# notice if it has.  It is also used as a method to implement 
+# This function checks if a threshold has been crossed.  It is also used as a method to implement 
 # mid-break-interval threshold crossing detection for cluster deployments.
 function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_pct: double): bool
 	{
@@ -570,7 +629,7 @@ function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_p
 
 	return F;
 	}
-		
+
 function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
 	{
 	if ( ! filter?$threshold_crossed )
@@ -586,3 +645,10 @@ function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
 	if ( filter?$threshold_series )
 		++val$threshold_series_index;
 	}
+
+function create_index_rollup(name: string, rollup: RollupCallback)
+	{
+	local r: Rollup = [$callback=rollup];
+	r$filters=set();
+	rollups[name] = r;
+	}
diff --git a/scripts/base/frameworks/metrics/non-cluster.bro b/scripts/base/frameworks/metrics/non-cluster.bro
index a94370d776..b76ca3ea48 100644
--- a/scripts/base/frameworks/metrics/non-cluster.bro
+++ b/scripts/base/frameworks/metrics/non-cluster.bro
@@ -2,15 +2,31 @@
 
 module Metrics;
 
-event Metrics::log_it(filter: Filter)
+event Metrics::finish_period(filter: Filter)
 	{
-	local id = filter$id;
-	local name = filter$name;
+	local data = store[filter$id, filter$name];
+	if ( filter?$rollup )
+		{
+		for ( index in data )
+			{
+			if ( index !in rollup_store )
+				rollup_store[index] = table();
+			rollup_store[index][filter$id, filter$name] = data[index];
+
+			# If all of the result vals are stored then the rollup callback can be executed.
+			if ( |rollup_store[index]| == |rollups[filter$rollup]$filters| )
+				{
+				rollups[filter$rollup]$callback(index, rollup_store[index]);
+				}
+			}
+		}
+
+	if ( filter?$period_finished )
+		filter$period_finished(network_time(), filter$id, filter$name, data);
 
-	write_log(network_time(), filter, store[id, name]);
 	reset(filter);
 	
-	schedule filter$every { Metrics::log_it(filter) };
+	schedule filter$every { Metrics::finish_period(filter) };
 	}
 	
 	
diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro
index e5c604a5b2..1271d6eb32 100644
--- a/scripts/policy/frameworks/metrics/conn-example.bro
+++ b/scripts/policy/frameworks/metrics/conn-example.bro
@@ -7,11 +7,16 @@
 event bro_init()
 	{
 	#Metrics::add_filter("conns.originated", [$aggregation_mask=24, $break_interval=1mins]);
-	Metrics::add_filter("conns.originated",  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
+	Metrics::add_filter("conns.originated", [$every=1mins, $measure=set(Metrics::SUM), 
+	                                         $aggregation_table=Site::local_nets_table, 
+	                                         $period_finished=Metrics::write_log]);
 	
 	
 	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter("conns.responded",  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
+	Metrics::add_filter("conns.responded",  [$every=1mins, $measure=set(Metrics::SUM),
+	                                         $aggregation_table=Site::local_nets_table, 
+	                                         $period_finished=Metrics::write_log]);
+
 	}
 
 event connection_established(c: connection)
diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro
index 3c60f3c931..b3284580e8 100644
--- a/scripts/policy/frameworks/metrics/http-example.bro
+++ b/scripts/policy/frameworks/metrics/http-example.bro
@@ -8,15 +8,16 @@
 
 event bro_init()
 	{
-	# TODO: these are waiting on a fix with table vals + records before they will work.
-	#Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER, 
-	#                    [$pred(index: Metrics::Index) = { return Site::is_local_addr(index$host); },
-	#                     $aggregation_mask=24,
-	#                     $break_interval=1min]);
+	Metrics::add_filter("http.request.by_host_header", 
+	                    [$every=1min, $measure=set(Metrics::SUM), 
+	                     $pred(index: Metrics::Index, data: Metrics::DataPoint) = { return T; return Site::is_local_addr(index$host); },
+	                     $aggregation_mask=24,
+	                     $period_finished=Metrics::write_log]);
 	
 	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter("http.request.by_status_code", [$aggregation_table=Site::local_nets_table,
-	                                                    $break_interval=1min]);
+	Metrics::add_filter("http.request.by_status_code", [$every=1min, $measure=set(Metrics::SUM),
+	                                                    $aggregation_table=Site::local_nets_table,
+	                                                    $period_finished=Metrics::write_log]);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro
index 64e63bc215..3b9b848edb 100644
--- a/scripts/policy/frameworks/metrics/ssl-example.bro
+++ b/scripts/policy/frameworks/metrics/ssl-example.bro
@@ -10,11 +10,10 @@ event bro_init()
 	{
 	Metrics::add_filter("ssl.by_servername", 
 		[$name="no-google-ssl-servers",
+		 $every=10secs, $measure=set(Metrics::SUM),
 		 $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
 		    return (/google\.com$/ !in index$str); 
-		 },
-		 $break_interval=10secs
-		]);
+		 }]);
 	}
 
 event SSL::log_ssl(rec: SSL::Info)
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index a89d0d8eb3..d88eb8fe6e 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -1,49 +1,80 @@
 @load base/protocols/http
 @load base/protocols/ssl
-
 @load base/frameworks/metrics
 
 module AppMetrics;
 
 export {
-	## The metric break interval for the default stats collected by this script.
-	const break_interval = 1hr &redef;
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		ts:         time   &log;
+		app:        string &log;
+		uniq_hosts: count  &log;
+		hits:       count  &log;
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 1min &redef;
 }
 
+function app_metrics_rollup(index: Metrics::Index, vals: table[string, string] of Metrics::ResultVal) 
+	{
+	local l: Info;
+	l$ts = network_time();
+	for ( [metric_name, filter_name] in vals )
+		{
+		local val = vals[metric_name, filter_name];
+		l$app = index$str;
+		if ( metric_name == "apps.bytes" )
+			l$bytes = double_to_count(floor(val$sum));
+		else if ( metric_name == "apps.hits" )
+			{
+			l$hits = val$num;
+			l$uniq_hosts = val$unique;
+			}
+		}
+	Log::write(LOG, l);
+	}
+
 event bro_init() &priority=3
 	{
-	Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM)]);
-	Metrics::add_filter("apps.hits",  [$every=break_interval, $measure=set(Metrics::SUM, Metrics::UNIQUE)]);
+	Log::create_stream(AppMetrics::LOG, [$columns=Info]);
+
+	Metrics::create_index_rollup("AppMetrics", app_metrics_rollup);
+	Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM),    $period_finished=Metrics::write_log, $rollup="AppMetrics"]);
+	Metrics::add_filter("apps.hits",  [$every=break_interval, $measure=set(Metrics::UNIQUE), $rollup="AppMetrics"]);
 	}
 
 function do_metric(id: conn_id, hostname: string, size: count)
 	{
-	if ( /youtube/ in hostname && size > 512*1024 ) 
+	if ( /youtube\.com$/ in hostname && size > 512*1024 )
 		{
 		Metrics::add_data("apps.bytes", [$str="youtube"], [$num=size]);
 		Metrics::add_data("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /facebook.com|fbcdn.net/ in hostname && size > 20 )
+	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
 		{
 		Metrics::add_data("apps.bytes", [$str="facebook"], [$num=size]);
 		Metrics::add_data("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /google.com/ in hostname && size > 20 ) 
+	else if ( /\.google\.com$/ in hostname && size > 20 ) 
 		{
 		Metrics::add_data("apps.bytes", [$str="google"], [$num=size]);
 		Metrics::add_data("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /nflximg.com/ in hostname && size > 200*1024 ) 
+	else if ( /nflximg\.com$/ in hostname && size > 200*1024 ) 
 		{
 		Metrics::add_data("apps.bytes", [$str="netflix"], [$num=size]);
 		Metrics::add_data("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /pandora.com/ in hostname && size > 512*1024 )
+	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
 		{
 		Metrics::add_data("apps.bytes", [$str="pandora"], [$num=size]);
 		Metrics::add_data("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /gmail.com/ in hostname && size > 20 )
+	else if ( /gmail\.com$/ in hostname && size > 20 )
 		{
 		Metrics::add_data("apps.bytes", [$str="gmail"], [$num=size]);
 		Metrics::add_data("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
@@ -63,7 +94,7 @@ event ssl_established(c: connection)
 event connection_finished(c: connection)
 	{
 	if ( c?$resp_hostname )
-		do_metric(c$id, c$resp_hostname, c$resp$num_bytes_ip);
+		do_metric(c$id, c$resp_hostname, c$resp$size);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro
index b2d23020f8..1f0726299d 100644
--- a/scripts/policy/misc/capture-loss.bro
+++ b/scripts/policy/misc/capture-loss.bro
@@ -8,7 +8,6 @@
 ##! for a sequence number that's above a gap).
 
 @load base/frameworks/notice
-@load base/frameworks/metrics
 
 module CaptureLoss;
 
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 5a8e3f7830..a0228a7955 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -43,6 +43,10 @@ export {
 	## Custom threholds based on service for address scan.  This is primarily 
 	## useful for setting reduced thresholds for specific ports.
 	const addr_scan_custom_thresholds: table[port] of count &redef;
+
+	global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
+	
+	global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
 }
 
 
@@ -94,16 +98,14 @@ function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 event bro_init() &priority=5
 	{
 	# Note: addr scans are trcked similar to:  table[src_ip, port] of set(dst);	
-	Metrics::add_filter("scan.addr.fail", [$log=F,
-	                                       $every=addr_scan_interval,
+	Metrics::add_filter("scan.addr.fail", [$every=addr_scan_interval,
 	                                       $measure=set(Metrics::UNIQUE),
 	                                       $threshold_func=check_addr_scan_threshold,
 	                                       $threshold=addr_scan_threshold,
 	                                       $threshold_crossed=addr_scan_threshold_crossed]); 
 
 	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
-	Metrics::add_filter("scan.port.fail", [$log=F,
-	                                       $every=port_scan_interval,
+	Metrics::add_filter("scan.port.fail", [$every=port_scan_interval,
 	                                       $measure=set(Metrics::UNIQUE),
 	                                       $threshold=port_scan_threshold,
 	                                       $threshold_crossed=port_scan_threshold_crossed]); 
@@ -146,11 +148,11 @@ function add_metrics(id: conn_id, reverse: bool)
 	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
 	#	return F;
 
-	# Probably do a hook point here?
-	Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
+	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
+		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
 
-	# Probably do a hook point here?
-	Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
+	if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
+		Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 
 function is_failed_conn(c: connection): bool
diff --git a/scripts/policy/protocols/conn/conn-stats-per-host.bro b/scripts/policy/protocols/conn/conn-stats-per-host.bro
index df58081163..fad2331f44 100644
--- a/scripts/policy/protocols/conn/conn-stats-per-host.bro
+++ b/scripts/policy/protocols/conn/conn-stats-per-host.bro
@@ -6,10 +6,12 @@ event bro_init() &priority=5
 	{
 	Metrics::add_filter("conn.orig.data", 
 	                    [$every=5mins,
-	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
+	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
+	                     $period_finished=Metrics::write_log]);
 	Metrics::add_filter("conn.resp.data", 
 	                    [$every=5mins,
-	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
+	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
+	                     $period_finished=Metrics::write_log]);
 	}
 
 
diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro
index 0fb5fa2134..057e23e088 100644
--- a/scripts/policy/protocols/conn/metrics.bro
+++ b/scripts/policy/protocols/conn/metrics.bro
@@ -3,8 +3,10 @@
 
 event bro_init() &priority=3
 	{
-	Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM)]);
-	Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM)]);
+	Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM),
+	                                      $period_finished=Metrics::write_log]);
+	Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM),
+	                                     $period_finished=Metrics::write_log]);
 	}
 
 event connection_established(c: connection) &priority=3
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index 98794673f1..bdc86c68bb 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#open	2012-12-04-15-53-23
-#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
-#types	time	interval	string	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
-1354636403.682565	3.000000	default	test.metric	-	6.5.4.3	-	1354636401.774655	1354636401.782720	2	6.0	1.0	5.0	3.0	4.0	2.0	-
-1354636403.682565	3.000000	default	test.metric	-	1.2.3.4	-	1354636401.774655	1354636401.782720	9	437.0	3.0	95.0	48.555556	674.469136	25.970544	-
-1354636403.682565	3.000000	default	test.metric	-	7.2.1.5	-	1354636401.774655	1354636401.782720	2	145.0	54.0	91.0	72.5	342.25	18.5	-
-#close	2012-12-04-15-53-23
+#open	2012-12-17-18-43-15
+#fields	ts	ts_delta	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
+#types	time	interval	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
+1355769795.365325	3.000000	test.metric	-	6.5.4.3	-	1355769793.449322	1355769793.458467	2	6.0	1.0	5.0	3.0	4.0	2.0	2
+1355769795.365325	3.000000	test.metric	-	1.2.3.4	-	1355769793.449322	1355769793.458467	9	437.0	3.0	95.0	48.555556	674.469136	25.970544	8
+1355769795.365325	3.000000	test.metric	-	7.2.1.5	-	1355769793.449322	1355769793.458467	2	145.0	54.0	91.0	72.5	342.25	18.5	2
+#close	2012-12-17-18-43-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index 63bf7c95fb..51d892e8d5 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#open	2012-12-04-15-55-13
-#fields	ts	ts_delta	filter_name	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
-#types	time	interval	string	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
-1354636513.492214	3.000000	foo-bar	test.metric	-	6.5.4.3	-	1354636513.492214	1354636513.492214	1	2.0	2.0	2.0	2.0	0.0	0.0	-
-1354636513.492214	3.000000	foo-bar	test.metric	-	1.2.3.4	-	1354636513.492214	1354636513.492214	5	221.0	5.0	94.0	44.2	915.36	30.254917	-
-1354636513.492214	3.000000	foo-bar	test.metric	-	7.2.1.5	-	1354636513.492214	1354636513.492214	1	1.0	1.0	1.0	1.0	0.0	0.0	-
-#close	2012-12-04-15-55-13
+#open	2012-12-17-18-43-45
+#fields	ts	ts_delta	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
+#types	time	interval	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
+1355769825.947161	3.000000	test.metric	-	6.5.4.3	-	1355769825.947161	1355769825.947161	1	2.0	2.0	2.0	2.0	0.0	0.0	-
+1355769825.947161	3.000000	test.metric	-	1.2.3.4	-	1355769825.947161	1355769825.947161	5	221.0	5.0	94.0	44.2	915.36	30.254917	-
+1355769825.947161	3.000000	test.metric	-	7.2.1.5	-	1355769825.947161	1355769825.947161	1	1.0	1.0	1.0	1.0	0.0	0.0	-
+#close	2012-12-17-18-43-45
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index 41ef9b57dc..c68a4f7beb 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -19,11 +19,23 @@ redef Cluster::nodes = {
 
 redef Log::default_rotation_interval = 0secs;
 
+global n = 0;
+
 event bro_init() &priority=5
 	{
 	Metrics::add_filter("test.metric", 
 	                    [$every=3secs,
-	                     $measure=set(Metrics::SUM, Metrics::MIN, Metrics::MAX, Metrics::AVG, Metrics::STD_DEV, Metrics::VARIANCE)]);
+	                     $measure=set(Metrics::SUM, Metrics::MIN, Metrics::MAX, Metrics::AVG, Metrics::STD_DEV, Metrics::VARIANCE, Metrics::UNIQUE),
+	                     $period_finished(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable) =
+	                     	{
+	                     	Metrics::write_log(ts, metric_name, filter_name, data);
+	                     	if ( ++n == 3 )
+	                     		{
+	                     		terminate_communication();
+	                     		terminate();
+	                     		}
+	                     	}
+	                     	]);
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -64,22 +76,10 @@ event ready_for_data()
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
-global n = 0;
 global peer_count = 0;
 
-event Metrics::log_metrics(rec: Metrics::Info)
-	{
-	++n;
-	if ( n == 3 )
-		{
-		terminate_communication();
-		terminate();
-		}
-	}
-
 event remote_connection_handshake_done(p: event_peer)
 	{
-	print p;
 	++peer_count;
 	if ( peer_count == 3 )
 		event ready_for_data();
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic.bro b/testing/btest/scripts/base/frameworks/metrics/basic.bro
index 12163ed689..e665f2ea5c 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic.bro
@@ -6,7 +6,9 @@ event bro_init() &priority=5
 	Metrics::add_filter("test.metric", 
 		[$name="foo-bar",
 		 $every=3secs,
-		 $measure=set(Metrics::SUM, Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
+		 $measure=set(Metrics::SUM, Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
+		 $period_finished=Metrics::write_log]);
+
 	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=5]);
 	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=22]);
 	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=94]);
diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
index 3341fa1887..b16645dbe6 100644
--- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
@@ -26,8 +26,7 @@ event bro_init() &priority=5
 	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
 	                     	print "A test metric threshold was crossed!";
 	                     	terminate();
-	                     }
-	                     ]);
+	                     }]);
 	}
 
 event remote_connection_closed(p: event_peer)
diff --git a/testing/btest/scripts/base/frameworks/metrics/thresholding.bro b/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
index bd0cd6faae..f39443fc2a 100644
--- a/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
@@ -15,8 +15,7 @@ event bro_init() &priority=5
 	                     $threshold=5,
 	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
 	                     	print fmt("THRESHOLD: hit a threshold value at %.0f for %s", val$sum, Metrics::index2str(index));
-	                     },
-	                     $log=F]);
+	                     }]);
 
 	Metrics::add_filter("test.metric", 
 	                    [$name="foobar2",
@@ -25,8 +24,7 @@ event bro_init() &priority=5
 	                     $threshold_series=vector(3,6,800),
 	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
 	                     	print fmt("THRESHOLD_SERIES: hit a threshold series value at %.0f for %s", val$sum, Metrics::index2str(index));
-		                 },
-	                     $log=F]);
+		                 }]);
 	Metrics::add_filter("test.metric", 
 	                    [$every=3secs,
 	                     $measure=set(Metrics::SUM),
@@ -36,8 +34,7 @@ event bro_init() &priority=5
 	                     },
 	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
 	                     	print fmt("THRESHOLD_FUNC: hit a threshold function value at %.0f for %s", val$sum, Metrics::index2str(index));
-	                     },
-	                     $log=F]);
+	                     }]);
 
 	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
 	Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=2]);

From 7edef1f2c4d2e36008c3caa2f09d9ee3aa9a17ca Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 18 Dec 2012 01:31:52 -0500
Subject: [PATCH 34/70] Disable the hook execution in the scan.bro script. 
 It's not working like I expected.

---
 scripts/policy/misc/scan.bro | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index a0228a7955..1def14d07e 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -148,10 +148,12 @@ function add_metrics(id: conn_id, reverse: bool)
 	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
 	#	return F;
 
-	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
+	# Hooks don't seem to be working like I expected.  They'll have to wait a bit longer.
+	
+	#if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
 		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
 
-	if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
+	#if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
 		Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 

From f31de38c9b2a73dfb1200761626c4585eb15c688 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 18 Dec 2012 01:44:46 -0500
Subject: [PATCH 35/70] Bug fix.

---
 scripts/base/frameworks/metrics/main.bro | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 534529e020..4803a521c7 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -324,7 +324,6 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	result$num = rv1$num + rv2$num;
 
 	# Merge $sum
-	result$sum = rv1$sum + rv2$sum;
 	if ( rv1?$sum || rv2?$sum )
 		{
 		result$sum = rv1?$sum ? rv1$sum : 0;

From ed36f376439a3f0e29b4e3ce0aa08c42293a4a91 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 18 Dec 2012 02:23:56 -0500
Subject: [PATCH 36/70] A few more small fixes.

---
 scripts/policy/misc/app-metrics.bro            | 2 +-
 scripts/policy/misc/detect-traceroute/main.bro | 2 +-
 scripts/policy/protocols/http/detect-sqli.bro  | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index d88eb8fe6e..f8e4ae2491 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -16,7 +16,7 @@ export {
 	};
 
 	## The frequency of logging the stats collected by this script.
-	const break_interval = 1min &redef;
+	const break_interval = 15mins &redef;
 }
 
 function app_metrics_rollup(index: Metrics::Index, vals: table[string, string] of Metrics::ResultVal) 
diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index 051d81c5c7..e62d370e45 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -34,7 +34,7 @@ export {
 	## Interval at which to watch for the
 	## :bro:id:`ICMPTimeExceeded::icmp_time_exceeded_threshold` variable to be crossed.
 	## At the end of each interval the counter is reset.
-	const icmp_time_exceeded_interval = 1min &redef;
+	const icmp_time_exceeded_interval = 3min &redef;
 
 	## The log record for the traceroute log.
 	type Info: record {
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 06f14219d1..21164bc126 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -74,7 +74,7 @@ event bro_init() &priority=3
 	                     	        $email_body_sections=vector(format_sqli_samples(val$samples)),
 	                     	        $src=index$host,
 	                     	        $identifier=cat(index$host)]);
-	                     }, $log=F]);
+	                     }]);
 
 	Metrics::add_filter("http.sqli.victim",
 	                    [$every=sqli_requests_interval,
@@ -87,7 +87,7 @@ event bro_init() &priority=3
 	                     	        $email_body_sections=vector(format_sqli_samples(val$samples)),
 	                     	        $src=index$host,
 	                     	        $identifier=cat(index$host)]);
-	                     }, $log=F]);
+	                     }]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,

From 6e9e3a5e8860b6bf36ceb19a4b06e4a3447169e3 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 18 Dec 2012 10:31:38 -0500
Subject: [PATCH 37/70] Small fixes.

 - Slight wording change in scan notices.

 - AppMetrics no longer writes to metrics.log.
---
 scripts/policy/misc/app-metrics.bro | 2 +-
 scripts/policy/misc/scan.bro        | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index f8e4ae2491..0a4fc8b39f 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -43,7 +43,7 @@ event bro_init() &priority=3
 	Log::create_stream(AppMetrics::LOG, [$columns=Info]);
 
 	Metrics::create_index_rollup("AppMetrics", app_metrics_rollup);
-	Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM),    $period_finished=Metrics::write_log, $rollup="AppMetrics"]);
+	Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM),    $rollup="AppMetrics"]);
 	Metrics::add_filter("apps.hits",  [$every=break_interval, $measure=set(Metrics::UNIQUE), $rollup="AppMetrics"]);
 	}
 
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 1def14d07e..1c13f8224b 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -71,7 +71,7 @@ function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
 	local dur = duration_to_mins_secs(val$end-val$begin);
-	local message=fmt("%s scanned %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
+	local message=fmt("%s scanned at least %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
 
 	NOTICE([$note=Address_Scan,
 	        $src=index$host,
@@ -85,7 +85,7 @@ function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
 	local dur = duration_to_mins_secs(val$end-val$begin);
-	local message = fmt("%s scanned %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
+	local message = fmt("%s scanned at least %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
 
 	NOTICE([$note=Port_Scan, 
 	        $src=index$host,

From 9c00ef3ccd7cba2721b888e229800a23eb82faa0 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 18 Dec 2012 12:22:28 -0500
Subject: [PATCH 38/70] Doing some code reorganization and small changes to
 hopefully fix a memory issue.

---
 scripts/base/frameworks/metrics/cluster.bro | 45 ++++++++++-----------
 scripts/base/frameworks/metrics/main.bro    |  7 +++-
 2 files changed, 27 insertions(+), 25 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 60342b327f..59abd1a606 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -49,29 +49,6 @@ export {
 	global send_data: event(uid: string, id: string, filter_name: string, data: MetricTable);
 }
 
-# This is maintained by managers so they can know what data they requested and
-# when they requested it.
-global requested_results: table[string] of time = table() &create_expire=5mins;
-
-# TODO: The next 4 variables make the assumption that a value never 
-#       takes longer than 5 minutes to transmit from workers to manager.  This needs to 
-#       be tunable or self-tuning.  These should also be restructured to be
-#       maintained within a single variable.
-
-# This variable is maintained by manager nodes as they collect and aggregate 
-# results.
-global filter_results: table[string, string, string] of MetricTable &read_expire=1min;
-
-# This variable is maintained by manager nodes to track how many "dones" they
-# collected per collection unique id.  Once the number of results for a uid 
-# matches the number of peer nodes that results should be coming from, the 
-# result is written out and deleted from here.
-# TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &read_expire=1min &default=0;
-
-# This variable is maintained by managers to track intermediate responses as 
-# they are getting a global view for a certain index.
-global index_requests: table[string, string, string, Index] of ResultVal &read_expire=1min;
 
 # This variable is maintained by all hosts for different purposes. Non-managers
 # maintain it to know what indexes they have recently sent as intermediate
@@ -162,6 +139,26 @@ event Metrics::cluster_index_request(uid: string, id: string, filter_name: strin
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
+# This variable is maintained by manager nodes as they collect and aggregate 
+# results.
+global filter_results: table[string, string, string] of MetricTable &read_expire=1min;
+
+# This is maintained by managers so they can know what data they requested and
+# when they requested it.
+global requested_results: table[string] of time = table() &create_expire=5mins;
+
+# This variable is maintained by manager nodes to track how many "dones" they
+# collected per collection unique id.  Once the number of results for a uid 
+# matches the number of peer nodes that results should be coming from, the 
+# result is written out and deleted from here.
+# TODO: add an &expire_func in case not all results are received.
+global done_with: table[string] of count &read_expire=1min &default=0;
+
+# This variable is maintained by managers to track intermediate responses as 
+# they are getting a global view for a certain index.
+global index_requests: table[string, string, string, Index] of ResultVal &read_expire=1min;
+
+
 # Manager's handle logging.
 event Metrics::finish_period(filter: Filter)
 	{
@@ -170,6 +167,8 @@ event Metrics::finish_period(filter: Filter)
 	
 	# Set some tracking variables.
 	requested_results[uid] = network_time();
+	if ( [uid, filter$id, filter$name] in filter_results )
+		delete filter_results[uid, filter$id, filter$name];
 	filter_results[uid, filter$id, filter$name] = table();
 	
 	# Request data from peers.
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 4803a521c7..0cfe96dfc6 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -227,7 +227,7 @@ export {
 }
 
 redef record Filter += {
-	# The metric that this filter applies to.  The value is automatically set.
+	# Internal use only.  The metric that this filter applies to.  The value is automatically set.
 	id: string &optional;
 };
 
@@ -263,7 +263,7 @@ global metric_filters: table[string] of vector of Filter = table();
 global filter_store: table[string, string] of Filter = table();
 
 # This is indexed by metric id and filter name.
-global store: table[string, string] of MetricTable = table() &default=table();
+global store: table[string, string] of MetricTable = table();
 
 # This is a hook for watching thresholds being crossed.  It is called whenever
 # index values are updated and the new val is given as the `val` argument.
@@ -427,6 +427,9 @@ function write_log(ts: time, metric_name: string, filter_name: string, data: Met
 
 function reset(filter: Filter)
 	{
+	if ( [filter$id, filter$name] in store )
+		delete store[filter$id, filter$name];
+
 	store[filter$id, filter$name] = table();
 	}
 

From 50827d8df0e6e34eda1e1465a7e4c535bc5c047e Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 21 Dec 2012 23:17:27 -0500
Subject: [PATCH 39/70] Implement an option to disable intermediate updates for
 the metrics framework.

 - There are some large sites having trouble and I'm thinking it
   might be overload from intermediate updates.
---
 scripts/base/frameworks/metrics/cluster.bro | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 59abd1a606..f14f3b1518 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -22,6 +22,11 @@ export {
 	## recently.
 	const cluster_request_global_view_percent = 0.1 &redef;
 
+	## Intermediate updates can cause overload situations on very large clusters.
+	## This option may help reduce load and correct intermittent problems.
+	## The goal for this option is also meant to be temporary.
+	const enable_intermediate_updates = T &redef;
+
 	# Event sent by the manager in a cluster to initiate the 
 	# collection of metrics values for a filter.
 	global cluster_filter_request: event(uid: string, id: string, filter_name: string);
@@ -77,7 +82,8 @@ function data_added(filter: Filter, index: Index, val: ResultVal)
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an 
 	# intermediate update.
-	if ( check_thresholds(filter, index, val, cluster_request_global_view_percent) )
+	if ( enable_intermediate_updates && 
+	     check_thresholds(filter, index, val, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
 		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index);

From c3a6916572436cffd4fc0e63bbdb8bcea584cd69 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 4 Jan 2013 16:54:13 -0500
Subject: [PATCH 40/70] More cluster tuning for the metrics framework.

 - Fixed several state maintenance issues for intermediate updates.

 - Added a new tuning variable Metrics::max_outstanding_global_views
   which limits the number of in-flight intermediate updates per
   metric filter.

 - Changed the default global view threshold percent to 20% (up from 10%)
---
 scripts/base/frameworks/metrics/cluster.bro | 60 +++++++++++++--------
 1 file changed, 39 insertions(+), 21 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index f14f3b1518..01e127e4bf 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -20,7 +20,13 @@ export {
 	## requirement that the manager requests a global view for the index
 	## since it may opt not to if it requested a global view for the index
 	## recently.
-	const cluster_request_global_view_percent = 0.1 &redef;
+	const cluster_request_global_view_percent = 0.2 &redef;
+
+	## This is to deal with intermediate update overload.  A manager will only allow
+	## this many intermediate update requests to the workers to be inflight at 
+	## any given time.  Requested intermediate updates are currently thrown out
+	## and not performed.  In practice this should hopefully have a minimal effect.
+	const max_outstanding_global_views = 10 &redef;
 
 	## Intermediate updates can cause overload situations on very large clusters.
 	## This option may help reduce load and correct intermittent problems.
@@ -55,21 +61,17 @@ export {
 }
 
 
-# This variable is maintained by all hosts for different purposes. Non-managers
-# maintain it to know what indexes they have recently sent as intermediate
-# updates so they don't overwhelm their manager. Managers maintain it so they
-# don't overwhelm workers with intermediate index requests. The count that is
-# yielded is the number of times the percentage threshold has been crossed and
-# an intermediate result has been received. The manager may optionally request
-# the index again before data expires from here if too many workers are crossing
-# the percentage threshold (not implemented yet!).
-global recent_global_view_indexes: table[string, string, Index] of count &create_expire=1min &default=0;
-
 # Add events to the cluster framework to make this work.
 redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
 redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
+# This variable is maintained to know what indexes they have recently sent as 
+# intermediate updates so they don't overwhelm their manager. The count that is
+# yielded is the number of times the percentage threshold has been crossed and
+# an intermediate result has been received.
+global recent_global_view_indexes: table[string, string, Index] of count &create_expire=1min &default=0;
+
 # This is done on all non-manager node types in the event that a metric is 
 # being collected somewhere other than a worker.
 function data_added(filter: Filter, index: Index, val: ResultVal)
@@ -134,9 +136,14 @@ event Metrics::cluster_index_request(uid: string, id: string, filter_name: strin
 	{
 	if ( [id, filter_name] in store && index in store[id, filter_name] )
 		{
-		local data = store[id, filter_name][index];
 		#print fmt("WORKER %s: received the cluster_index_request event for %s=%s.", Cluster::node, index2str(index), data);
-		event Metrics::cluster_index_response(uid, id, filter_name, index, data);
+		event Metrics::cluster_index_response(uid, id, filter_name, index, store[id, filter_name][index]);
+		}
+	else
+		{
+		# We need to send an empty response if we don't have the data so that the manager
+		# can know that it heard back from all of the workers.
+		event Metrics::cluster_index_response(uid, id, filter_name, index, [$begin=network_time(), $end=network_time()]);
 		}
 	}
 
@@ -164,8 +171,12 @@ global done_with: table[string] of count &read_expire=1min &default=0;
 # they are getting a global view for a certain index.
 global index_requests: table[string, string, string, Index] of ResultVal &read_expire=1min;
 
+# This variable is maintained by managers to prevent overwhelming communication due
+# to too many intermediate updates.  Each metric filter is tracked separately so that 
+# one metric won't overwhelm and degrade other quieter metrics.
+global outstanding_global_views: table[string, string] of count;
 
-# Manager's handle logging.
+# Managers handle logging.
 event Metrics::finish_period(filter: Filter)
 	{
 	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
@@ -194,26 +205,28 @@ function data_added(filter: Filter, index: Index, val: ResultVal)
 event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: ResultVal)
 	{
 	#print fmt("%0.6f MANAGER: receiving index data from %s - %s=%s", network_time(), get_event_peer()$descr, index2str(index), val);
-	if ( [uid, id, filter_name, index] in index_requests )
+
+	# We only want to try and do a value merge if there are actually measured datapoints
+	# in the ResultVal.
+	if ( val$num > 0 && [uid, id, filter_name, index] in index_requests )
 		index_requests[uid, id, filter_name, index] = merge_result_vals(index_requests[uid, id, filter_name, index], val);
 	else
 		index_requests[uid, id, filter_name, index] = val;
 
-	local ir = index_requests[uid, id, filter_name, index];
-	
 	# Mark that this worker is done.
 	++done_with[uid];
 
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
-
 	if ( Cluster::worker_count == done_with[uid] )
 		{
+		local ir = index_requests[uid, id, filter_name, index];
 		if ( check_thresholds(filter_store[id, filter_name], index, ir, 1.0) )
 			{
 			threshold_crossed(filter_store[id, filter_name], index, ir);
 			}
 		delete done_with[uid];
 		delete index_requests[uid, id, filter_name, index];
+		--outstanding_global_views[id, filter_name];
 		}
 	}
 
@@ -223,11 +236,16 @@ event Metrics::cluster_index_intermediate_response(id: string, filter_name: stri
 	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting index data for %s", index2str(index));
 
-	# If a worker recently sent this as an intermediate update, don't request it.
-	if ( [id, filter_name, index] in recent_global_view_indexes )
+	if ( [id, filter_name] in outstanding_global_views &&
+	     |outstanding_global_views[id, filter_name]| > max_outstanding_global_views )
+		{
+		# Don't do this intermediate update.  Perhaps at some point in the future 
+		# we will queue and randomly select from these ignored intermediate
+		# update requests.
 		return;
+		}
 
-	++recent_global_view_indexes[id, filter_name, index];
+	++outstanding_global_views[id, filter_name];
 
 	local uid = unique_id("");
 	event Metrics::cluster_index_request(uid, id, filter_name, index);

From ab7087f95337d5b072a0998aff770137cec5dfc2 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 4 Jan 2013 21:01:49 -0500
Subject: [PATCH 41/70] Hooks work now, enabling the hooks in the scan.bro
 script.

---
 scripts/policy/misc/scan.bro | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 1c13f8224b..4f19f29eb8 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -147,13 +147,11 @@ function add_metrics(id: conn_id, reverse: bool)
 	## Blacklisting/whitelisting subnets
 	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
 	#	return F;
-
-	# Hooks don't seem to be working like I expected.  They'll have to wait a bit longer.
 	
-	#if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
+	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
 		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
 
-	#if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
+	if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
 		Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 

From 9e1d5d87de38a847afb4cf7885378d5df80ce3e2 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 4 Jan 2013 23:34:57 -0500
Subject: [PATCH 42/70] New script to detect FTP bruteforcing.

 - Created a new time utils library.
---
 scripts/base/init-default.bro                 |  1 +
 scripts/base/utils/time.bro                   |  9 ++++
 scripts/policy/misc/scan.bro                  |  8 +--
 .../protocols/ftp/detect-bruteforcing.bro     | 53 +++++++++++++++++++
 4 files changed, 65 insertions(+), 6 deletions(-)
 create mode 100644 scripts/base/utils/time.bro
 create mode 100644 scripts/policy/protocols/ftp/detect-bruteforcing.bro

diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro
index 563f8af0bc..35e78531fc 100644
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@@ -15,6 +15,7 @@
 @load base/utils/queue
 @load base/utils/strings
 @load base/utils/thresholds
+@load base/utils/time
 @load base/utils/urls
 
 # This has some deep interplay between types and BiFs so it's 
diff --git a/scripts/base/utils/time.bro b/scripts/base/utils/time.bro
new file mode 100644
index 0000000000..abae46c144
--- /dev/null
+++ b/scripts/base/utils/time.bro
@@ -0,0 +1,9 @@
+
+## Given an interval, returns a string of the form 3m34s to
+## give a minimalized human readable string for the minutes 
+## and seconds represented by the interval.
+function duration_to_mins_secs(dur: interval): string
+	{
+	local dur_count = double_to_count(interval_to_double(dur));
+	return fmt("%dm%ds", dur_count/60, dur_count%60);
+	}
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 4f19f29eb8..7f5f43dbd9 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -7,6 +7,8 @@
 @load base/frameworks/notice
 @load base/frameworks/metrics
 
+@load base/utils/time
+
 module Scan;
 
 export {
@@ -61,12 +63,6 @@ function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVa
 	         val$sum > addr_scan_custom_thresholds[service] );
 	}
 
-function duration_to_mins_secs(dur: interval): string
-	{
-	local dur_count = double_to_count(interval_to_double(dur));
-	return fmt("%dm%ds", dur_count/60, dur_count%60);
-	}
-
 function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
 	{
 	local side = Site::is_local_addr(index$host) ? "local" : "remote";
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
new file mode 100644
index 0000000000..9206882071
--- /dev/null
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -0,0 +1,53 @@
+
+@load base/protocols/ftp
+@load base/frameworks/metrics
+
+@load base/utils/time
+
+module FTP;
+
+export {
+	redef enum Notice::Type += { 
+		## Indicates a host bruteforcing FTP logins by watching for too many
+		## rejected usernames or failed passwords.
+		Bruteforcing
+	};
+
+	## How many rejected usernames or passwords are required before being 
+	## considered to be bruteforcing.
+	const bruteforce_threshold = 2 &redef;
+}
+
+
+event bro_init()
+	{
+	Metrics::add_filter("ftp.failed_auth", [$every=15min,
+	                                        $measure=set(Metrics::SUM, Metrics::UNIQUE),
+	                                        $threshold_func(index: Metrics::Index, val: Metrics::ResultVal) = 
+	                                        	{
+	                                        	return val$num >= bruteforce_threshold;
+	                                        	},
+	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = 
+	                                        	{
+	                                        	print "booyah";
+	                                        	local dur = duration_to_mins_secs(val$end-val$begin);
+	                                        	local message = fmt("%s had %d failed logins on %d FTP servers in %s", index$host, val$num, val$unique, dur);
+	                                        	NOTICE([$note=FTP::Bruteforcing, 
+	                                        	        $src=index$host,
+	                                        	        $msg=message,
+	                                        	        $identifier=cat(index$host)]);
+	                                        	}]);
+	}
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	local cmd = c$ftp$cmdarg$cmd;
+	if ( cmd == "USER" || cmd == "PASS" )
+		{
+		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
+			{
+			print "yep";
+			Metrics::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$host=c$id$resp_h]);
+			}
+		}
+	}
\ No newline at end of file

From 283f7840b358cc78ea7254bc9001f380dd3fe014 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 4 Jan 2013 23:38:10 -0500
Subject: [PATCH 43/70] Removing some debugging print statements I accidently
 left behind.

---
 scripts/policy/protocols/ftp/detect-bruteforcing.bro | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 9206882071..69ef4d9c65 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -29,7 +29,6 @@ event bro_init()
 	                                        	},
 	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = 
 	                                        	{
-	                                        	print "booyah";
 	                                        	local dur = duration_to_mins_secs(val$end-val$begin);
 	                                        	local message = fmt("%s had %d failed logins on %d FTP servers in %s", index$host, val$num, val$unique, dur);
 	                                        	NOTICE([$note=FTP::Bruteforcing, 
@@ -45,9 +44,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
 	if ( cmd == "USER" || cmd == "PASS" )
 		{
 		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
-			{
-			print "yep";
 			Metrics::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$host=c$id$resp_h]);
-			}
 		}
 	}
\ No newline at end of file

From 9366411cf407dfe5b769565c677e36dfc1f7f493 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 4 Jan 2013 23:49:09 -0500
Subject: [PATCH 44/70] Fix the FTP bruteforce threshold to what it's really
 supposed to be.

---
 scripts/policy/protocols/ftp/detect-bruteforcing.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 69ef4d9c65..76053f7f5c 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -15,7 +15,7 @@ export {
 
 	## How many rejected usernames or passwords are required before being 
 	## considered to be bruteforcing.
-	const bruteforce_threshold = 2 &redef;
+	const bruteforce_threshold = 20 &redef;
 }
 
 

From bcd7fe114d9e22007b0f2bc3eab9fdc5efa4f066 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 5 Jan 2013 22:27:17 -0500
Subject: [PATCH 45/70] Fixed an issue causing reporter messages from the
 metrics framework.

---
 scripts/base/frameworks/metrics/cluster.bro | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro
index 01e127e4bf..721f2a212e 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/metrics/cluster.bro
@@ -174,7 +174,7 @@ global index_requests: table[string, string, string, Index] of ResultVal &read_e
 # This variable is maintained by managers to prevent overwhelming communication due
 # to too many intermediate updates.  Each metric filter is tracked separately so that 
 # one metric won't overwhelm and degrade other quieter metrics.
-global outstanding_global_views: table[string, string] of count;
+global outstanding_global_views: table[string, string] of count &default=0;
 
 # Managers handle logging.
 event Metrics::finish_period(filter: Filter)
@@ -226,7 +226,9 @@ event Metrics::cluster_index_response(uid: string, id: string, filter_name: stri
 			}
 		delete done_with[uid];
 		delete index_requests[uid, id, filter_name, index];
-		--outstanding_global_views[id, filter_name];
+		# Check that there is an outstanding view before subtracting.
+		if ( outstanding_global_views[id, filter_name] > 0 )
+			--outstanding_global_views[id, filter_name];
 		}
 	}
 

From 720089c03f2ce058e426ac1ee8c4f1b65c31ac48 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 5 Jan 2013 22:37:19 -0500
Subject: [PATCH 46/70] Add a feature to better select the value threshold will
 apply to in the metrics framework.

 - The feature was primarily added to allow the value to be
   modified for cluster based intermediate threshold checks
   without requiring the user to write the metrics filter
   differently for cluster consideration.  It's also a nice
   way to calculate some related information to the metric
   without accidently applying thresholds to that value.

 - Fixed a few small bugs in ftp detect-bruteforcing script
   and adapted it to the new threshold value selection feature.
---
 scripts/base/frameworks/metrics/main.bro         |  9 +++++++++
 .../policy/protocols/ftp/detect-bruteforcing.bro | 16 +++++++++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 0cfe96dfc6..9b167d200f 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -149,6 +149,12 @@ export {
 		## This is a special case of the normalize_func.
 		aggregation_mask:  count                    &optional;
 		
+		## Optionally provide a function to calculate a value from the ResultVal 
+		## structure which will be used for thresholding.  If no function is 
+		## provided, then in the following order of preference either the 
+		## $unique or the $sum fields will be used.
+		threshold_val_func: function(val: Metrics::ResultVal): count  &optional;
+
 		## A direct threshold for calling the $threshold_crossed function when 
 		## the SUM is greater than or equal to this value.
 		threshold:         count                    &optional;
@@ -602,6 +608,9 @@ function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_p
 	else if ( val?$sum )
 		watch = val$sum;
 
+	if ( filter?$threshold_val_func )
+		watch = filter$threshold_val_func(val);
+
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
 		watch = watch/modify_pct;
 
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 76053f7f5c..11d6ec71a1 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -16,17 +16,19 @@ export {
 	## How many rejected usernames or passwords are required before being 
 	## considered to be bruteforcing.
 	const bruteforce_threshold = 20 &redef;
+
+	## The time period in which the threshold needs to be crossed before
+	## being reset.
+	const bruteforce_measurement_interval = 15mins;
 }
 
 
 event bro_init()
 	{
-	Metrics::add_filter("ftp.failed_auth", [$every=15min,
-	                                        $measure=set(Metrics::SUM, Metrics::UNIQUE),
-	                                        $threshold_func(index: Metrics::Index, val: Metrics::ResultVal) = 
-	                                        	{
-	                                        	return val$num >= bruteforce_threshold;
-	                                        	},
+	Metrics::add_filter("ftp.failed_auth", [$every=bruteforce_measurement_interval,
+	                                        $measure=set(Metrics::UNIQUE),
+	                                        $threshold_val_func(val: Metrics::ResultVal) = { return val$num; },
+	                                        $threshold=bruteforce_threshold,
 	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = 
 	                                        	{
 	                                        	local dur = duration_to_mins_secs(val$end-val$begin);
@@ -44,6 +46,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
 	if ( cmd == "USER" || cmd == "PASS" )
 		{
 		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
-			Metrics::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$host=c$id$resp_h]);
+			Metrics::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
 		}
 	}
\ No newline at end of file

From e3856d76813ef6a8cc85d895855853727798a29d Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 27 Feb 2013 11:25:01 -0500
Subject: [PATCH 47/70] Removing a field that is no longer logged through the
 standard metrics log.

---
 scripts/base/frameworks/metrics/main.bro | 1 -
 1 file changed, 1 deletion(-)

diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
index 9b167d200f..6101d1cf4a 100644
--- a/scripts/base/frameworks/metrics/main.bro
+++ b/scripts/base/frameworks/metrics/main.bro
@@ -424,7 +424,6 @@ function write_log(ts: time, metric_name: string, filter_name: string, data: Met
 		local m: Info = [$ts=ts,
 		                 $ts_delta=filter$every,
 		                 $metric=filter$id,
-		                 $filter_name=filter$name,
 		                 $index=index,
 		                 $result=data[index]];
 		Log::write(LOG, m);

From 8778761c07e3d9009dd1e619a9e82bd525285d4a Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 13 Mar 2013 22:55:03 -0400
Subject: [PATCH 48/70] Checkpoint

---
 doc/scripts/DocSourcesList.cmake              |   6 +-
 .../{metrics => measurement}/__load__.bro     |   2 +
 .../{metrics => measurement}/cluster.bro      |  60 +-
 scripts/base/frameworks/measurement/main.bro  | 367 ++++++++++
 .../frameworks/measurement/non-cluster.bro    |  21 +
 .../measurement/plugins/__load__.bro          |   7 +
 .../measurement/plugins/average.bro           |  35 +
 .../frameworks/measurement/plugins/max.bro    |  37 +
 .../frameworks/measurement/plugins/min.bro    |  35 +
 .../measurement/plugins/std-dev.bro           |  36 +
 .../frameworks/measurement/plugins/sum.bro    |  35 +
 .../frameworks/measurement/plugins/unique.bro |  51 ++
 .../measurement/plugins/variance.bro          |  65 ++
 .../base/frameworks/measurement/simple.bro    |   6 +
 scripts/base/frameworks/metrics/main.bro      | 664 ------------------
 .../base/frameworks/metrics/non-cluster.bro   |  37 -
 scripts/base/init-default.bro                 |   2 +-
 scripts/base/protocols/ssh/main.bro           | 108 ++-
 .../frameworks/metrics/conn-example.bro       |   2 +-
 .../frameworks/metrics/http-example.bro       |   2 +-
 .../policy/frameworks/metrics/ssl-example.bro |   2 +-
 scripts/policy/misc/app-metrics.bro           |  68 +-
 .../policy/misc/detect-traceroute/main.bro    |   2 +-
 scripts/policy/misc/scan.bro                  |  10 +-
 .../protocols/conn/conn-stats-per-host.bro    |   2 +-
 scripts/policy/protocols/conn/metrics.bro     |   2 +-
 .../protocols/ftp/detect-bruteforcing.bro     |   7 +-
 scripts/policy/protocols/http/detect-sqli.bro |   2 +-
 scripts/policy/protocols/smtp/metrics.bro     |   2 +-
 .../protocols/ssh/detect-bruteforcing.bro     |   6 +-
 30 files changed, 833 insertions(+), 848 deletions(-)
 rename scripts/base/frameworks/{metrics => measurement}/__load__.bro (93%)
 rename scripts/base/frameworks/{metrics => measurement}/cluster.bro (83%)
 create mode 100644 scripts/base/frameworks/measurement/main.bro
 create mode 100644 scripts/base/frameworks/measurement/non-cluster.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/__load__.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/average.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/max.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/min.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/std-dev.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/sum.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/unique.bro
 create mode 100644 scripts/base/frameworks/measurement/plugins/variance.bro
 create mode 100644 scripts/base/frameworks/measurement/simple.bro
 delete mode 100644 scripts/base/frameworks/metrics/main.bro
 delete mode 100644 scripts/base/frameworks/metrics/non-cluster.bro

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index 5ce7221909..4e957d03a0 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -46,9 +46,9 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
+rest_target(${psd} base/frameworks/measurement/cluster.bro)
+rest_target(${psd} base/frameworks/measurement/main.bro)
+rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
diff --git a/scripts/base/frameworks/metrics/__load__.bro b/scripts/base/frameworks/measurement/__load__.bro
similarity index 93%
rename from scripts/base/frameworks/metrics/__load__.bro
rename to scripts/base/frameworks/measurement/__load__.bro
index 35f6b30fb5..fc784e1632 100644
--- a/scripts/base/frameworks/metrics/__load__.bro
+++ b/scripts/base/frameworks/measurement/__load__.bro
@@ -1,5 +1,7 @@
 @load ./main
 
+@load ./plugins
+
 # The cluster framework must be loaded first.
 @load base/frameworks/cluster
 
diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/measurement/cluster.bro
similarity index 83%
rename from scripts/base/frameworks/metrics/cluster.bro
rename to scripts/base/frameworks/measurement/cluster.bro
index 721f2a212e..6ccf5bb2f9 100644
--- a/scripts/base/frameworks/metrics/cluster.bro
+++ b/scripts/base/frameworks/measurement/cluster.bro
@@ -7,7 +7,7 @@
 @load base/frameworks/cluster
 @load ./main
 
-module Metrics;
+module Measurement;
 
 export {
 	## Allows a user to decide how large of result groups the 
@@ -48,13 +48,13 @@ export {
 	global cluster_index_request: event(uid: string, id: string, filter_name: string, index: Index);
 
 	# This event is sent by nodes in response to a 
-	# :bro:id:`Metrics::cluster_index_request` event.
+	# :bro:id:`Measurement::cluster_index_request` event.
 	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, val: ResultVal);
 
 	# This is sent by workers to indicate that they crossed the percent of the 
 	# current threshold by the percentage defined globally in 
-	# :bro:id:`Metrics::cluster_request_global_view_percent`
-	global cluster_index_intermediate_response: event(id: string, filter_name: string, index: Metrics::Index);
+	# :bro:id:`Measurement::cluster_request_global_view_percent`
+	global cluster_index_intermediate_response: event(id: string, filter_name: string, index: Measurement::Index);
 
 	# This event is scheduled internally on workers to send result chunks.
 	global send_data: event(uid: string, id: string, filter_name: string, data: MetricTable);
@@ -62,8 +62,8 @@ export {
 
 
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
-redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
+redef Cluster::manager2worker_events += /Measurement::cluster_(filter_request|index_request)/;
+redef Cluster::worker2manager_events += /Measurement::cluster_(filter_response|index_response|index_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # This variable is maintained to know what indexes they have recently sent as 
@@ -88,12 +88,12 @@ function data_added(filter: Filter, index: Index, val: ResultVal)
 	     check_thresholds(filter, index, val, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index);
+		event Measurement::cluster_index_intermediate_response(filter$id, filter$name, index);
 		++recent_global_view_indexes[filter$id, filter$name, index];
 		}
 	}
 
-event Metrics::send_data(uid: string, id: string, filter_name: string, data: MetricTable)
+event Measurement::send_data(uid: string, id: string, filter_name: string, data: MetricTable)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 	
@@ -115,35 +115,35 @@ event Metrics::send_data(uid: string, id: string, filter_name: string, data: Met
 	if ( |data| == 0 )
 		done = T;
 	
-	event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
+	event Measurement::cluster_filter_response(uid, id, filter_name, local_data, done);
 	if ( ! done )
-		event Metrics::send_data(uid, id, filter_name, data);
+		event Measurement::send_data(uid, id, filter_name, data);
 	}
 
-event Metrics::cluster_filter_request(uid: string, id: string, filter_name: string)
+event Measurement::cluster_filter_request(uid: string, id: string, filter_name: string)
 	{
 	#print fmt("WORKER %s: received the cluster_filter_request event for %s.", Cluster::node, id);
 	
 	# Initiate sending all of the data for the requested filter.
-	event Metrics::send_data(uid, id, filter_name, store[id, filter_name]);
+	event Measurement::send_data(uid, id, filter_name, store[id, filter_name]);
 	
 	# Lookup the actual filter and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
 	reset(filter_store[id, filter_name]);
 	}
 	
-event Metrics::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
+event Measurement::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
 	{
 	if ( [id, filter_name] in store && index in store[id, filter_name] )
 		{
 		#print fmt("WORKER %s: received the cluster_index_request event for %s=%s.", Cluster::node, index2str(index), data);
-		event Metrics::cluster_index_response(uid, id, filter_name, index, store[id, filter_name][index]);
+		event Measurement::cluster_index_response(uid, id, filter_name, index, store[id, filter_name][index]);
 		}
 	else
 		{
 		# We need to send an empty response if we don't have the data so that the manager
 		# can know that it heard back from all of the workers.
-		event Metrics::cluster_index_response(uid, id, filter_name, index, [$begin=network_time(), $end=network_time()]);
+		event Measurement::cluster_index_response(uid, id, filter_name, index, [$begin=network_time(), $end=network_time()]);
 		}
 	}
 
@@ -177,7 +177,7 @@ global index_requests: table[string, string, string, Index] of ResultVal &read_e
 global outstanding_global_views: table[string, string] of count &default=0;
 
 # Managers handle logging.
-event Metrics::finish_period(filter: Filter)
+event Measurement::finish_period(filter: Filter)
 	{
 	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
 	local uid = unique_id("");
@@ -189,9 +189,9 @@ event Metrics::finish_period(filter: Filter)
 	filter_results[uid, filter$id, filter$name] = table();
 	
 	# Request data from peers.
-	event Metrics::cluster_filter_request(uid, filter$id, filter$name);
+	event Measurement::cluster_filter_request(uid, filter$id, filter$name);
 	# Schedule the next finish_period event.
-	schedule filter$every { Metrics::finish_period(filter) };
+	schedule filter$every { Measurement::finish_period(filter) };
 	}
 
 # This is unlikely to be called often, but it's here in case there are metrics
@@ -202,7 +202,7 @@ function data_added(filter: Filter, index: Index, val: ResultVal)
 		threshold_crossed(filter, index, val);
 	}
 	
-event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: ResultVal)
+event Measurement::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: ResultVal)
 	{
 	#print fmt("%0.6f MANAGER: receiving index data from %s - %s=%s", network_time(), get_event_peer()$descr, index2str(index), val);
 
@@ -233,7 +233,7 @@ event Metrics::cluster_index_response(uid: string, id: string, filter_name: stri
 	}
 
 # Managers handle intermediate updates here.
-event Metrics::cluster_index_intermediate_response(id: string, filter_name: string, index: Index)
+event Measurement::cluster_index_intermediate_response(id: string, filter_name: string, index: Index)
 	{
 	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting index data for %s", index2str(index));
@@ -250,10 +250,10 @@ event Metrics::cluster_index_intermediate_response(id: string, filter_name: stri
 	++outstanding_global_views[id, filter_name];
 
 	local uid = unique_id("");
-	event Metrics::cluster_index_request(uid, id, filter_name, index);
+	event Measurement::cluster_index_request(uid, id, filter_name, index);
 	}
 
-event Metrics::cluster_filter_response(uid: string, id: string, filter_name: string, data: MetricTable, done: bool)
+event Measurement::cluster_filter_response(uid: string, id: string, filter_name: string, data: MetricTable, done: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 	
@@ -294,22 +294,6 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
 			delete requested_results[uid];
 			}
 		
-		if ( filter?$rollup )
-			{
-			for ( index in local_data )
-				{
-				if ( index !in rollup_store )
-					rollup_store[index] = table();
-				rollup_store[index][id, filter_name] = local_data[index];
-
-				# If all of the result vals are stored then the rollup callback can be executed.
-				if ( |rollup_store[index]| == |rollups[filter$rollup]$filters| )
-					{
-					rollups[filter$rollup]$callback(index, rollup_store[index]);
-					}
-				}
-			}
-
 		if ( filter?$period_finished )
 			filter$period_finished(ts, filter$id, filter$name, local_data);
 
diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
new file mode 100644
index 0000000000..3809fb16cc
--- /dev/null
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -0,0 +1,367 @@
+##! The metrics framework provides a way to count and measure data.  
+
+@load base/utils/queue
+
+module Measurement;
+
+export {
+	## The metrics logging stream identifier.
+	redef enum Log::ID += { LOG };
+	
+	## This is the interval for how often threshold based notices will happen 
+	## after they have already fired.
+	const threshold_crossed_restart_interval = 1hr &redef;
+	
+	## The various calculations are all defined as plugins.
+	type Calculation: enum {
+		PLACEHOLDER
+	};
+
+	## Represents a thing which is having metrics collected for it.  An instance
+	## of this record type and an id together represent a single measurement.
+	type Index: record {
+		## A non-address related metric or a sub-key for an address based metric.
+		## An example might be successful SSH connections by client IP address
+		## where the client string would be the index value.
+		## Another example might be number of HTTP requests to a particular
+		## value in a Host header.  This is an example of a non-host based
+		## metric since multiple IP addresses could respond for the same Host
+		## header value.
+		str:          string &optional;
+	
+		## Host is the value to which this metric applies.
+		host:         addr &optional;
+	} &log;
+	
+	## Represents data being added for a single metric data point.
+	## Only supply a single value here at a time.
+	type DataPoint: record {
+		## Count value.
+		num:       count       &optional;
+		## Double value.
+		dbl:       double      &optional;
+		## String value.
+		str:       string      &optional;
+	};
+
+	## Value supplied when a metric is finished.  It contains all
+	## of the measurements collected for the metric.  Most of the
+	## fields are added by calculation plugins.
+	type ResultVal: record {
+		## The time when this result was first started.
+		begin:    time          &log;
+
+		## The time when the last value was added to this result.
+		end:      time          &log;
+
+		## The number of measurements received.
+		num:      count         &log &default=0;
+
+		## A sample of something being measured.  This is helpful in 
+		## some cases for collecting information to do further detection
+		## or better logging for forensic purposes.
+		samples:  vector of string   &optional;
+	};
+	
+	type Measurement: record {
+		## The calculations to perform on the data.
+		apply:          set[Calculation];
+		
+		## A predicate so that you can decide per index if you would like
+		## to accept the data being inserted.
+		pred:           function(index: Measurement::Index, data: Measurement::DataPoint): bool &optional;
+		
+		## A function to normalize the index.  This can be used to aggregate or
+		## normalize the entire index.
+		normalize_func: function(index: Measurement::Index): Index &optional;
+
+		## A number of sample DataPoints to collect.
+		samples:        count &optional;
+	};
+
+
+	type Results: record {
+		begin: time;
+		end:   time;
+		result
+	};
+
+	## Type to store a table of metrics result values.
+	type ResultTable: table[Index] of Results;
+
+	## Filters define how the data from a metric is aggregated and handled.  
+	## Filters can be used to set how often the measurements are cut 
+	## and logged or how the data within them is aggregated.
+	type Filter: record {
+		## A name for the filter in case multiple filters are being
+		## applied to the same metric.  In most cases the default 
+		## filter name is fine and this field does not need to be set.
+		id:              string;
+
+		## The interval at which this filter should be "broken" and written
+		## to the logging stream.  The counters are also reset to zero at 
+		## this time so any threshold based detection needs to be set to a 
+		## number that should be expected to happen within this period.
+		every:              interval;
+
+		## Optionally provide a function to calculate a value from the ResultVal 
+		## structure which will be used for thresholding.  If no function is 
+		## provided, then in the following order of preference either the 
+		## $unique or the $sum fields will be used.
+		threshold_val_func: function(val: Measurement::ResultVal): count  &optional;
+
+		## The threshold value for calling the $threshold_crossed callback.
+		threshold:          count                    &optional;
+		
+		## A series of thresholds for calling the $threshold_crossed callback.
+		threshold_series:   vector of count          &optional;
+		
+		## A callback with the full collection of ResultVals for this filter.
+		## It's best to not access any global state outside of the variables
+		## given to the callback because there is no assurance provided as to
+		## where the callback will be executed on clusters.
+		period_finished:    function(data: Measurement::ResultTable) &optional;
+
+		## A callback that is called when a threshold is crossed.
+		threshold_crossed:  function(index: Measurement::Index, val: Measurement::ResultVal) &optional;
+	};
+	
+	## Function to associate a metric filter with a metric ID.
+	## 
+	## id: The metric ID that the filter should be associated with.
+	##
+	## filter: The record representing the filter configuration.
+	global add_filter: function(id: string, filter: Measurement::Filter);
+	
+	## Add data into a metric.  This should be called when
+	## a script has measured some point value and is ready to increment the
+	## counters.
+	##
+	## id: The metric identifier that the data represents.
+	##
+	## index: The metric index that the value is to be added to.
+	##
+	## increment: How much to increment the counter by.
+	global add_data: function(id: string, index: Measurement::Index, data: Measurement::DataPoint);
+
+	## Helper function to represent a :bro:type:`Measurement::Index` value as 
+	## a simple string.
+	## 
+	## index: The metric index that is to be converted into a string.
+	##
+	## Returns: A string reprentation of the metric index.
+	global index2str: function(index: Measurement::Index): string;
+	
+	## Event to access metrics records as they are passed to the logging framework.
+	global log_metrics: event(rec: Measurement::Info);
+	
+}
+
+redef record Filter += {
+	# Internal use only.  The metric that this filter applies to.  The value is automatically set.
+	id: string &optional;
+};
+
+redef record ResultVal += {
+	# Internal use only.  This is the queue where samples
+	# are maintained since the queue is self managing for
+	# the number of samples requested.
+	sample_queue: Queue::Queue &optional;
+
+	# Internal use only.  Indicates if a simple threshold was already crossed.
+	is_threshold_crossed: bool &default=F;
+
+	# Internal use only.  Current index for threshold series.
+	threshold_series_index: count &default=0;
+};
+
+# Store the filters indexed on the metric identifier and filter name.
+global filter_store: table[string, string] of Filter = table();
+
+# This is indexed by metric id and filter name.
+global store: table[string, string] of ResultTable = table();
+
+# This is a hook for watching thresholds being crossed.  It is called whenever
+# index values are updated and the new val is given as the `val` argument.
+# It's only prototyped here because cluster and non-cluster have separate 
+# implementations.
+global data_added: function(filter: Filter, index: Index, val: ResultVal);
+
+# Prototype the hook point for plugins to do calculations.
+global add_to_calculation: hook(filter: Filter, val: double, data: DataPoint, result: ResultVal);
+# Prototype the hook point for plugins to merge Measurements.
+global plugin_merge_measurements: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
+
+# Event that is used to "finish" metrics and adapt the metrics
+# framework for clustered or non-clustered usage.
+global finish_period: event(filter: Measurement::Filter);
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Measurement::LOG, [$columns=Info, $ev=log_metrics]);
+	}
+
+function index2str(index: Index): string
+	{
+	local out = "";
+	if ( index?$host )
+		out = fmt("%shost=%s", out, index$host);
+	if ( index?$str )
+		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", index$str);
+	return fmt("metric_index(%s)", out);
+	}
+
+function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
+	{
+	local result: ResultVal;
+
+	# Merge $begin (take the earliest one)
+	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
+
+	# Merge $end (take the latest one)
+	result$end = (rv1$end > rv2$end) ? rv1$end : rv2$end;
+
+	# Merge $num
+	result$num = rv1$num + rv2$num;
+
+	hook plugin_merge_measurements(result, rv1, rv2);
+	
+	# Merge $sample_queue
+	if ( rv1?$sample_queue && rv2?$sample_queue )
+		result$sample_queue = Queue::merge(rv1$sample_queue, rv2$sample_queue);
+	else if ( rv1?$sample_queue )
+		result$sample_queue = rv1$sample_queue;
+	else if ( rv2?$sample_queue )
+		result$sample_queue = rv2$sample_queue;
+	
+	# Merge $threshold_series_index
+	result$threshold_series_index = (rv1$threshold_series_index > rv2$threshold_series_index) ? rv1$threshold_series_index : rv2$threshold_series_index;
+
+	# Merge $is_threshold_crossed
+	if ( rv1$is_threshold_crossed || rv2$is_threshold_crossed )
+		result$is_threshold_crossed = T;
+
+	return result;
+	}
+	
+function reset(filter: Filter)
+	{
+	if ( [filter$id, filter$name] in store )
+		delete store[filter$id, filter$name];
+
+	store[filter$id, filter$name] = table();
+	}
+
+function add_filter(id: string, filter: Filter)
+	{
+	if ( [id, filter$name] in store )
+		{
+		Reporter::warning(fmt("invalid Metric filter (%s): Filter with same name already exists.", filter$name));
+		return;
+		}
+
+	if ( ! filter?$id )
+		filter$id = id;
+	
+	filter_store[id, filter$name] = filter;
+	store[id, filter$name] = table();
+
+	schedule filter$every { Measurement::finish_period(filter) };
+	}
+
+function add_data(id: string, index: Index, data: DataPoint)
+	{
+	# Try to add the data to all of the defined filters for the metric.
+	for ( [metric_id, filter_id] in filter_store )
+		{
+		local filter = filter_store[metric_id, filter_id];
+		
+		# If this filter has a predicate, run the predicate and skip this
+		# index if the predicate return false.
+		if ( filter?$pred && ! filter$pred(index, data) )
+			next;
+		
+		#if ( filter?$normalize_func )
+		#	index = filter$normalize_func(copy(index));
+		
+		local metric_tbl = store[id, filter$name];
+		if ( index !in metric_tbl )
+			metric_tbl[index] = [$begin=network_time(), $end=network_time()];
+
+		local result = metric_tbl[index];
+
+		# If a string was given, fall back to 1.0 as the value.
+		local val = 1.0;
+		if ( data?$num || data?$dbl )
+			val = data?$dbl ? data$dbl : data$num;
+
+		++result$num;
+		# Continually update the $end field.
+		result$end=network_time();
+
+		#if ( filter?$samples && filter$samples > 0 && data?$str )
+		#	{
+		#	if ( ! result?$sample_queue )
+		#		result$sample_queue = Queue::init([$max_len=filter$samples]);
+		#	Queue::push(result$sample_queue, data$str);
+		#	}
+
+		hook add_to_calculation(filter, val, data, result);
+		data_added(filter, index, result);
+		}
+	}
+
+# This function checks if a threshold has been crossed.  It is also used as a method to implement 
+# mid-break-interval threshold crossing detection for cluster deployments.
+function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_pct: double): bool
+	{
+	if ( ! (filter?$threshold || filter?$threshold_series) )
+		return;
+
+	local watch = 0.0;
+	if ( val?$unique )
+		watch = val$unique;
+	else if ( val?$sum )
+		watch = val$sum;
+
+	if ( filter?$threshold_val_func )
+		watch = filter$threshold_val_func(val);
+
+	if ( modify_pct < 1.0 && modify_pct > 0.0 )
+		watch = watch/modify_pct;
+
+	if ( ! val$is_threshold_crossed &&
+	     filter?$threshold && watch >= filter$threshold )
+		{
+		# A default threshold was given and the value crossed it.
+		return T;
+		}
+
+	if ( filter?$threshold_series &&
+	     |filter$threshold_series| >= val$threshold_series_index &&
+	     watch >= filter$threshold_series[val$threshold_series_index] )
+		{
+		# A threshold series was given and the value crossed the next 
+		# value in the series.
+		return T;
+		}
+
+	return F;
+	}
+
+function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
+	{
+	if ( ! filter?$threshold_crossed )
+		return;
+
+	if ( val?$sample_queue )
+		val$samples = Queue::get_str_vector(val$sample_queue);
+
+	filter$threshold_crossed(index, val);
+	val$is_threshold_crossed = T;
+
+	# Bump up to the next threshold series index if a threshold series is being used.
+	if ( filter?$threshold_series )
+		++val$threshold_series_index;
+	}
+
diff --git a/scripts/base/frameworks/measurement/non-cluster.bro b/scripts/base/frameworks/measurement/non-cluster.bro
new file mode 100644
index 0000000000..11bb7f16dc
--- /dev/null
+++ b/scripts/base/frameworks/measurement/non-cluster.bro
@@ -0,0 +1,21 @@
+@load ./main
+
+module Measurement;
+
+event Measurement::finish_period(filter: Filter)
+	{
+	local data = store[filter$id, filter$name];
+	if ( filter?$period_finished )
+		filter$period_finished(network_time(), filter$id, filter$name, data);
+
+	reset(filter);
+	
+	schedule filter$every { Measurement::finish_period(filter) };
+	}
+	
+	
+function data_added(filter: Filter, index: Index, val: ResultVal)
+	{
+	if ( check_thresholds(filter, index, val, 1.0) )
+		threshold_crossed(filter, index, val);
+	}
diff --git a/scripts/base/frameworks/measurement/plugins/__load__.bro b/scripts/base/frameworks/measurement/plugins/__load__.bro
new file mode 100644
index 0000000000..b708f917d1
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/__load__.bro
@@ -0,0 +1,7 @@
+@load ./average
+@load ./max
+@load ./min
+@load ./std-dev
+@load ./sum
+@load ./unique
+@load ./variance
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/average.bro b/scripts/base/frameworks/measurement/plugins/average.bro
new file mode 100644
index 0000000000..d3e1bef4d5
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/average.bro
@@ -0,0 +1,35 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Calculate the average of the values.
+		AVERAGE
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the average of all values.
+		average: double &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( AVERAGE in filter$measure )
+		{
+		if ( ! result?$average ) 
+			result$average = val;
+		else
+			result$average += (val - result$average) / result$num;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$average && rv2?$average )
+		result$average = ((rv1$average*rv1$num) + (rv2$average*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$average )
+		result$average = rv1$average;
+	else if ( rv2?$average )
+		result$average = rv2$average;
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/max.bro b/scripts/base/frameworks/measurement/plugins/max.bro
new file mode 100644
index 0000000000..806713dbd4
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/max.bro
@@ -0,0 +1,37 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the maximum value.
+		MAX
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the maximum value given.
+		max:      double        &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( MAX in filter$measure )
+		{
+		if ( ! result?$max ) 
+			result$max = val;
+		else if ( val > result$max )
+			result$max = val;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$max && rv2?$max )
+		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
+	else if ( rv1?$max )
+		result$max = rv1$max;
+	else if ( rv2?$max )
+		result$max = rv2$max;
+	}
+
+
diff --git a/scripts/base/frameworks/measurement/plugins/min.bro b/scripts/base/frameworks/measurement/plugins/min.bro
new file mode 100644
index 0000000000..e0d4003b31
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/min.bro
@@ -0,0 +1,35 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the minimum value.
+		MIN
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the minimum value given.
+		min:      double        &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( MIN in filter$measure )
+		{
+		if ( ! result?$min ) 
+			result$min = val;
+		else if ( val < result$min )
+			result$min = val;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$min && rv2?$min )
+		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
+	else if ( rv1?$min )
+		result$min = rv1$min;
+	else if ( rv2?$min )
+		result$min = rv2$min;
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/std-dev.bro b/scripts/base/frameworks/measurement/plugins/std-dev.bro
new file mode 100644
index 0000000000..cbd0db3416
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/std-dev.bro
@@ -0,0 +1,36 @@
+@load ./sum
+@load ./variance
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the standard deviation of the values.
+		STD_DEV
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the standard deviation.
+		std_dev:  double &log &optional;
+	};
+}
+
+# This depends on the variance plugin which uses priority -5
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=-10
+	{
+	if ( STD_DEV in filter$measure )
+		{
+		if ( result?$variance )
+			result$std_dev = sqrt(result$variance);
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
+	{
+	if ( rv1?$sum || rv2?$sum )
+		{
+		result$sum = rv1?$sum ? rv1$sum : 0;
+		if ( rv2?$sum )
+			result$sum += rv2$sum;
+		}
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/sum.bro b/scripts/base/frameworks/measurement/plugins/sum.bro
new file mode 100644
index 0000000000..2f615ffb6c
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/sum.bro
@@ -0,0 +1,35 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Sums the values given.  For string values,
+		## this will be the number of strings given.
+		SUM
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the sum of all values.
+		sum:      double        &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( SUM in filter$measure )
+		{
+		if ( ! result?$sum ) 
+			result$sum = 0;
+		result$sum += val;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$sum || rv2?$sum )
+		{
+		result$sum = rv1?$sum ? rv1$sum : 0;
+		if ( rv2?$sum )
+			result$sum += rv2$sum;
+		}
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/unique.bro b/scripts/base/frameworks/measurement/plugins/unique.bro
new file mode 100644
index 0000000000..66cab47897
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/unique.bro
@@ -0,0 +1,51 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Calculate the number of unique values.
+		UNIQUE
+	};
+
+	redef record ResultVal += {
+		## If cardinality is being tracked, the number of unique
+		## items is tracked here.
+		unique: count &log &optional;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  This is not meant to be publically available 
+	# because we don't want to trust that we can inspect the values 
+	# since we will like move to a probalistic data structure in the future.
+	# TODO: in the future this will optionally be a hyperloglog structure
+	unique_vals: set[DataPoint] &optional;
+};
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( UNIQUE in filter$measure )
+		{
+		if ( ! result?$unique_vals ) 
+			result$unique_vals=set();
+		add result$unique_vals[data];
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$unique_vals || rv2?$unique_vals )
+		{
+		if ( rv1?$unique_vals )
+			result$unique_vals = rv1$unique_vals;
+		
+		if ( rv2?$unique_vals )
+			if ( ! result?$unique_vals )
+				result$unique_vals = rv2$unique_vals;
+			else
+				for ( val2 in rv2$unique_vals )
+					add result$unique_vals[val2];
+
+		result$unique = |result$unique_vals|;
+		}
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/variance.bro b/scripts/base/frameworks/measurement/plugins/variance.bro
new file mode 100644
index 0000000000..df83361c35
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/variance.bro
@@ -0,0 +1,65 @@
+@load ./average
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the variance of the values.
+		VARIANCE
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the variance.
+		variance: double &log &optional;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  Used for incrementally calculating variance.
+	prev_avg: double &optional;
+
+	# Internal use only.  For calculating incremental variance.
+	var_s: double &optional;
+};
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=5
+	{
+	if ( VARIANCE in filter$measure )
+		result$prev_avg = result$average;
+	}
+
+# Reduced priority since this depends on the average
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=-5
+	{
+	if ( VARIANCE in filter$measure )
+		{
+		if ( ! result?$var_s )
+			result$var_s = 0.0;
+		result$var_s += (val - result$prev_avg) * (val - result$average);
+		result$variance = (val > 0) ? result$var_s/val : 0.0;
+		}
+	}
+
+# Reduced priority since this depends on the average
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-5
+	{
+	if ( rv1?$var_s && rv2?$var_s )
+		{
+		local rv1_avg_sq = (rv1$average - result$average);
+		rv1_avg_sq = rv1_avg_sq*rv1_avg_sq;
+		local rv2_avg_sq = (rv2$average - result$average);
+		rv2_avg_sq = rv2_avg_sq*rv2_avg_sq;
+		result$var_s = rv1$num*(rv1$var_s/rv1$num + rv1_avg_sq) + rv2$num*(rv2$var_s/rv2$num + rv2_avg_sq);
+		}
+	else if ( rv1?$var_s )
+		result$var_s = rv1$var_s;
+	else if ( rv2?$var_s )
+		result$var_s = rv2$var_s;
+
+	if ( rv1?$prev_avg && rv2?$prev_avg )
+		result$prev_avg = ((rv1$prev_avg*rv1$num) + (rv2$prev_avg*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$prev_avg )
+		result$prev_avg = rv1$prev_avg;
+	else if ( rv2?$prev_avg )
+		result$prev_avg = rv2$prev_avg;
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/simple.bro b/scripts/base/frameworks/measurement/simple.bro
new file mode 100644
index 0000000000..51bf7e8c44
--- /dev/null
+++ b/scripts/base/frameworks/measurement/simple.bro
@@ -0,0 +1,6 @@
+
+module Metrics;
+
+export {
+	
+}
\ No newline at end of file
diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro
deleted file mode 100644
index 6101d1cf4a..0000000000
--- a/scripts/base/frameworks/metrics/main.bro
+++ /dev/null
@@ -1,664 +0,0 @@
-##! The metrics framework provides a way to count and measure data.  
-
-@load base/utils/queue
-
-module Metrics;
-
-export {
-	## The metrics logging stream identifier.
-	redef enum Log::ID += { LOG };
-	
-	## This is the interval for how often threshold based notices will happen 
-	## after they have already fired.
-	const threshold_crossed_restart_interval = 1hr &redef;
-	
-	type Calculation: enum {
-		## Sums the values given.  For string values,
-		## this will be the number of strings given.
-		SUM,
-		## Find the minimum value.
-		MIN,
-		## Find the maximum value.
-		MAX,
-		## Find the variance of the values.
-		VARIANCE,
-		## Find the standard deviation of the values.
-		STD_DEV,
-		## Calculate the average of the values.
-		AVG,
-		## Calculate the number of unique values.
-		UNIQUE,
-	};
-
-	## Represents a thing which is having metrics collected for it.  An instance
-	## of this record type and an id together represent a single measurement.
-	type Index: record {
-		## A non-address related metric or a sub-key for an address based metric.
-		## An example might be successful SSH connections by client IP address
-		## where the client string would be the index value.
-		## Another example might be number of HTTP requests to a particular
-		## value in a Host header.  This is an example of a non-host based
-		## metric since multiple IP addresses could respond for the same Host
-		## header value.
-		str:          string &optional;
-	
-		## Host is the value to which this metric applies.
-		host:         addr &optional;
-		
-		## The CIDR block that this metric applies to.  This is typically
-		## only used internally for host based aggregation.
-		network:      subnet &optional;
-	} &log;
-	
-	## Represents data being added for a single metric data point.
-	## Only supply a single value here at a time.
-	type DataPoint: record {
-		## Count value.
-		num:       count       &optional;
-		## Double value.
-		dbl:       double      &optional;
-		## String value.
-		str:       string      &optional;
-	};
-
-	## Value supplied when a metric is finished.  It contains all
-	## of the measurements collected for the metric.
-	type ResultVal: record {
-		## The time when this result was first started.
-		begin:    time          &log;
-
-		## The time when the last value was added to this result.
-		end:      time          &log;
-
-		## The number of measurements received.
-		num:      count         &log &default=0;
-
-		## For numeric data, this tracks the sum of all values.
-		sum:      double        &log &optional;
-
-		## For numeric data, this tracks the minimum value given.
-		min:      double        &log &optional;
-
-		## For numeric data, this tracks the maximum value given.
-		max:      double        &log &optional;
-
-		## For numeric data, this calculates the average of all values.
-		avg:      double        &log &optional;
-
-		## For numeric data, this calculates the variance.
-		variance: double        &log &optional;
-
-		## For numeric data, this calculates the standard deviation.
-		std_dev:  double        &log &optional;
-
-		## If cardinality is being tracked, the number of unique
-		## items is tracked here.
-		unique:   count         &log &optional;
-
-		## A sample of something being measured.  This is helpful in 
-		## some cases for collecting information to do further detection
-		## or better logging for forensic purposes.
-		samples:  vector of string   &optional;
-	};
-
-	## The record type that is used for logging metrics.
-	type Info: record {
-		## Timestamp at which the metric was "broken".
-		ts:           time       &log;
-		## Interval between logging of this filter and the last time it was logged.
-		ts_delta:     interval   &log;
-		## What measurement the metric represents.
-		metric:       string     &log;
-		## What the metric value applies to.
-		index:        Index      &log;
-		## The simple numeric value of the metric.
-		result:       ResultVal  &log;
-	};
-	
-	## Type to store a table of metrics result values.
-	type MetricTable: table[Index] of ResultVal;
-
-	## Filters define how the data from a metric is aggregated and handled.  
-	## Filters can be used to set how often the measurements are cut 
-	## and logged or how the data within them is aggregated.  It's also 
-	## possible to disable logging and use filters solely for thresholding.
-	type Filter: record {
-		## A name for the filter in case multiple filters are being
-		## applied to the same metric.  In most cases the default 
-		## filter name is fine and this field does not need to be set.
-		name:              string                   &default="default";
-
-		## The interval at which this filter should be "broken" and written
-		## to the logging stream.  The counters are also reset to zero at 
-		## this time so any threshold based detection needs to be set to a 
-		## number that should be expected to happen within this period.
-		every:             interval;
-		
-		## The measurements to perform on the data.
-		measure:           set[Calculation]         &optional;
-		
-		## A predicate so that you can decide per index if you would like
-		## to accept the data being inserted.
-		pred:              function(index: Metrics::Index, data: Metrics::DataPoint): bool &optional;
-		
-		## A function to normalize the index.  This can be used to aggregate or
-		## normalize the entire index.
-		normalize_func:    function(index: Metrics::Index): Index &optional;
-		
-		## Global mask by to aggregate traffic measuring an attribute of hosts.
-		## This is a special case of the normalize_func.
-		aggregation_mask:  count                    &optional;
-		
-		## Optionally provide a function to calculate a value from the ResultVal 
-		## structure which will be used for thresholding.  If no function is 
-		## provided, then in the following order of preference either the 
-		## $unique or the $sum fields will be used.
-		threshold_val_func: function(val: Metrics::ResultVal): count  &optional;
-
-		## A direct threshold for calling the $threshold_crossed function when 
-		## the SUM is greater than or equal to this value.
-		threshold:         count                    &optional;
-		
-		## A series of thresholds for calling the $threshold_crossed function.
-		threshold_series:  vector of count          &optional;
-		
-		## A predicate so that you can decide when to flexibly declare when 
-		## a threshold crossed, and do extra work.
-		threshold_func:    function(index: Metrics::Index, val: Metrics::ResultVal): bool &optional;
-		
-		## A callback with the full collection of ResultVals for this filter.  This 
-		## is defined as a redef because the function includes a :bro:type:`Filter`
-		## record which is self referential before the Filter type has been fully 
-		## defined and doesn't work.
-		period_finished:   function(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable) &optional;
-
-		## A callback that is called when a threshold is crossed.
-		threshold_crossed: function(index: Metrics::Index, val: Metrics::ResultVal) &optional;
-
-		## A rollup to register this filter with.
-		rollup:            string &optional;
-
-		## A number of sample DataPoint strings to collect for the threshold 
-		## crossing callback.
-		samples:           count                    &optional;
-	};
-	
-	## Function to associate a metric filter with a metric ID.
-	## 
-	## id: The metric ID that the filter should be associated with.
-	##
-	## filter: The record representing the filter configuration.
-	global add_filter: function(id: string, filter: Metrics::Filter);
-	
-	## Add data into a metric.  This should be called when
-	## a script has measured some point value and is ready to increment the
-	## counters.
-	##
-	## id: The metric identifier that the data represents.
-	##
-	## index: The metric index that the value is to be added to.
-	##
-	## increment: How much to increment the counter by.
-	global add_data: function(id: string, index: Metrics::Index, data: Metrics::DataPoint);
-
-	## The callback definition for rollup functions.
-	type RollupCallback: function(index: Metrics::Index, vals: table[string, string] of Metrics::ResultVal);
-
-	## Add a rollup function for merging multiple filters with matching 
-	## indexes.  If the metrics filters being merged don't have equivalent times
-	## in the $every field, an error will be generated.
-	##
-	## name: An arbitrary name for this filter rollup.
-	##
-	## vals: Each ResultVal record indexed by the appropriate metric name and filter name.
-	global create_index_rollup: function(name: string, rollup: RollupCallback);
-
-	## Helper function to represent a :bro:type:`Metrics::Index` value as 
-	## a simple string.
-	## 
-	## index: The metric index that is to be converted into a string.
-	##
-	## Returns: A string reprentation of the metric index.
-	global index2str: function(index: Metrics::Index): string;
-	
-	## A helper function to use with the `period_finished` field in filters.  Using 
-	## this function is not recommended however since each metric likely has 
-	## different data and different semantics which would be better served by writing
-	## a custom function that logs in more domain specific fashion.
-	global write_log: function(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable);
-
-	## Event to access metrics records as they are passed to the logging framework.
-	global log_metrics: event(rec: Metrics::Info);
-	
-}
-
-redef record Filter += {
-	# Internal use only.  The metric that this filter applies to.  The value is automatically set.
-	id: string &optional;
-};
-
-redef record ResultVal += {
-	# Internal use only.  Used for incrementally calculating variance.
-	prev_avg:      double      &optional;
-
-	# Internal use only.  For calculating variance.
-	var_s:         double      &optional;
-
-	# Internal use only.  This is not meant to be publically available 
-	# because we don't want to trust that we can inspect the values 
-	# since we will like move to a probalistic data structure in the future.
-	# TODO: in the future this will optionally be a hyperloglog structure
-	unique_vals:  set[DataPoint] &optional;
-
-	# Internal use only.  This is the queue where samples
-	# are maintained since the queue is self managing for
-	# the number of samples requested.
-	sample_queue: Queue::Queue &optional;
-
-	# Internal use only.  Indicates if a simple threshold was already crossed.
-	is_threshold_crossed: bool &default=F;
-
-	# Internal use only.  Current index for threshold series.
-	threshold_series_index: count &default=0;
-};
-
-# Store the filters indexed on the metric identifier.
-global metric_filters: table[string] of vector of Filter = table();
-
-# Store the filters indexed on the metric identifier and filter name.
-global filter_store: table[string, string] of Filter = table();
-
-# This is indexed by metric id and filter name.
-global store: table[string, string] of MetricTable = table();
-
-# This is a hook for watching thresholds being crossed.  It is called whenever
-# index values are updated and the new val is given as the `val` argument.
-# It's only prototyped here because cluster and non-cluster have separate 
-# implementations.
-global data_added: function(filter: Filter, index: Index, val: ResultVal);
-
-type Rollup: record {
-	callback: RollupCallback;
-	filters:  set[Filter] &optional;
-};
-global rollups: table[string] of Rollup;
-global rollup_store: table[Index] of table[string, string] of ResultVal = {};
-
-
-## Event that is used to "finish" metrics and adapt the metrics
-## framework for clustered or non-clustered usage.
-global finish_period: event(filter: Metrics::Filter);
-
-event bro_init() &priority=5
-	{
-	Log::create_stream(Metrics::LOG, [$columns=Info, $ev=log_metrics]);
-	}
-
-function index2str(index: Index): string
-	{
-	local out = "";
-	if ( index?$host )
-		out = fmt("%shost=%s", out, index$host);
-	if ( index?$network )
-		out = fmt("%s%snetwork=%s", out, |out|==0 ? "" : ", ", index$network);
-	if ( index?$str )
-		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", index$str);
-	return fmt("metric_index(%s)", out);
-	}
-	
-function do_calculated_fields(val: ResultVal)
-	{
-	if ( val?$unique_vals )
-		val$unique = |val$unique_vals|;
-	if ( val?$var_s )
-		val$variance = (val$num > 1) ? val$var_s/val$num : 0.0;
-	if ( val?$variance )
-		val$std_dev = sqrt(val$variance);
-	}
-
-function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
-	{
-	local result: ResultVal;
-
-	# Merge $begin (take the earliest one)
-	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
-
-	# Merge $end (take the latest one)
-	result$end = (rv1$end > rv2$end) ? rv1$end : rv2$end;
-
-	# Merge $num
-	result$num = rv1$num + rv2$num;
-
-	# Merge $sum
-	if ( rv1?$sum || rv2?$sum )
-		{
-		result$sum = rv1?$sum ? rv1$sum : 0;
-		if ( rv2?$sum )
-			result$sum += rv2$sum;
-		}
-	
-	# Merge $max
-	if ( rv1?$max && rv2?$max )
-		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
-	else if ( rv1?$max )
-		result$max = rv1$max;
-	else if ( rv2?$max )
-		result$max = rv2$max;
-
-	# Merge $min
-	if ( rv1?$min && rv2?$min )
-		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
-	else if ( rv1?$min )
-		result$min = rv1$min;
-	else if ( rv2?$min )
-		result$min = rv2$min;
-
-	# Merge $avg
-	if ( rv1?$avg && rv2?$avg )
-		result$avg = ((rv1$avg*rv1$num) + (rv2$avg*rv2$num))/(rv1$num+rv2$num);
-	else if ( rv1?$avg )
-		result$avg = rv1$avg;
-	else if ( rv2?$avg )
-		result$avg = rv2$avg;
-
-	# Merge $prev_avg
-	if ( rv1?$prev_avg && rv2?$prev_avg )
-		result$prev_avg = ((rv1$prev_avg*rv1$num) + (rv2$prev_avg*rv2$num))/(rv1$num+rv2$num);
-	else if ( rv1?$prev_avg )
-		result$prev_avg = rv1$prev_avg;
-	else if ( rv2?$prev_avg )
-		result$prev_avg = rv2$prev_avg;
-
-	# Merge $var_s
-	if ( rv1?$var_s && rv2?$var_s )
-		{
-		local rv1_avg_sq = (rv1$avg - result$avg);
-		rv1_avg_sq = rv1_avg_sq*rv1_avg_sq;
-		local rv2_avg_sq = (rv2$avg - result$avg);
-		rv2_avg_sq = rv2_avg_sq*rv2_avg_sq;
-		result$var_s = rv1$num*(rv1$var_s/rv1$num + rv1_avg_sq) + rv2$num*(rv2$var_s/rv2$num + rv2_avg_sq);
-		}
-	else if ( rv1?$var_s )
-		result$var_s = rv1$var_s;
-	else if ( rv2?$var_s )
-		result$var_s = rv2$var_s;
-
-	# Merge $unique_vals
-	if ( rv1?$unique_vals || rv2?$unique_vals )
-		{
-		if ( rv1?$unique_vals )
-			result$unique_vals = rv1$unique_vals;
-		
-		if ( rv2?$unique_vals )
-			if ( ! result?$unique_vals )
-				result$unique_vals = rv2$unique_vals;
-			else
-				for ( val2 in rv2$unique_vals )
-					add result$unique_vals[val2];
-		}
-
-	# Merge $sample_queue
-	if ( rv1?$sample_queue && rv2?$sample_queue )
-		result$sample_queue = Queue::merge(rv1$sample_queue, rv2$sample_queue);
-	else if ( rv1?$sample_queue )
-		result$sample_queue = rv1$sample_queue;
-	else if ( rv2?$sample_queue )
-		result$sample_queue = rv2$sample_queue;
-	
-	# Merge $threshold_series_index
-	result$threshold_series_index = (rv1$threshold_series_index > rv2$threshold_series_index) ? rv1$threshold_series_index : rv2$threshold_series_index;
-
-	# Merge $is_threshold_crossed
-	if ( rv1$is_threshold_crossed || rv2$is_threshold_crossed )
-		result$is_threshold_crossed = T;
-
-	do_calculated_fields(result);
-	return result;
-	}
-	
-function write_log(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable)
-	{
-	local filter = filter_store[metric_name, filter_name];
-	for ( index in data )
-		{
-		local m: Info = [$ts=ts,
-		                 $ts_delta=filter$every,
-		                 $metric=filter$id,
-		                 $index=index,
-		                 $result=data[index]];
-		Log::write(LOG, m);
-		}
-	}
-
-function reset(filter: Filter)
-	{
-	if ( [filter$id, filter$name] in store )
-		delete store[filter$id, filter$name];
-
-	store[filter$id, filter$name] = table();
-	}
-
-function add_filter(id: string, filter: Filter)
-	{
-	if ( filter?$normalize_func && filter?$aggregation_mask )
-		{
-		Reporter::warning(fmt("invalid Metric filter (%s): Defined both $normalize_func and $aggregation_mask.", filter$name));
-		return;
-		}
-	if ( [id, filter$name] in store )
-		{
-		Reporter::warning(fmt("invalid Metric filter (%s): Filter with same name already exists.", filter$name));
-		return;
-		}
-	if ( filter?$rollup )
-		{
-		if ( filter$rollup !in rollups )
-			{
-			Reporter::warning(fmt("invalid Metric filter (%s): %s rollup doesn't exist.", filter$name, filter$rollup));
-			return;
-			}
-		else
-			{
-			local every_field = 0secs;
-			for ( filt in rollups )
-				{
-				if ( [id, filt] !in filter_store )
-					next;
-				
-				if ( every_field == 0secs )
-					every_field = filter_store[id, filt]$every;
-				else if ( every_field == filter_store[id, filt]$every )
-					{
-					Reporter::warning(fmt("invalid Metric rollup for %s: Filters with differing $every fields applied to %s.", filter$name, filter$rollup));
-					return;
-					}
-				}
-			}
-			add rollups[filter$rollup]$filters[filter];
-		}
-
-	if ( ! filter?$id )
-		filter$id = id;
-	
-	if ( id !in metric_filters )
-		metric_filters[id] = vector();
-	metric_filters[id][|metric_filters[id]|] = filter;
-
-	filter_store[id, filter$name] = filter;
-	store[id, filter$name] = table();
-
-	schedule filter$every { Metrics::finish_period(filter) };
-	}
-
-function add_data(id: string, index: Index, data: DataPoint)
-	{
-	if ( id !in metric_filters )
-		return;
-	
-	local filters = metric_filters[id];
-	
-	# Try to add the data to all of the defined filters for the metric.
-	for ( filter_id in filters )
-		{
-		local filter = filters[filter_id];
-		
-		# If this filter has a predicate, run the predicate and skip this
-		# index if the predicate return false.
-		if ( filter?$pred && ! filter$pred(index, data) )
-			next;
-		
-		if ( filter?$normalize_func )
-			index = filter$normalize_func(copy(index));
-
-		if ( index?$host && filter?$aggregation_mask )
-			{
-			index$network = mask_addr(index$host, filter$aggregation_mask);
-			delete index$host;
-			}
-		
-		local metric_tbl = store[id, filter$name];
-		if ( index !in metric_tbl )
-			metric_tbl[index] = [$begin=network_time(), $end=network_time()];
-
-		local result = metric_tbl[index];
-
-		# If a string was given, fall back to 1.0 as the value.
-		local val = 1.0;
-		if ( data?$num || data?$dbl )
-			val = data?$dbl ? data$dbl : data$num;
-
-		++result$num;
-		# Continually update the $end field.
-		result$end=network_time();
-
-		if ( filter?$samples && filter$samples > 0 && data?$str )
-			{
-			if ( ! result?$sample_queue )
-				result$sample_queue = Queue::init([$max_len=filter$samples]);
-			Queue::push(result$sample_queue, data$str);
-			}
-
-		if ( SUM in filter$measure )
-			{
-			if ( ! result?$sum ) result$sum = 0;
-			result$sum += val;
-			}
-
-		if ( MIN in filter$measure )
-			{
-			if ( ! result?$min ) 
-				result$min = val;
-			else if ( val < result$min )
-				result$min = val;
-			}
-
-		if ( MAX in filter$measure )
-			{
-			if ( ! result?$max ) 
-				result$max = val;
-			else if ( val > result$max )
-				result$max = val;
-			}
-	
-		if ( AVG in filter$measure || VARIANCE in filter$measure )
-			{
-			if ( ! result?$avg ) 
-				{
-				result$avg = val;
-				result$prev_avg = val;
-				}
-			else
-				{
-				result$prev_avg = result$avg;
-				result$avg += (val - result$avg) / result$num;
-				}
-			}
-
-		if ( VARIANCE in filter$measure )
-			{
-			if ( ! result?$var_s ) result$var_s = 0.0;
-			result$var_s += (val - result$prev_avg)*(val - result$avg);
-			}
-
-		#if ( STD_DEV in filter$measure )
-		#	{
-		#	#if ( result?$variance )
-		#	#	result$std_dev = sqrt(result$variance);
-		#	}
-
-		if ( UNIQUE in filter$measure )
-			{
-			if ( ! result?$unique_vals ) result$unique_vals=set();
-			add result$unique_vals[data];
-			}
-
-		do_calculated_fields(result);
-		data_added(filter, index, result);
-		}
-	}
-
-# This function checks if a threshold has been crossed.  It is also used as a method to implement 
-# mid-break-interval threshold crossing detection for cluster deployments.
-function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_pct: double): bool
-	{
-	local watch = 0.0;
-	if ( val?$unique )
-		watch = val$unique;
-	else if ( val?$sum )
-		watch = val$sum;
-
-	if ( filter?$threshold_val_func )
-		watch = filter$threshold_val_func(val);
-
-	if ( modify_pct < 1.0 && modify_pct > 0.0 )
-		watch = watch/modify_pct;
-
-	if ( ! val$is_threshold_crossed &&
-	     filter?$threshold && watch >= filter$threshold )
-		{
-		# A default threshold was given and the value crossed it.
-		return T;
-		}
-
-	if ( filter?$threshold_series &&
-	     |filter$threshold_series| >= val$threshold_series_index &&
-	     watch >= filter$threshold_series[val$threshold_series_index] )
-		{
-		# A threshold series was given and the value crossed the next 
-		# value in the series.
-		return T;
-		}
-
-	if ( ! val$is_threshold_crossed &&
-	     filter?$threshold_func &&
-	     filter$threshold_func(index, val) )
-		{
-		# The threshold function indicated it was crossed.
-		return T;
-		}
-
-	return F;
-	}
-
-function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
-	{
-	if ( ! filter?$threshold_crossed )
-		return;
-
-	if ( val?$sample_queue )
-		val$samples = Queue::get_str_vector(val$sample_queue);
-
-	filter$threshold_crossed(index, val);
-	val$is_threshold_crossed = T;
-
-	# Bump up to the next threshold series index if a threshold series is being used.
-	if ( filter?$threshold_series )
-		++val$threshold_series_index;
-	}
-
-function create_index_rollup(name: string, rollup: RollupCallback)
-	{
-	local r: Rollup = [$callback=rollup];
-	r$filters=set();
-	rollups[name] = r;
-	}
diff --git a/scripts/base/frameworks/metrics/non-cluster.bro b/scripts/base/frameworks/metrics/non-cluster.bro
deleted file mode 100644
index b76ca3ea48..0000000000
--- a/scripts/base/frameworks/metrics/non-cluster.bro
+++ /dev/null
@@ -1,37 +0,0 @@
-@load ./main
-
-module Metrics;
-
-event Metrics::finish_period(filter: Filter)
-	{
-	local data = store[filter$id, filter$name];
-	if ( filter?$rollup )
-		{
-		for ( index in data )
-			{
-			if ( index !in rollup_store )
-				rollup_store[index] = table();
-			rollup_store[index][filter$id, filter$name] = data[index];
-
-			# If all of the result vals are stored then the rollup callback can be executed.
-			if ( |rollup_store[index]| == |rollups[filter$rollup]$filters| )
-				{
-				rollups[filter$rollup]$callback(index, rollup_store[index]);
-				}
-			}
-		}
-
-	if ( filter?$period_finished )
-		filter$period_finished(network_time(), filter$id, filter$name, data);
-
-	reset(filter);
-	
-	schedule filter$every { Metrics::finish_period(filter) };
-	}
-	
-	
-function data_added(filter: Filter, index: Index, val: ResultVal)
-	{
-	if ( check_thresholds(filter, index, val, 1.0) )
-		threshold_crossed(filter, index, val);
-	}
diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro
index 35e78531fc..bb64accaea 100644
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@@ -29,7 +29,7 @@
 @load base/frameworks/communication
 @load base/frameworks/control
 @load base/frameworks/cluster
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/frameworks/intel
 @load base/frameworks/reporter
 @load base/frameworks/tunnels
diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro
index cd20f4e913..d782c4fa37 100644
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@@ -17,12 +17,6 @@ export {
 	## The SSH protocol logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	redef enum Notice::Type += { 
-		## Indicates that a heuristically detected "successful" SSH 
-		## authentication occurred.
-		Login 
-	};
-
 	type Info: record {
 		## Time when the SSH connection began.
 		ts:              time         &log;
@@ -30,9 +24,9 @@ export {
 		uid:             string       &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
-		## Indicates if the login was heuristically guessed to be "success"
-		## or "failure".
-		status:          string       &log &optional;
+		## Indicates if the login was heuristically guessed to be "success",
+		## "failure", or "undetermined".  
+		status:          string       &log &default="undetermined";
 		## Direction of the connection.  If the client was a local host 
 		## logging into an external host, this would be OUTBOUND. INBOUND
 		## would be set for the opposite situation.
@@ -54,12 +48,12 @@ export {
 	
 	## The size in bytes of data sent by the server at which the SSH 
 	## connection is presumed to be successful.
-	const authentication_data_size = 5500 &redef;
+	const authentication_data_size = 4000 &redef;
 	
 	## If true, we tell the event engine to not look at further data
 	## packets after the initial SSH handshake. Helps with performance
 	## (especially with large file transfers) but precludes some
-	## kinds of analyses (e.g., tracking connection size).
+	## kinds of analyses.
 	const skip_processing_after_detection = F &redef;
 	
 	## Event that is generated when the heuristic thinks that a login
@@ -104,54 +98,60 @@ function set_session(c: connection)
 
 function check_ssh_connection(c: connection, done: bool)
 	{
-	# If done watching this connection, just return.
+	# If already done watching this connection, just return.
 	if ( c$ssh$done )
 		return;
 	
-	# Make sure conn_size_analyzer is active by checking 
-	# resp$num_bytes_ip.  In general it should always be active though.
-	if ( ! c$resp?$num_bytes_ip )
-		return;
-	
-	# Remove the IP and TCP header length from the total size.
-	# TODO: Fix for IPv6.  This whole approach also seems to break in some 
-	#       cases where there are more header bytes than num_bytes_ip.
-	local header_bytes = c$resp$num_pkts*32 + c$resp$num_pkts*20;
-	local server_bytes = c$resp$num_bytes_ip;
-	if ( server_bytes >= header_bytes )
-		server_bytes = server_bytes - header_bytes;
-	else
-		server_bytes = c$resp$size;
-	
-	# If this is still a live connection and the byte count has not crossed 
-	# the threshold, just return and let the rescheduled check happen later.
-	if ( ! done && server_bytes < authentication_data_size )
-		return;
-
-	# Make sure the server has sent back more than 50 bytes to filter out
-	# hosts that are just port scanning.  Nothing is ever logged if the server
-	# doesn't send back at least 50 bytes.
-	if ( server_bytes < 50 )
-		return;
-
-	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
-	c$ssh$resp_size = server_bytes;
-	
-	if ( server_bytes < authentication_data_size )
+	if ( done )
 		{
-		c$ssh$status  = "failure";
-		event SSH::heuristic_failed_login(c);
+		# If this connection is done, then we can look to see if 
+		# this matches the conditions for a failed login.  Failed
+		# logins are only detected at connection state removal.
+
+		if ( # Require originators to have sent at least 50 bytes. 
+		     c$orig$size > 50 &&
+		     # Responders must be below 4000 bytes.
+		     c$resp$size < 4000 && 
+		     # Responder must have sent fewer than 40 packets.
+		     c$resp$num_pkts < 40 &&
+		     # If there was a content gap we can't reliably do this heuristic.
+		     c$conn$missed_bytes == 0)# && 
+		     # Only "normal" connections can count.
+		     #c$conn?$conn_state && c$conn$conn_state in valid_states )
+			{
+			c$ssh$status = "failure";
+			event SSH::heuristic_failed_login(c);
+			}
+
+		if ( c$resp$size > authentication_data_size )
+			{
+			c$ssh$status = "success";
+			event SSH::heuristic_successful_login(c);
+			}
 		}
 	else
-		{ 
-		# presumed successful login
-		c$ssh$status = "success";
-		event SSH::heuristic_successful_login(c);
+		{
+		# If this connection is still being tracked, then it's possible
+		# to watch for it to be a successful connection.
+		if ( c$resp$size > authentication_data_size )
+			{
+			c$ssh$status = "success";
+			event SSH::heuristic_successful_login(c);
+			}
+		else
+			# This connection must be tracked longer.  Let the scheduled
+			# check happen again.
+			return;
 		}
+
+	# Set the direction for the log.
+	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
 	
 	# Set the "done" flag to prevent the watching event from rescheduling
 	# after detection is done.
 	c$ssh$done=T;
+
+	Log::write(SSH::LOG, c$ssh);
 	
 	if ( skip_processing_after_detection )
 		{
@@ -161,18 +161,6 @@ function check_ssh_connection(c: connection, done: bool)
 		}
 	}
 
-event SSH::heuristic_successful_login(c: connection) &priority=-5
-	{
-	NOTICE([$note=Login, 
-	        $msg="Heuristically detected successful SSH login.",
-	        $conn=c]);
-	
-	Log::write(SSH::LOG, c$ssh);
-	}
-event SSH::heuristic_failed_login(c: connection) &priority=-5
-	{
-	Log::write(SSH::LOG, c$ssh);
-	}
 
 event connection_state_remove(c: connection) &priority=-5
 	{
diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro
index 1271d6eb32..3f87ecb283 100644
--- a/scripts/policy/frameworks/metrics/conn-example.bro
+++ b/scripts/policy/frameworks/metrics/conn-example.bro
@@ -1,7 +1,7 @@
 ##! An example of using the metrics framework to collect connection metrics 
 ##! aggregated into /24 CIDR ranges.
 
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/utils/site
 
 event bro_init()
diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro
index b3284580e8..d7aa304754 100644
--- a/scripts/policy/frameworks/metrics/http-example.bro
+++ b/scripts/policy/frameworks/metrics/http-example.bro
@@ -2,7 +2,7 @@
 ##! only local networks.  Additionally, the status code for the response from
 ##! the request is added into the metric.
 
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/protocols/http
 @load base/utils/site
 
diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro
index 3b9b848edb..400373c06c 100644
--- a/scripts/policy/frameworks/metrics/ssl-example.bro
+++ b/scripts/policy/frameworks/metrics/ssl-example.bro
@@ -3,7 +3,7 @@
 ##! establishments.  Names ending in google.com are being filtered out as an
 ##! example of the predicate based filtering in metrics filters.
 
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/protocols/ssl
 
 event bro_init()
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index 0a4fc8b39f..68deddaa29 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -1,8 +1,8 @@
 @load base/protocols/http
 @load base/protocols/ssl
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 
-module AppMetrics;
+module AppMeasurement;
 
 export {
 	redef enum Log::ID += { LOG };
@@ -19,7 +19,11 @@ export {
 	const break_interval = 15mins &redef;
 }
 
-function app_metrics_rollup(index: Metrics::Index, vals: table[string, string] of Metrics::ResultVal) 
+redef record connection += {
+	resp_hostname: string &optional;
+};
+
+function app_metrics_rollup(index: Measurement::Index, vals: table[string, string] of Measurement::ResultVal) 
 	{
 	local l: Info;
 	l$ts = network_time();
@@ -35,55 +39,67 @@ function app_metrics_rollup(index: Metrics::Index, vals: table[string, string] o
 			l$uniq_hosts = val$unique;
 			}
 		}
-	Log::write(LOG, l);
 	}
 
 event bro_init() &priority=3
 	{
-	Log::create_stream(AppMetrics::LOG, [$columns=Info]);
+	Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
 
-	Metrics::create_index_rollup("AppMetrics", app_metrics_rollup);
-	Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM),    $rollup="AppMetrics"]);
-	Metrics::add_filter("apps.hits",  [$every=break_interval, $measure=set(Metrics::UNIQUE), $rollup="AppMetrics"]);
+	#Measurement::create_index_rollup("AppMeasurement", app_metrics_rollup);
+	#Measurement::add_filter("apps.bytes", [$every=break_interval, $measure=set(Measurement::SUM),    $rollup="AppMeasurement"]);
+	#Measurement::add_filter("apps.hits",  [$every=break_interval, $measure=set(Measurement::UNIQUE), $rollup="AppMeasurement"]);
+
+	Measurement::create([$epoch=break_interval, 
+	                     $measurements=table(["apps.bytes"] = [$apply=set(Measurement::SUM)],
+	                                         ["apps.hits"]  = [$apply=set(Measurement::UNIQUE)]),
+	                     $period_finished(result: Measurement::Results) = 
+	                     	{
+	                     	local l: Info;
+	                     	l$ts = network_time();
+	                     	for ( index in result )
+	                     		{
+	                     		l$bytes      = double_to_count(floor(result[index]["apps.bytes"]$sum));
+	                     		l$hits       = result[index]["apps.hits"]$num;
+	                     		l$uniq_hosts = result[index]["apps.hits"]$unique;
+	                     		Log::write(LOG, l);
+	                     		}
+	                     	}]);
 	}
 
 function do_metric(id: conn_id, hostname: string, size: count)
 	{
-	if ( /youtube\.com$/ in hostname && size > 512*1024 )
+	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
 		{
-		Metrics::add_data("apps.bytes", [$str="youtube"], [$num=size]);
-		Metrics::add_data("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
+		Measurement::add_data("apps.bytes", [$str="youtube"], [$num=size]);
+		Measurement::add_data("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
 		{
-		Metrics::add_data("apps.bytes", [$str="facebook"], [$num=size]);
-		Metrics::add_data("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
+		Measurement::add_data("apps.bytes", [$str="facebook"], [$num=size]);
+		Measurement::add_data("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /\.google\.com$/ in hostname && size > 20 ) 
 		{
-		Metrics::add_data("apps.bytes", [$str="google"], [$num=size]);
-		Metrics::add_data("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
+		Measurement::add_data("apps.bytes", [$str="google"], [$num=size]);
+		Measurement::add_data("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /nflximg\.com$/ in hostname && size > 200*1024 ) 
+	else if ( /\.nflximg\.com$/ in hostname && size > 200*1024 ) 
 		{
-		Metrics::add_data("apps.bytes", [$str="netflix"], [$num=size]);
-		Metrics::add_data("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
+		Measurement::add_data("apps.bytes", [$str="netflix"], [$num=size]);
+		Measurement::add_data("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
 		{
-		Metrics::add_data("apps.bytes", [$str="pandora"], [$num=size]);
-		Metrics::add_data("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
+		Measurement::add_data("apps.bytes", [$str="pandora"], [$num=size]);
+		Measurement::add_data("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
 		}
-	else if ( /gmail\.com$/ in hostname && size > 20 )
+	else if ( /\.gmail\.com$/ in hostname && size > 20 )
 		{
-		Metrics::add_data("apps.bytes", [$str="gmail"], [$num=size]);
-		Metrics::add_data("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
+		Measurement::add_data("apps.bytes", [$str="gmail"], [$num=size]);
+		Measurement::add_data("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
 		}
 }
 
-redef record connection += {
-	resp_hostname: string &optional;
-};
 
 event ssl_established(c: connection)
 	{
diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index e62d370e45..7656ed8d03 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -2,7 +2,7 @@
 ##! toward hosts that have sent low TTL packets.
 ##! It generates a notice when the number of ICMP Time Exceeded 
 ##! messages for a source-destination pair exceeds threshold
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/frameworks/signatures
 @load-sigs ./detect-low-ttls.sig
 
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 7f5f43dbd9..570dbfe6b0 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -5,7 +5,7 @@
 ##!            All the authors of the old scan.bro
 
 @load base/frameworks/notice
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 
 @load base/utils/time
 
@@ -24,7 +24,7 @@ export {
 		## unique ports on a single host over the previous 
 		## :bro:id:`port_scan_interval` time range.
 		Port_Scan,
-		};
+	};
 
 	## Failed connection attempts are tracked over this time interval for the address 
 	## scan detection.  A higher interval will detect slower scanners, but may 
@@ -42,7 +42,7 @@ export {
 	## connections with on a single victim host.
 	const port_scan_threshold = 15 &redef;
 
-	## Custom threholds based on service for address scan.  This is primarily 
+	## Custom thresholds based on service for address scan.  This is primarily 
 	## useful for setting reduced thresholds for specific ports.
 	const addr_scan_custom_thresholds: table[port] of count &redef;
 
@@ -74,7 +74,7 @@ function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 	        $p=to_port(index$str),
 	        $sub=side,
 	        $msg=message,
-	        $identifier=cat(index)]);
+	        $identifier=cat(index$host)]);
 	}
 
 function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
@@ -88,7 +88,7 @@ function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
 	        $dst=to_addr(index$str),
 	        $sub=side,
 	        $msg=message,
-	        $identifier=cat(index)]);
+	        $identifier=cat(index$host)]);
 	}
 
 event bro_init() &priority=5
diff --git a/scripts/policy/protocols/conn/conn-stats-per-host.bro b/scripts/policy/protocols/conn/conn-stats-per-host.bro
index fad2331f44..d537d13b72 100644
--- a/scripts/policy/protocols/conn/conn-stats-per-host.bro
+++ b/scripts/policy/protocols/conn/conn-stats-per-host.bro
@@ -1,6 +1,6 @@
 
 @load base/protocols/conn
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 
 event bro_init() &priority=5
 	{
diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro
index 057e23e088..62ca96ea0a 100644
--- a/scripts/policy/protocols/conn/metrics.bro
+++ b/scripts/policy/protocols/conn/metrics.bro
@@ -1,4 +1,4 @@
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/utils/site
 
 event bro_init() &priority=3
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 11d6ec71a1..59c8525c7e 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -1,6 +1,6 @@
 
 @load base/protocols/ftp
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 
 @load base/utils/time
 
@@ -19,7 +19,7 @@ export {
 
 	## The time period in which the threshold needs to be crossed before
 	## being reset.
-	const bruteforce_measurement_interval = 15mins;
+	const bruteforce_measurement_interval = 15mins &redef;
 }
 
 
@@ -32,7 +32,8 @@ event bro_init()
 	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = 
 	                                        	{
 	                                        	local dur = duration_to_mins_secs(val$end-val$begin);
-	                                        	local message = fmt("%s had %d failed logins on %d FTP servers in %s", index$host, val$num, val$unique, dur);
+	                                        	local plural = val$unique>1 ? "s" : "";
+	                                        	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", index$host, val$num, val$unique, plural, dur);
 	                                        	NOTICE([$note=FTP::Bruteforcing, 
 	                                        	        $src=index$host,
 	                                        	        $msg=message,
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 21164bc126..410d3fde31 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -1,7 +1,7 @@
 ##! SQL injection attack detection in HTTP.
 
 @load base/frameworks/notice
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/protocols/http
 
 module HTTP;
diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro
index ac803ac621..19a3220805 100644
--- a/scripts/policy/protocols/smtp/metrics.bro
+++ b/scripts/policy/protocols/smtp/metrics.bro
@@ -3,7 +3,7 @@
 ##!   "How much mail is being sent from each local mail server per hour?"
 
 @load base/protocols/smtp
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/utils/site
 @load base/utils/directions-and-hosts
 
diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index edf6379bec..44e94eb361 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -2,7 +2,7 @@
 ##! bruteforcing over SSH.
 
 @load base/protocols/ssh
-@load base/frameworks/metrics
+@load base/frameworks/measurement
 @load base/frameworks/notice
 @load base/frameworks/intel
 
@@ -54,8 +54,8 @@ event bro_init()
 	                                          	        $identifier=cat(index$host)]);
 	                                          	# Insert the guesser into the intel framework.
 	                                          	Intel::insert([$host=index$host,
-	                                          		           $meta=[$source="local", 
-	                                          		                  $desc=fmt("Bro observed %0.f apparently failed SSH connections.", val$sum)]]);
+	                                          	               $meta=[$source="local", 
+	                                          	                      $desc=fmt("Bro observed %0.f apparently failed SSH connections.", val$sum)]]);
 	                                          }]);
 	}
 

From 6dc204b385e37fbf89a425df8cba9c95d2c20a6a Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 19 Mar 2013 11:39:58 -0400
Subject: [PATCH 49/70] Checkpoint, don't try running this.  It's broken all
 over the place.

---
 .../base/frameworks/measurement/__load__.bro  |   1 -
 .../base/frameworks/measurement/cluster.bro   | 206 +++++------
 scripts/base/frameworks/measurement/main.bro  | 338 ++++++++----------
 .../frameworks/measurement/non-cluster.bro    |  21 +-
 .../measurement/plugins/__load__.bro          |   1 +
 .../measurement/plugins/average.bro           |  10 +-
 .../frameworks/measurement/plugins/max.bro    |  10 +-
 .../frameworks/measurement/plugins/min.bro    |  10 +-
 .../frameworks/measurement/plugins/sample.bro |  45 +++
 .../measurement/plugins/std-dev.bro           |  10 +-
 .../frameworks/measurement/plugins/sum.bro    |  10 +-
 .../frameworks/measurement/plugins/unique.bro |  13 +-
 .../measurement/plugins/variance.bro          |  16 +-
 scripts/policy/misc/app-metrics.bro           |  40 +--
 14 files changed, 352 insertions(+), 379 deletions(-)
 create mode 100644 scripts/base/frameworks/measurement/plugins/sample.bro

diff --git a/scripts/base/frameworks/measurement/__load__.bro b/scripts/base/frameworks/measurement/__load__.bro
index fc784e1632..c2b77e706a 100644
--- a/scripts/base/frameworks/measurement/__load__.bro
+++ b/scripts/base/frameworks/measurement/__load__.bro
@@ -1,5 +1,4 @@
 @load ./main
-
 @load ./plugins
 
 # The cluster framework must be loaded first.
diff --git a/scripts/base/frameworks/measurement/cluster.bro b/scripts/base/frameworks/measurement/cluster.bro
index 6ccf5bb2f9..9c35b85b32 100644
--- a/scripts/base/frameworks/measurement/cluster.bro
+++ b/scripts/base/frameworks/measurement/cluster.bro
@@ -17,8 +17,8 @@ export {
 	## The percent of the full threshold value that needs to be met 
 	## on a single worker for that worker to send the value to its manager in
 	## order for it to request a global view for that value.  There is no
-	## requirement that the manager requests a global view for the index
-	## since it may opt not to if it requested a global view for the index
+	## requirement that the manager requests a global view for the key
+	## since it may opt not to if it requested a global view for the key
 	## recently.
 	const cluster_request_global_view_percent = 0.2 &redef;
 
@@ -34,75 +34,74 @@ export {
 	const enable_intermediate_updates = T &redef;
 
 	# Event sent by the manager in a cluster to initiate the 
-	# collection of metrics values for a filter.
-	global cluster_filter_request: event(uid: string, id: string, filter_name: string);
+	# collection of metrics values for a measurement.
+	global cluster_measurement_request: event(uid: string, mid: string);
 
 	# Event sent by nodes that are collecting metrics after receiving
-	# a request for the metric filter from the manager.
-	global cluster_filter_response: event(uid: string, id: string, filter_name: string, data: MetricTable, done: bool);
+	# a request for the metric measurement from the manager.
+	global cluster_measurement_response: event(uid: string, mid: string, data: ResultTable, done: bool);
 
 	# This event is sent by the manager in a cluster to initiate the
-	# collection of a single index value from a filter.  It's typically
+	# collection of a single key value from a measurement.  It's typically
 	# used to get intermediate updates before the break interval triggers
 	# to speed detection of a value crossing a threshold.
-	global cluster_index_request: event(uid: string, id: string, filter_name: string, index: Index);
+	global cluster_key_request: event(uid: string, mid: string, key: Key);
 
 	# This event is sent by nodes in response to a 
-	# :bro:id:`Measurement::cluster_index_request` event.
-	global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, val: ResultVal);
+	# :bro:id:`Measurement::cluster_key_request` event.
+	global cluster_key_response: event(uid: string, mid: string, key: Key, result: ResultTable);
 
 	# This is sent by workers to indicate that they crossed the percent of the 
 	# current threshold by the percentage defined globally in 
 	# :bro:id:`Measurement::cluster_request_global_view_percent`
-	global cluster_index_intermediate_response: event(id: string, filter_name: string, index: Measurement::Index);
+	global cluster_key_intermediate_response: event(mid: string, key: Measurement::Key);
 
 	# This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, id: string, filter_name: string, data: MetricTable);
+	global send_data: event(uid: string, id: string, measurement_name: string, data: ResultTable);
 }
 
-
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /Measurement::cluster_(filter_request|index_request)/;
-redef Cluster::worker2manager_events += /Measurement::cluster_(filter_response|index_response|index_intermediate_response)/;
+redef Cluster::manager2worker_events += /Measurement::cluster_(measurement_request|key_request)/;
+redef Cluster::worker2manager_events += /Measurement::cluster_(measurement_response|key_response|key_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
-# This variable is maintained to know what indexes they have recently sent as 
+# This variable is maintained to know what keysthey have recently sent as 
 # intermediate updates so they don't overwhelm their manager. The count that is
 # yielded is the number of times the percentage threshold has been crossed and
 # an intermediate result has been received.
-global recent_global_view_indexes: table[string, string, Index] of count &create_expire=1min &default=0;
+global recent_global_view_keys: table[string, string, Key] of count &create_expire=1min &default=0;
 
 # This is done on all non-manager node types in the event that a metric is 
 # being collected somewhere other than a worker.
-function data_added(filter: Filter, index: Index, val: ResultVal)
+function data_added(measurement: Filter, key: Key, val: Result)
 	{
 	# If an intermediate update for this value was sent recently, don't send
 	# it again.
-	if ( [filter$id, filter$name, index] in recent_global_view_indexes )
+	if ( [measurement$id, measurement$name, key] in recent_global_view_keys )
 		return;
 
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an 
 	# intermediate update.
 	if ( enable_intermediate_updates && 
-	     check_thresholds(filter, index, val, cluster_request_global_view_percent) )
+	     check_thresholds(measurement, key, val, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event Measurement::cluster_index_intermediate_response(filter$id, filter$name, index);
-		++recent_global_view_indexes[filter$id, filter$name, index];
+		event Measurement::cluster_key_intermediate_response(measurement$id, measurement$name, key);
+		++recent_global_view_keys[measurement$id, measurement$name, key];
 		}
 	}
 
-event Measurement::send_data(uid: string, id: string, filter_name: string, data: MetricTable)
+event Measurement::send_data(uid: string, id: string, data: ResultTable)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 	
-	local local_data: MetricTable;
+	local local_data: ResultTable;
 	local num_added = 0;
-	for ( index in data )
+	for ( key in data )
 		{
-		local_data[index] = data[index];
-		delete data[index];
+		local_data[key] = data[key];
+		delete data[key];
 		
 		# Only send cluster_send_in_groups_of at a time.  Queue another
 		# event to send the next group.
@@ -115,35 +114,35 @@ event Measurement::send_data(uid: string, id: string, filter_name: string, data:
 	if ( |data| == 0 )
 		done = T;
 	
-	event Measurement::cluster_filter_response(uid, id, filter_name, local_data, done);
+	event Measurement::cluster_measurement_response(uid, local_data, done);
 	if ( ! done )
-		event Measurement::send_data(uid, id, filter_name, data);
+		event Measurement::send_data(uid, mid, data);
 	}
 
-event Measurement::cluster_filter_request(uid: string, id: string, filter_name: string)
+event Measurement::cluster_measurement_request(uid: string, mid: string)
 	{
-	#print fmt("WORKER %s: received the cluster_filter_request event for %s.", Cluster::node, id);
+	#print fmt("WORKER %s: received the cluster_measurement_request event for %s.", Cluster::node, id);
 	
-	# Initiate sending all of the data for the requested filter.
-	event Measurement::send_data(uid, id, filter_name, store[id, filter_name]);
+	# Initiate sending all of the data for the requested measurement.
+	event Measurement::send_data(uid, mid, result_store[mid]);
 	
-	# Lookup the actual filter and reset it, the reference to the data
+	# Lookup the actual measurement and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
-	reset(filter_store[id, filter_name]);
+	reset(measurement_store[mid]);
 	}
 	
-event Measurement::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
+event Measurement::cluster_key_request(uid: string, mid: string, key: Key)
 	{
-	if ( [id, filter_name] in store && index in store[id, filter_name] )
+	if ( [mid] in result_store && key in result_store[mid] )
 		{
-		#print fmt("WORKER %s: received the cluster_index_request event for %s=%s.", Cluster::node, index2str(index), data);
-		event Measurement::cluster_index_response(uid, id, filter_name, index, store[id, filter_name][index]);
+		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
+		event Measurement::cluster_key_response(uid, mid, key, result_store[mid][key]);
 		}
 	else
 		{
 		# We need to send an empty response if we don't have the data so that the manager
 		# can know that it heard back from all of the workers.
-		event Measurement::cluster_index_response(uid, id, filter_name, index, [$begin=network_time(), $end=network_time()]);
+		event Measurement::cluster_key_response(uid, mid, key, [$begin=network_time(), $end=network_time()]);
 		}
 	}
 
@@ -153,12 +152,8 @@ event Measurement::cluster_index_request(uid: string, id: string, filter_name: s
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 # This variable is maintained by manager nodes as they collect and aggregate 
-# results.
-global filter_results: table[string, string, string] of MetricTable &read_expire=1min;
-
-# This is maintained by managers so they can know what data they requested and
-# when they requested it.
-global requested_results: table[string] of time = table() &create_expire=5mins;
+# results.  It's index on a uid.
+global measurement_results: table[string] of ResultTable &read_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid 
@@ -168,50 +163,49 @@ global requested_results: table[string] of time = table() &create_expire=5mins;
 global done_with: table[string] of count &read_expire=1min &default=0;
 
 # This variable is maintained by managers to track intermediate responses as 
-# they are getting a global view for a certain index.
-global index_requests: table[string, string, string, Index] of ResultVal &read_expire=1min;
+# they are getting a global view for a certain key. Indexed on a uid.
+global key_requests: table[string] of Result &read_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
-# to too many intermediate updates.  Each metric filter is tracked separately so that 
-# one metric won't overwhelm and degrade other quieter metrics.
-global outstanding_global_views: table[string, string] of count &default=0;
+# to too many intermediate updates.  Each measurement is tracked separately so that 
+# one metric won't overwhelm and degrade other quieter metrics. Indexed on a 
+# measurement id.
+global outstanding_global_views: table[string] of count &default=0;
 
 # Managers handle logging.
-event Measurement::finish_period(filter: Filter)
+event Measurement::finish_period(m: Measurement)
 	{
-	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
+	#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
 	local uid = unique_id("");
 	
-	# Set some tracking variables.
-	requested_results[uid] = network_time();
-	if ( [uid, filter$id, filter$name] in filter_results )
-		delete filter_results[uid, filter$id, filter$name];
-	filter_results[uid, filter$id, filter$name] = table();
+	if ( uid in measurement_results )
+		delete measurement_results[uid];
+	measurement_results[uid] = table();
 	
 	# Request data from peers.
-	event Measurement::cluster_filter_request(uid, filter$id, filter$name);
+	event Measurement::cluster_measurement_request(uid, m$id);
 	# Schedule the next finish_period event.
-	schedule filter$every { Measurement::finish_period(filter) };
+	schedule m$epoch { Measurement::finish_period(m) };
 	}
 
-# This is unlikely to be called often, but it's here in case there are metrics
+# This is unlikely to be called often, but it's here in case there are measurements
 # being collected by managers.
-function data_added(filter: Filter, index: Index, val: ResultVal)
+function data_added(m: Measurement, key: Key, result: Result)
 	{
-	if ( check_thresholds(filter, index, val, 1.0) )
-		threshold_crossed(filter, index, val);
+	#if ( check_thresholds(m, key, val, 1.0) )
+	#	threshold_crossed(m, key, val);
 	}
 	
-event Measurement::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: ResultVal)
+event Measurement::cluster_key_response(uid: string, mid: string, key: Key, result: Result)
 	{
-	#print fmt("%0.6f MANAGER: receiving index data from %s - %s=%s", network_time(), get_event_peer()$descr, index2str(index), val);
+	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), val);
 
 	# We only want to try and do a value merge if there are actually measured datapoints
-	# in the ResultVal.
-	if ( val$num > 0 && [uid, id, filter_name, index] in index_requests )
-		index_requests[uid, id, filter_name, index] = merge_result_vals(index_requests[uid, id, filter_name, index], val);
+	# in the Result.
+	if ( result$num > 0 && uid in key_requests )
+		key_requests[uid] = compose_resultvals(key_requests[uid], result);
 	else
-		index_requests[uid, id, filter_name, index] = val;
+		key_requests[uid] = result;
 
 	# Mark that this worker is done.
 	++done_with[uid];
@@ -219,27 +213,27 @@ event Measurement::cluster_index_response(uid: string, id: string, filter_name:
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		local ir = index_requests[uid, id, filter_name, index];
-		if ( check_thresholds(filter_store[id, filter_name], index, ir, 1.0) )
-			{
-			threshold_crossed(filter_store[id, filter_name], index, ir);
-			}
+		local m = measurement_store[mid];
+		local ir = key_requests[uid];
+		if ( check_thresholds(m, key, ir, 1.0) )
+			threshold_crossed(m, key, ir);
+
 		delete done_with[uid];
-		delete index_requests[uid, id, filter_name, index];
+		delete key_requests[uid];
 		# Check that there is an outstanding view before subtracting.
-		if ( outstanding_global_views[id, filter_name] > 0 )
-			--outstanding_global_views[id, filter_name];
+		if ( outstanding_global_views[mid] > 0 )
+			--outstanding_global_views[mid];
 		}
 	}
 
 # Managers handle intermediate updates here.
-event Measurement::cluster_index_intermediate_response(id: string, filter_name: string, index: Index)
+event Measurement::cluster_key_intermediate_response(mid: string, key: Key)
 	{
-	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
-	#print fmt("MANAGER: requesting index data for %s", index2str(index));
+	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
+	#print fmt("MANAGER: requesting key data for %s", key2str(key));
 
-	if ( [id, filter_name] in outstanding_global_views &&
-	     |outstanding_global_views[id, filter_name]| > max_outstanding_global_views )
+	if ( [mid] in outstanding_global_views &&
+	     |outstanding_global_views[mid]| > max_outstanding_global_views )
 		{
 		# Don't do this intermediate update.  Perhaps at some point in the future 
 		# we will queue and randomly select from these ignored intermediate
@@ -247,38 +241,38 @@ event Measurement::cluster_index_intermediate_response(id: string, filter_name:
 		return;
 		}
 
-	++outstanding_global_views[id, filter_name];
+	++outstanding_global_views[mid];
 
 	local uid = unique_id("");
-	event Measurement::cluster_index_request(uid, id, filter_name, index);
+	event Measurement::cluster_key_request(uid, mid, key);
 	}
 
-event Measurement::cluster_filter_response(uid: string, id: string, filter_name: string, data: MetricTable, done: bool)
+event Measurement::cluster_measurement_response(uid: string, mid: string, data: ResultTable, done: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
-	
+
 	# Mark another worker as being "done" for this uid.
 	if ( done )
 		++done_with[uid];
 
-	local local_data = filter_results[uid, id, filter_name];
-	local filter = filter_store[id, filter_name];
+	local local_data = measurement_results[uid];
+	local m = measurement_store[mid];
 
-	for ( index in data )
+	for ( key in data )
 		{
-		if ( index in local_data )
-			local_data[index] = merge_result_vals(local_data[index], data[index]);
+		if ( key in local_data )
+			local_data[key] = compose_resultvals(local_data[key], data[key]);
 		else
-			local_data[index] = data[index];
+			local_data[key] = data[key];
 
-		# If a filter is done being collected, thresholds for each index
+		# If a measurement is done being collected, thresholds for each key
 		# need to be checked so we're doing it here to avoid doubly iterating 
-		# over each index.
+		# over each key.
 		if ( Cluster::worker_count == done_with[uid] )
 			{
-			if ( check_thresholds(filter, index, local_data[index], 1.0) )
+			if ( check_thresholds(m, key, local_data[key], 1.0) )
 				{
-				threshold_crossed(filter, index, local_data[index]);
+				threshold_crossed(m, key, local_data[key]);
 				}
 			}
 		}
@@ -286,22 +280,14 @@ event Measurement::cluster_filter_response(uid: string, id: string, filter_name:
 	# If the data has been collected from all peers, we are done and ready to finish.
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		local ts = network_time();
-		# Log the time this was initially requested if it's available.
-		if ( uid in requested_results )
-			{
-			ts = requested_results[uid];
-			delete requested_results[uid];
-			}
-		
-		if ( filter?$period_finished )
-			filter$period_finished(ts, filter$id, filter$name, local_data);
+		if ( m?$period_finished )
+			m$period_finished(local_data);
 
 		# Clean up
-		delete filter_results[uid, id, filter_name];
+		delete measurement_results[uid];
 		delete done_with[uid];
-		# Not sure I need to reset the filter on the manager.
-		reset(filter);
+		# Not sure I need to reset the measurement on the manager.
+		reset(m);
 		}
 	}
 
diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
index 3809fb16cc..a7f22ed3b7 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -5,24 +5,16 @@
 module Measurement;
 
 export {
-	## The metrics logging stream identifier.
-	redef enum Log::ID += { LOG };
-	
-	## This is the interval for how often threshold based notices will happen 
-	## after they have already fired.
-	const threshold_crossed_restart_interval = 1hr &redef;
-	
 	## The various calculations are all defined as plugins.
 	type Calculation: enum {
 		PLACEHOLDER
 	};
 
-	## Represents a thing which is having metrics collected for it.  An instance
-	## of this record type and an id together represent a single measurement.
-	type Index: record {
+	## Represents a thing which is having measurement results collected for it.
+	type Key: record {
 		## A non-address related metric or a sub-key for an address based metric.
 		## An example might be successful SSH connections by client IP address
-		## where the client string would be the index value.
+		## where the client string would be the key value.
 		## Another example might be number of HTTP requests to a particular
 		## value in a Host header.  This is an example of a non-host based
 		## metric since multiple IP addresses could respond for the same Host
@@ -44,176 +36,152 @@ export {
 		str:       string      &optional;
 	};
 
-	## Value supplied when a metric is finished.  It contains all
-	## of the measurements collected for the metric.  Most of the
-	## fields are added by calculation plugins.
-	type ResultVal: record {
-		## The time when this result was first started.
+	type Reducer: record {
+		## Data stream identifier for the reducer to attach to.
+		stream:         string;
+
+		## The calculations to perform on the data points.
+		apply:          set[Calculation];
+		
+		## A predicate so that you can decide per key if you would like
+		## to accept the data being inserted.
+		pred:           function(key: Measurement::Key, data: Measurement::DataPoint): bool &optional;
+		
+		## A function to normalize the key.  This can be used to aggregate or
+		## normalize the entire key.
+		normalize_key:  function(key: Measurement::Key): Key &optional;
+	};
+
+	## Value calculated for a data point stream fed into a reducer.
+	## Most of the fields are added by plugins.
+	type Result: record {
+		## The time when the first data point was added to this result value.
 		begin:    time          &log;
 
-		## The time when the last value was added to this result.
+		## The time when the last data point was added to this result value.
 		end:      time          &log;
 
 		## The number of measurements received.
 		num:      count         &log &default=0;
-
-		## A sample of something being measured.  This is helpful in 
-		## some cases for collecting information to do further detection
-		## or better logging for forensic purposes.
-		samples:  vector of string   &optional;
-	};
-	
-	type Measurement: record {
-		## The calculations to perform on the data.
-		apply:          set[Calculation];
-		
-		## A predicate so that you can decide per index if you would like
-		## to accept the data being inserted.
-		pred:           function(index: Measurement::Index, data: Measurement::DataPoint): bool &optional;
-		
-		## A function to normalize the index.  This can be used to aggregate or
-		## normalize the entire index.
-		normalize_func: function(index: Measurement::Index): Index &optional;
-
-		## A number of sample DataPoints to collect.
-		samples:        count &optional;
 	};
 
-
-	type Results: record {
-		begin: time;
-		end:   time;
-		result
-	};
-
-	## Type to store a table of metrics result values.
-	type ResultTable: table[Index] of Results;
+	## Type to store a table of measurement results.  First table is
+	## indexed on the measurement Key and the enclosed table is 
+	## indexed on the data id that the Key was relevant for.
+	type ResultTable: table[Key] of table[string] of Result;
 
 	## Filters define how the data from a metric is aggregated and handled.  
 	## Filters can be used to set how often the measurements are cut 
 	## and logged or how the data within them is aggregated.
-	type Filter: record {
-		## A name for the filter in case multiple filters are being
-		## applied to the same metric.  In most cases the default 
-		## filter name is fine and this field does not need to be set.
-		id:              string;
-
-		## The interval at which this filter should be "broken" and written
-		## to the logging stream.  The counters are also reset to zero at 
+	type Measurement: record {
+		## The interval at which this filter should be "broken" and the
+		## callback called.  The counters are also reset to zero at 
 		## this time so any threshold based detection needs to be set to a 
 		## number that should be expected to happen within this period.
-		every:              interval;
+		epoch:              interval;
 
-		## Optionally provide a function to calculate a value from the ResultVal 
-		## structure which will be used for thresholding.  If no function is 
-		## provided, then in the following order of preference either the 
-		## $unique or the $sum fields will be used.
-		threshold_val_func: function(val: Measurement::ResultVal): count  &optional;
+		## The reducers for the measurement indexed by data id.
+		reducers:           set[Reducer];
+
+		## Optionally provide a function to calculate a value from the Result 
+		## structure which will be used for thresholding.
+		threshold_val:      function(result: Measurement::Result): count  &optional;
 
 		## The threshold value for calling the $threshold_crossed callback.
 		threshold:          count                    &optional;
 		
 		## A series of thresholds for calling the $threshold_crossed callback.
 		threshold_series:   vector of count          &optional;
+
+		## A callback that is called when a threshold is crossed.
+		threshold_crossed:  function(key: Measurement::Key, result: Measurement::Result) &optional;
 		
-		## A callback with the full collection of ResultVals for this filter.
+		## A callback with the full collection of Results for this filter.
 		## It's best to not access any global state outside of the variables
 		## given to the callback because there is no assurance provided as to
 		## where the callback will be executed on clusters.
 		period_finished:    function(data: Measurement::ResultTable) &optional;
-
-		## A callback that is called when a threshold is crossed.
-		threshold_crossed:  function(index: Measurement::Index, val: Measurement::ResultVal) &optional;
 	};
 	
-	## Function to associate a metric filter with a metric ID.
-	## 
-	## id: The metric ID that the filter should be associated with.
-	##
-	## filter: The record representing the filter configuration.
-	global add_filter: function(id: string, filter: Measurement::Filter);
-	
+	## Create a measurement.
+	global create: function(m: Measurement::Measurement);
+
 	## Add data into a metric.  This should be called when
 	## a script has measured some point value and is ready to increment the
 	## counters.
 	##
 	## id: The metric identifier that the data represents.
 	##
-	## index: The metric index that the value is to be added to.
+	## key: The metric key that the value is to be added to.
 	##
-	## increment: How much to increment the counter by.
-	global add_data: function(id: string, index: Measurement::Index, data: Measurement::DataPoint);
+	## data: The data point to send into the stream.
+	global add_data: function(id: string, key: Measurement::Key, data: Measurement::DataPoint);
 
-	## Helper function to represent a :bro:type:`Measurement::Index` value as 
+	## Helper function to represent a :bro:type:`Measurement::Key` value as 
 	## a simple string.
 	## 
-	## index: The metric index that is to be converted into a string.
+	## key: The metric key that is to be converted into a string.
 	##
-	## Returns: A string reprentation of the metric index.
-	global index2str: function(index: Measurement::Index): string;
-	
-	## Event to access metrics records as they are passed to the logging framework.
-	global log_metrics: event(rec: Measurement::Info);
+	## Returns: A string representation of the metric key.
+	global key2str: function(key: Measurement::Key): string;
 	
 }
 
-redef record Filter += {
-	# Internal use only.  The metric that this filter applies to.  The value is automatically set.
-	id: string &optional;
+redef record Reducer += {
+	# Internal use only.  Measurement ID.
+	mid: string &optional;
 };
 
-redef record ResultVal += {
-	# Internal use only.  This is the queue where samples
-	# are maintained since the queue is self managing for
-	# the number of samples requested.
-	sample_queue: Queue::Queue &optional;
-
+redef record Result += {
 	# Internal use only.  Indicates if a simple threshold was already crossed.
 	is_threshold_crossed: bool &default=F;
 
-	# Internal use only.  Current index for threshold series.
+	# Internal use only.  Current key for threshold series.
 	threshold_series_index: count &default=0;
 };
 
-# Store the filters indexed on the metric identifier and filter name.
-global filter_store: table[string, string] of Filter = table();
+redef record Measurement += {
+	# Internal use only (mostly for cluster coherency).
+	id: string &optional;
+};
 
-# This is indexed by metric id and filter name.
-global store: table[string, string] of ResultTable = table();
+# Store of reducers indexed on the data id.
+global reducer_store: table[string] of set[Reducer] = table();
 
-# This is a hook for watching thresholds being crossed.  It is called whenever
-# index values are updated and the new val is given as the `val` argument.
+# Store of results indexed on the measurement id.
+global result_store: table[string] of ResultTable = table();
+
+# Store of measurements indexed on the measurement id.
+global measurement_store: table[string] of Measurement = table();
+
+# This is called whenever
+# key values are updated and the new val is given as the `val` argument.
 # It's only prototyped here because cluster and non-cluster have separate 
 # implementations.
-global data_added: function(filter: Filter, index: Index, val: ResultVal);
+global data_added: function(m: Measurement, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
-global add_to_calculation: hook(filter: Filter, val: double, data: DataPoint, result: ResultVal);
-# Prototype the hook point for plugins to merge Measurements.
-global plugin_merge_measurements: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
+global add_to_reducer: hook(r: Reducer, val: double, data: DataPoint, result: Result);
+# Prototype the hook point for plugins to merge Results.
+global compose_resultvals_hook: hook(result: Result, rv1: Result, rv2: Result);
 
-# Event that is used to "finish" metrics and adapt the metrics
+# Event that is used to "finish" measurements and adapt the measurement
 # framework for clustered or non-clustered usage.
-global finish_period: event(filter: Measurement::Filter);
+global finish_period: event(m: Measurement);
 
-event bro_init() &priority=5
-	{
-	Log::create_stream(Measurement::LOG, [$columns=Info, $ev=log_metrics]);
-	}
-
-function index2str(index: Index): string
+function key2str(key: Key): string
 	{
 	local out = "";
-	if ( index?$host )
-		out = fmt("%shost=%s", out, index$host);
-	if ( index?$str )
-		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", index$str);
-	return fmt("metric_index(%s)", out);
+	if ( key?$host )
+		out = fmt("%shost=%s", out, key$host);
+	if ( key?$str )
+		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", key$str);
+	return fmt("metric_key(%s)", out);
 	}
 
-function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
+function compose_resultvals(rv1: Result, rv2: Result): Result
 	{
-	local result: ResultVal;
+	local result: Result;
 
 	# Merge $begin (take the earliest one)
 	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
@@ -224,16 +192,6 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	# Merge $num
 	result$num = rv1$num + rv2$num;
 
-	hook plugin_merge_measurements(result, rv1, rv2);
-	
-	# Merge $sample_queue
-	if ( rv1?$sample_queue && rv2?$sample_queue )
-		result$sample_queue = Queue::merge(rv1$sample_queue, rv2$sample_queue);
-	else if ( rv1?$sample_queue )
-		result$sample_queue = rv1$sample_queue;
-	else if ( rv2?$sample_queue )
-		result$sample_queue = rv2$sample_queue;
-	
 	# Merge $threshold_series_index
 	result$threshold_series_index = (rv1$threshold_series_index > rv2$threshold_series_index) ? rv1$threshold_series_index : rv2$threshold_series_index;
 
@@ -241,105 +199,103 @@ function merge_result_vals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	if ( rv1$is_threshold_crossed || rv2$is_threshold_crossed )
 		result$is_threshold_crossed = T;
 
+	hook compose_resultvals_hook(result, rv1, rv2);
+
 	return result;
 	}
 	
-function reset(filter: Filter)
+function reset(m: Measurement)
 	{
-	if ( [filter$id, filter$name] in store )
-		delete store[filter$id, filter$name];
+	if ( m$id in result_store )
+		delete result_store[m$id];
 
-	store[filter$id, filter$name] = table();
+	result_store[m$id] = table();
 	}
 
-function add_filter(id: string, filter: Filter)
+function create(m: Measurement)
 	{
-	if ( [id, filter$name] in store )
+	m$id=unique_id("");
+	measurement_store[m$id] = m;
+
+	for ( reducer in m$reducers )
 		{
-		Reporter::warning(fmt("invalid Metric filter (%s): Filter with same name already exists.", filter$name));
-		return;
+		reducer$mid = m$id;
+		if ( reducer$stream !in reducer_store )
+			reducer_store[reducer$stream] = set();
+		add reducer_store[reducer$stream][reducer];
 		}
 
-	if ( ! filter?$id )
-		filter$id = id;
-	
-	filter_store[id, filter$name] = filter;
-	store[id, filter$name] = table();
-
-	schedule filter$every { Measurement::finish_period(filter) };
+	reset(m);
+	schedule m$epoch { Measurement::finish_period(m) };
 	}
 
-function add_data(id: string, index: Index, data: DataPoint)
+function add_data(data_id: string, key: Key, data: DataPoint)
 	{
-	# Try to add the data to all of the defined filters for the metric.
-	for ( [metric_id, filter_id] in filter_store )
+	# Try to add the data to all of the defined reducers.
+	if ( data_id !in reducer_store )
+		return;
+
+	for ( r in reducer_store[data_id] )
 		{
-		local filter = filter_store[metric_id, filter_id];
-		
-		# If this filter has a predicate, run the predicate and skip this
-		# index if the predicate return false.
-		if ( filter?$pred && ! filter$pred(index, data) )
+		# If this reducer has a predicate, run the predicate 
+		# and skip this key if the predicate return false.
+		if ( r?$pred && ! r$pred(key, data) )
 			next;
 		
-		#if ( filter?$normalize_func )
-		#	index = filter$normalize_func(copy(index));
+		if ( r?$normalize_key )
+			key = r$normalize_key(copy(key));
 		
-		local metric_tbl = store[id, filter$name];
-		if ( index !in metric_tbl )
-			metric_tbl[index] = [$begin=network_time(), $end=network_time()];
+		local m = measurement_store[r$mid];
+		local results = result_store[m$id];
+		if ( key !in results )
+			results[key] = table();
+		if ( data_id !in results[key] )
+			results[key][data_id] = [$begin=network_time(), $end=network_time()];
 
-		local result = metric_tbl[index];
+		local result = results[key][data_id];
+		++result$num;
+		# Continually update the $end field.
+		result$end=network_time();
 
 		# If a string was given, fall back to 1.0 as the value.
 		local val = 1.0;
 		if ( data?$num || data?$dbl )
 			val = data?$dbl ? data$dbl : data$num;
 
-		++result$num;
-		# Continually update the $end field.
-		result$end=network_time();
-
-		#if ( filter?$samples && filter$samples > 0 && data?$str )
-		#	{
-		#	if ( ! result?$sample_queue )
-		#		result$sample_queue = Queue::init([$max_len=filter$samples]);
-		#	Queue::push(result$sample_queue, data$str);
-		#	}
-
-		hook add_to_calculation(filter, val, data, result);
-		data_added(filter, index, result);
+		hook add_to_reducer(r, val, data, result);
+		data_added(m, key, result);
 		}
 	}
 
 # This function checks if a threshold has been crossed.  It is also used as a method to implement 
 # mid-break-interval threshold crossing detection for cluster deployments.
-function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_pct: double): bool
+function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct: double): bool
 	{
-	if ( ! (filter?$threshold || filter?$threshold_series) )
-		return;
+	if ( ! (m?$threshold || m?$threshold_series) )
+		return F;
 
 	local watch = 0.0;
-	if ( val?$unique )
-		watch = val$unique;
-	else if ( val?$sum )
-		watch = val$sum;
+	#if ( val?$unique )
+	#	watch = val$unique;
+	#else if ( val?$sum )
+	#	watch = val$sum;
 
-	if ( filter?$threshold_val_func )
-		watch = filter$threshold_val_func(val);
+	if ( m?$threshold_val )
+		watch = m$threshold_val(result);
 
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
 		watch = watch/modify_pct;
 
-	if ( ! val$is_threshold_crossed &&
-	     filter?$threshold && watch >= filter$threshold )
+	if ( ! result$is_threshold_crossed &&
+	     m?$threshold && watch >= m$threshold )
 		{
 		# A default threshold was given and the value crossed it.
 		return T;
 		}
 
-	if ( filter?$threshold_series &&
-	     |filter$threshold_series| >= val$threshold_series_index &&
-	     watch >= filter$threshold_series[val$threshold_series_index] )
+	if ( m?$threshold_series &&
+	     |m$threshold_series| >= result$threshold_series_index &&
+	     watch >= m$threshold_series[result$threshold_series_index] )
 		{
 		# A threshold series was given and the value crossed the next 
 		# value in the series.
@@ -349,19 +305,19 @@ function check_thresholds(filter: Filter, index: Index, val: ResultVal, modify_p
 	return F;
 	}
 
-function threshold_crossed(filter: Filter, index: Index, val: ResultVal)
+function threshold_crossed(m: Measurement, key: Key, result: Result)
 	{
-	if ( ! filter?$threshold_crossed )
+	if ( ! m?$threshold_crossed )
 		return;
 
-	if ( val?$sample_queue )
-		val$samples = Queue::get_str_vector(val$sample_queue);
+	#if ( val?$sample_queue )
+	#	val$samples = Queue::get_str_vector(val$sample_queue);
 
-	filter$threshold_crossed(index, val);
-	val$is_threshold_crossed = T;
+	m$threshold_crossed(key, result);
+	result$is_threshold_crossed = T;
 
 	# Bump up to the next threshold series index if a threshold series is being used.
-	if ( filter?$threshold_series )
-		++val$threshold_series_index;
+	if ( m?$threshold_series )
+		++result$threshold_series_index;
 	}
 
diff --git a/scripts/base/frameworks/measurement/non-cluster.bro b/scripts/base/frameworks/measurement/non-cluster.bro
index 11bb7f16dc..7a0a2a2c3e 100644
--- a/scripts/base/frameworks/measurement/non-cluster.bro
+++ b/scripts/base/frameworks/measurement/non-cluster.bro
@@ -2,20 +2,23 @@
 
 module Measurement;
 
-event Measurement::finish_period(filter: Filter)
+event Measurement::finish_period(m: Measurement)
 	{
-	local data = store[filter$id, filter$name];
-	if ( filter?$period_finished )
-		filter$period_finished(network_time(), filter$id, filter$name, data);
+	if ( m$id in result_store )
+		{
+		local data = result_store[m$id];
+		if ( m?$period_finished )
+			m$period_finished(data);
 
-	reset(filter);
+		reset(m);
+		}
 	
-	schedule filter$every { Measurement::finish_period(filter) };
+	schedule m$epoch { Measurement::finish_period(m) };
 	}
 	
 	
-function data_added(filter: Filter, index: Index, val: ResultVal)
+function data_added(m: Measurement, key: Key, result: Result)
 	{
-	if ( check_thresholds(filter, index, val, 1.0) )
-		threshold_crossed(filter, index, val);
+	if ( check_thresholds(m, key, result, 1.0) )
+		threshold_crossed(m, key, result);
 	}
diff --git a/scripts/base/frameworks/measurement/plugins/__load__.bro b/scripts/base/frameworks/measurement/plugins/__load__.bro
index b708f917d1..0d4c2ed302 100644
--- a/scripts/base/frameworks/measurement/plugins/__load__.bro
+++ b/scripts/base/frameworks/measurement/plugins/__load__.bro
@@ -1,6 +1,7 @@
 @load ./average
 @load ./max
 @load ./min
+@load ./sample
 @load ./std-dev
 @load ./sum
 @load ./unique
diff --git a/scripts/base/frameworks/measurement/plugins/average.bro b/scripts/base/frameworks/measurement/plugins/average.bro
index d3e1bef4d5..629cb2fc7b 100644
--- a/scripts/base/frameworks/measurement/plugins/average.bro
+++ b/scripts/base/frameworks/measurement/plugins/average.bro
@@ -1,5 +1,5 @@
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -7,15 +7,15 @@ export {
 		AVERAGE
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## For numeric data, this calculates the average of all values.
 		average: double &log &optional;
 	};
 }
 
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( AVERAGE in filter$measure )
+	if ( AVERAGE in r$apply )
 		{
 		if ( ! result?$average ) 
 			result$average = val;
@@ -24,7 +24,7 @@ hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: Re
 		}
 	}
 
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
 	{
 	if ( rv1?$average && rv2?$average )
 		result$average = ((rv1$average*rv1$num) + (rv2$average*rv2$num))/(rv1$num+rv2$num);
diff --git a/scripts/base/frameworks/measurement/plugins/max.bro b/scripts/base/frameworks/measurement/plugins/max.bro
index 806713dbd4..5138e3f684 100644
--- a/scripts/base/frameworks/measurement/plugins/max.bro
+++ b/scripts/base/frameworks/measurement/plugins/max.bro
@@ -1,5 +1,5 @@
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -7,15 +7,15 @@ export {
 		MAX
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## For numeric data, this tracks the maximum value given.
 		max:      double        &log &optional;
 	};
 }
 
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( MAX in filter$measure )
+	if ( MAX in r$apply )
 		{
 		if ( ! result?$max ) 
 			result$max = val;
@@ -24,7 +24,7 @@ hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: Re
 		}
 	}
 
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
 	{
 	if ( rv1?$max && rv2?$max )
 		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
diff --git a/scripts/base/frameworks/measurement/plugins/min.bro b/scripts/base/frameworks/measurement/plugins/min.bro
index e0d4003b31..ebdbb39373 100644
--- a/scripts/base/frameworks/measurement/plugins/min.bro
+++ b/scripts/base/frameworks/measurement/plugins/min.bro
@@ -1,5 +1,5 @@
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -7,15 +7,15 @@ export {
 		MIN
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## For numeric data, this tracks the minimum value given.
 		min:      double        &log &optional;
 	};
 }
 
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( MIN in filter$measure )
+	if ( MIN in r$apply )
 		{
 		if ( ! result?$min ) 
 			result$min = val;
@@ -24,7 +24,7 @@ hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: Re
 		}
 	}
 
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
 	{
 	if ( rv1?$min && rv2?$min )
 		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
diff --git a/scripts/base/frameworks/measurement/plugins/sample.bro b/scripts/base/frameworks/measurement/plugins/sample.bro
new file mode 100644
index 0000000000..3edd92ce13
--- /dev/null
+++ b/scripts/base/frameworks/measurement/plugins/sample.bro
@@ -0,0 +1,45 @@
+
+module Measurement;
+
+export {
+	
+	redef record Reducer += {
+		## A number of sample DataPoints to collect.
+		samples: count &default=0;
+	};
+
+	redef record Result += {
+		## A sample of something being measured.  This is helpful in 
+		## some cases for collecting information to do further detection
+		## or better logging for forensic purposes.
+		samples: vector of Measurement::DataPoint &optional;
+	};
+}
+
+redef record Result += {
+	# Internal use only.  This is the queue where samples
+	# are maintained since the queue is self managing for
+	# the number of samples requested.
+	sample_queue: Queue::Queue &optional;
+};
+
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+	{
+	if ( r$samples > 0 )
+		{
+		if ( ! result?$sample_queue )
+			result$sample_queue = Queue::init([$max_len=r$samples]);
+		Queue::push(result$sample_queue, data$str);
+		}
+	}
+
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+	{
+	# Merge $sample_queue
+	if ( rv1?$sample_queue && rv2?$sample_queue )
+		result$sample_queue = Queue::merge(rv1$sample_queue, rv2$sample_queue);
+	else if ( rv1?$sample_queue )
+		result$sample_queue = rv1$sample_queue;
+	else if ( rv2?$sample_queue )
+		result$sample_queue = rv2$sample_queue;
+	}
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/std-dev.bro b/scripts/base/frameworks/measurement/plugins/std-dev.bro
index cbd0db3416..6d13d7fc51 100644
--- a/scripts/base/frameworks/measurement/plugins/std-dev.bro
+++ b/scripts/base/frameworks/measurement/plugins/std-dev.bro
@@ -1,7 +1,7 @@
 @load ./sum
 @load ./variance
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -9,23 +9,23 @@ export {
 		STD_DEV
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## For numeric data, this calculates the standard deviation.
 		std_dev:  double &log &optional;
 	};
 }
 
 # This depends on the variance plugin which uses priority -5
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=-10
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( STD_DEV in filter$measure )
+	if ( STD_DEV in r$apply )
 		{
 		if ( result?$variance )
 			result$std_dev = sqrt(result$variance);
 		}
 	}
 
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result) &priority=-10
 	{
 	if ( rv1?$sum || rv2?$sum )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/sum.bro b/scripts/base/frameworks/measurement/plugins/sum.bro
index 2f615ffb6c..7e8b6ff692 100644
--- a/scripts/base/frameworks/measurement/plugins/sum.bro
+++ b/scripts/base/frameworks/measurement/plugins/sum.bro
@@ -1,5 +1,5 @@
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -8,15 +8,15 @@ export {
 		SUM
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## For numeric data, this tracks the sum of all values.
 		sum:      double        &log &optional;
 	};
 }
 
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( SUM in filter$measure )
+	if ( SUM in r$apply )
 		{
 		if ( ! result?$sum ) 
 			result$sum = 0;
@@ -24,7 +24,7 @@ hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: Re
 		}
 	}
 
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
 	{
 	if ( rv1?$sum || rv2?$sum )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/unique.bro b/scripts/base/frameworks/measurement/plugins/unique.bro
index 66cab47897..4f30206a4e 100644
--- a/scripts/base/frameworks/measurement/plugins/unique.bro
+++ b/scripts/base/frameworks/measurement/plugins/unique.bro
@@ -1,5 +1,5 @@
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -7,14 +7,14 @@ export {
 		UNIQUE
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## If cardinality is being tracked, the number of unique
 		## items is tracked here.
 		unique: count &log &optional;
 	};
 }
 
-redef record ResultVal += {
+redef record Result += {
 	# Internal use only.  This is not meant to be publically available 
 	# because we don't want to trust that we can inspect the values 
 	# since we will like move to a probalistic data structure in the future.
@@ -22,17 +22,18 @@ redef record ResultVal += {
 	unique_vals: set[DataPoint] &optional;
 };
 
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( UNIQUE in filter$measure )
+	if ( UNIQUE in r$apply )
 		{
 		if ( ! result?$unique_vals ) 
 			result$unique_vals=set();
 		add result$unique_vals[data];
+		result$unique = |result$unique_vals|;
 		}
 	}
 
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
 	{
 	if ( rv1?$unique_vals || rv2?$unique_vals )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/variance.bro b/scripts/base/frameworks/measurement/plugins/variance.bro
index df83361c35..07a7293539 100644
--- a/scripts/base/frameworks/measurement/plugins/variance.bro
+++ b/scripts/base/frameworks/measurement/plugins/variance.bro
@@ -1,6 +1,6 @@
 @load ./average
 
-module Metrics;
+module Measurement;
 
 export {
 	redef enum Calculation += { 
@@ -8,13 +8,13 @@ export {
 		VARIANCE
 	};
 
-	redef record ResultVal += {
+	redef record Result += {
 		## For numeric data, this calculates the variance.
 		variance: double &log &optional;
 	};
 }
 
-redef record ResultVal += {
+redef record Result += {
 	# Internal use only.  Used for incrementally calculating variance.
 	prev_avg: double &optional;
 
@@ -22,16 +22,16 @@ redef record ResultVal += {
 	var_s: double &optional;
 };
 
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=5
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
 	{
-	if ( VARIANCE in filter$measure )
+	if ( VARIANCE in r$apply )
 		result$prev_avg = result$average;
 	}
 
 # Reduced priority since this depends on the average
-hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=-5
+hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result) &priority=-5
 	{
-	if ( VARIANCE in filter$measure )
+	if ( VARIANCE in r$apply )
 		{
 		if ( ! result?$var_s )
 			result$var_s = 0.0;
@@ -41,7 +41,7 @@ hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: Re
 	}
 
 # Reduced priority since this depends on the average
-hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-5
+hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result) &priority=-5
 	{
 	if ( rv1?$var_s && rv2?$var_s )
 		{
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index 68deddaa29..d76511fe98 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -16,51 +16,33 @@ export {
 	};
 
 	## The frequency of logging the stats collected by this script.
-	const break_interval = 15mins &redef;
+	const break_interval = 1min &redef;
 }
 
 redef record connection += {
 	resp_hostname: string &optional;
 };
 
-function app_metrics_rollup(index: Measurement::Index, vals: table[string, string] of Measurement::ResultVal) 
-	{
-	local l: Info;
-	l$ts = network_time();
-	for ( [metric_name, filter_name] in vals )
-		{
-		local val = vals[metric_name, filter_name];
-		l$app = index$str;
-		if ( metric_name == "apps.bytes" )
-			l$bytes = double_to_count(floor(val$sum));
-		else if ( metric_name == "apps.hits" )
-			{
-			l$hits = val$num;
-			l$uniq_hosts = val$unique;
-			}
-		}
-	}
 
 event bro_init() &priority=3
 	{
 	Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
 
-	#Measurement::create_index_rollup("AppMeasurement", app_metrics_rollup);
-	#Measurement::add_filter("apps.bytes", [$every=break_interval, $measure=set(Measurement::SUM),    $rollup="AppMeasurement"]);
-	#Measurement::add_filter("apps.hits",  [$every=break_interval, $measure=set(Measurement::UNIQUE), $rollup="AppMeasurement"]);
-
+	local r1: Measurement::Reducer = [$stream="apps.bytes", $apply=set(Measurement::SUM)];
+	local r2: Measurement::Reducer = [$stream="apps.hits",  $apply=set(Measurement::UNIQUE)];
 	Measurement::create([$epoch=break_interval, 
-	                     $measurements=table(["apps.bytes"] = [$apply=set(Measurement::SUM)],
-	                                         ["apps.hits"]  = [$apply=set(Measurement::UNIQUE)]),
-	                     $period_finished(result: Measurement::Results) = 
+	                     $reducers=set(r1, r2),
+	                     $period_finished(data: Measurement::ResultTable) = 
 	                     	{
 	                     	local l: Info;
 	                     	l$ts = network_time();
-	                     	for ( index in result )
+	                     	for ( key in data )
 	                     		{
-	                     		l$bytes      = double_to_count(floor(result[index]["apps.bytes"]$sum));
-	                     		l$hits       = result[index]["apps.hits"]$num;
-	                     		l$uniq_hosts = result[index]["apps.hits"]$unique;
+	                     		local result = data[key];
+	                     		l$app        = key$str;
+	                     		l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
+	                     		l$hits       = result["apps.hits"]$num;
+	                     		l$uniq_hosts = result["apps.hits"]$unique;
 	                     		Log::write(LOG, l);
 	                     		}
 	                     	}]);

From 53f9948b02a0ff53f8bd94c4c79f10db603d9c3a Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 1 Apr 2013 14:16:37 -0400
Subject: [PATCH 50/70] Measurement framework tests all pass now.

---
 .../base/frameworks/measurement/cluster.bro   | 136 ++++++-----
 scripts/base/frameworks/measurement/main.bro  | 228 +++++++++++-------
 .../frameworks/measurement/non-cluster.bro    |  10 +-
 .../measurement/plugins/average.bro           |  14 +-
 .../frameworks/measurement/plugins/max.bro    |  16 +-
 .../frameworks/measurement/plugins/min.bro    |  16 +-
 .../frameworks/measurement/plugins/sample.bro |  14 +-
 .../measurement/plugins/std-dev.bro           |  29 ++-
 .../frameworks/measurement/plugins/sum.bro    |  27 ++-
 .../frameworks/measurement/plugins/unique.bro |  18 +-
 .../measurement/plugins/variance.bro          |  28 ++-
 .../manager-1..stdout                         |   4 +
 .../.stdout                                   |   3 +
 .../manager-1..stdout                         |   1 +
 .../.stdout                                   |   6 +
 .../frameworks/measurement/basic-cluster.bro  |  83 +++++++
 .../base/frameworks/measurement/basic.bro     |  34 +++
 .../cluster-intermediate-update.bro           |  33 ++-
 .../frameworks/measurement/thresholding.bro   |  73 ++++++
 .../base/frameworks/metrics/basic-cluster.bro |  88 -------
 .../scripts/base/frameworks/metrics/basic.bro |  20 --
 .../base/frameworks/metrics/thresholding.bro  |  44 ----
 22 files changed, 544 insertions(+), 381 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.measurement.basic-cluster/manager-1..stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.measurement.basic/.stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.measurement.cluster-intermediate-update/manager-1..stdout
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
 create mode 100644 testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro
 create mode 100644 testing/btest/scripts/base/frameworks/measurement/basic.bro
 rename testing/btest/scripts/base/frameworks/{metrics => measurement}/cluster-intermediate-update.bro (56%)
 create mode 100644 testing/btest/scripts/base/frameworks/measurement/thresholding.bro
 delete mode 100644 testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
 delete mode 100644 testing/btest/scripts/base/frameworks/metrics/basic.bro
 delete mode 100644 testing/btest/scripts/base/frameworks/metrics/thresholding.bro

diff --git a/scripts/base/frameworks/measurement/cluster.bro b/scripts/base/frameworks/measurement/cluster.bro
index 9c35b85b32..481b306417 100644
--- a/scripts/base/frameworks/measurement/cluster.bro
+++ b/scripts/base/frameworks/measurement/cluster.bro
@@ -33,70 +33,78 @@ export {
 	## The goal for this option is also meant to be temporary.
 	const enable_intermediate_updates = T &redef;
 
-	# Event sent by the manager in a cluster to initiate the 
-	# collection of metrics values for a measurement.
+	## Event sent by the manager in a cluster to initiate the 
+	## collection of metrics values for a measurement.
 	global cluster_measurement_request: event(uid: string, mid: string);
 
-	# Event sent by nodes that are collecting metrics after receiving
-	# a request for the metric measurement from the manager.
+	## Event sent by nodes that are collecting metrics after receiving
+	## a request for the metric measurement from the manager.
 	global cluster_measurement_response: event(uid: string, mid: string, data: ResultTable, done: bool);
 
-	# This event is sent by the manager in a cluster to initiate the
-	# collection of a single key value from a measurement.  It's typically
-	# used to get intermediate updates before the break interval triggers
-	# to speed detection of a value crossing a threshold.
+	## This event is sent by the manager in a cluster to initiate the
+	## collection of a single key value from a measurement.  It's typically
+	## used to get intermediate updates before the break interval triggers
+	## to speed detection of a value crossing a threshold.
 	global cluster_key_request: event(uid: string, mid: string, key: Key);
 
-	# This event is sent by nodes in response to a 
-	# :bro:id:`Measurement::cluster_key_request` event.
-	global cluster_key_response: event(uid: string, mid: string, key: Key, result: ResultTable);
+	## This event is sent by nodes in response to a 
+	## :bro:id:`Measurement::cluster_key_request` event.
+	global cluster_key_response: event(uid: string, mid: string, key: Key, result: Result);
 
-	# This is sent by workers to indicate that they crossed the percent of the 
-	# current threshold by the percentage defined globally in 
-	# :bro:id:`Measurement::cluster_request_global_view_percent`
+	## This is sent by workers to indicate that they crossed the percent of the 
+	## current threshold by the percentage defined globally in 
+	## :bro:id:`Measurement::cluster_request_global_view_percent`
 	global cluster_key_intermediate_response: event(mid: string, key: Measurement::Key);
 
-	# This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, id: string, measurement_name: string, data: ResultTable);
+	## This event is scheduled internally on workers to send result chunks.
+	global send_data: event(uid: string, mid: string, data: ResultTable);
 }
 
 # Add events to the cluster framework to make this work.
 redef Cluster::manager2worker_events += /Measurement::cluster_(measurement_request|key_request)/;
+redef Cluster::manager2worker_events += /Measurement::new_measurement/;
 redef Cluster::worker2manager_events += /Measurement::cluster_(measurement_response|key_response|key_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
-# This variable is maintained to know what keysthey have recently sent as 
+# This variable is maintained to know what keys have recently sent as 
 # intermediate updates so they don't overwhelm their manager. The count that is
 # yielded is the number of times the percentage threshold has been crossed and
 # an intermediate result has been received.
-global recent_global_view_keys: table[string, string, Key] of count &create_expire=1min &default=0;
+global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
+
+event bro_init() &priority=-100
+	{
+	# The manager is the only host allowed to track these.
+	measurement_store = table();
+	reducer_store = table();
+	}
 
 # This is done on all non-manager node types in the event that a metric is 
 # being collected somewhere other than a worker.
-function data_added(measurement: Filter, key: Key, val: Result)
+function data_added(m: Measurement, key: Key, result: Result)
 	{
 	# If an intermediate update for this value was sent recently, don't send
 	# it again.
-	if ( [measurement$id, measurement$name, key] in recent_global_view_keys )
+	if ( [m$id, key] in recent_global_view_keys )
 		return;
 
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an 
 	# intermediate update.
 	if ( enable_intermediate_updates && 
-	     check_thresholds(measurement, key, val, cluster_request_global_view_percent) )
+	     check_thresholds(m, key, result, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event Measurement::cluster_key_intermediate_response(measurement$id, measurement$name, key);
-		++recent_global_view_keys[measurement$id, measurement$name, key];
+		event Measurement::cluster_key_intermediate_response(m$id, key);
+		++recent_global_view_keys[m$id, key];
 		}
 	}
 
-event Measurement::send_data(uid: string, id: string, data: ResultTable)
+event Measurement::send_data(uid: string, mid: string, data: ResultTable)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
-	
-	local local_data: ResultTable;
+
+	local local_data: ResultTable = table();
 	local num_added = 0;
 	for ( key in data )
 		{
@@ -114,9 +122,9 @@ event Measurement::send_data(uid: string, id: string, data: ResultTable)
 	if ( |data| == 0 )
 		done = T;
 	
-	event Measurement::cluster_measurement_response(uid, local_data, done);
+	event Measurement::cluster_measurement_response(uid, mid, local_data, done);
 	if ( ! done )
-		event Measurement::send_data(uid, mid, data);
+		schedule 0.01 sec { Measurement::send_data(uid, mid, data) };
 	}
 
 event Measurement::cluster_measurement_request(uid: string, mid: string)
@@ -128,12 +136,13 @@ event Measurement::cluster_measurement_request(uid: string, mid: string)
 	
 	# Lookup the actual measurement and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
-	reset(measurement_store[mid]);
+	if ( mid in measurement_store )
+		reset(measurement_store[mid]);
 	}
 	
 event Measurement::cluster_key_request(uid: string, mid: string, key: Key)
 	{
-	if ( [mid] in result_store && key in result_store[mid] )
+	if ( mid in result_store && key in result_store[mid] )
 		{
 		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
 		event Measurement::cluster_key_response(uid, mid, key, result_store[mid][key]);
@@ -142,7 +151,7 @@ event Measurement::cluster_key_request(uid: string, mid: string, key: Key)
 		{
 		# We need to send an empty response if we don't have the data so that the manager
 		# can know that it heard back from all of the workers.
-		event Measurement::cluster_key_response(uid, mid, key, [$begin=network_time(), $end=network_time()]);
+		event Measurement::cluster_key_response(uid, mid, key, table());
 		}
 	}
 
@@ -152,62 +161,70 @@ event Measurement::cluster_key_request(uid: string, mid: string, key: Key)
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 # This variable is maintained by manager nodes as they collect and aggregate 
-# results.  It's index on a uid.
+# results.  
+# Index on a uid.
 global measurement_results: table[string] of ResultTable &read_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid 
 # matches the number of peer nodes that results should be coming from, the 
 # result is written out and deleted from here.
+# Indexed on a uid.
 # TODO: add an &expire_func in case not all results are received.
 global done_with: table[string] of count &read_expire=1min &default=0;
 
 # This variable is maintained by managers to track intermediate responses as 
-# they are getting a global view for a certain key. Indexed on a uid.
+# they are getting a global view for a certain key. 
+# Indexed on a uid.
 global key_requests: table[string] of Result &read_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
 # to too many intermediate updates.  Each measurement is tracked separately so that 
-# one metric won't overwhelm and degrade other quieter metrics. Indexed on a 
-# measurement id.
+# one won't overwhelm and degrade other quieter measurements. 
+# Indexed on a measurement id.
 global outstanding_global_views: table[string] of count &default=0;
 
+const zero_time = double_to_time(0.0);
 # Managers handle logging.
-event Measurement::finish_period(m: Measurement)
+event Measurement::finish_epoch(m: Measurement)
 	{
-	#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
-	local uid = unique_id("");
-	
-	if ( uid in measurement_results )
-		delete measurement_results[uid];
-	measurement_results[uid] = table();
-	
-	# Request data from peers.
-	event Measurement::cluster_measurement_request(uid, m$id);
-	# Schedule the next finish_period event.
-	schedule m$epoch { Measurement::finish_period(m) };
+	if ( network_time() > zero_time )
+		{
+		#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
+		local uid = unique_id("");
+		
+		if ( uid in measurement_results )
+			delete measurement_results[uid];
+		measurement_results[uid] = table();
+		
+		# Request data from peers.
+		event Measurement::cluster_measurement_request(uid, m$id);
+		}
+
+	# Schedule the next finish_epoch event.
+	schedule m$epoch { Measurement::finish_epoch(m) };
 	}
 
 # This is unlikely to be called often, but it's here in case there are measurements
 # being collected by managers.
 function data_added(m: Measurement, key: Key, result: Result)
 	{
-	#if ( check_thresholds(m, key, val, 1.0) )
-	#	threshold_crossed(m, key, val);
+	if ( check_thresholds(m, key, result, 1.0) )
+		threshold_crossed(m, key, result);
 	}
 	
 event Measurement::cluster_key_response(uid: string, mid: string, key: Key, result: Result)
 	{
-	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), val);
+	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
 
 	# We only want to try and do a value merge if there are actually measured datapoints
 	# in the Result.
-	if ( result$num > 0 && uid in key_requests )
-		key_requests[uid] = compose_resultvals(key_requests[uid], result);
+	if ( uid in key_requests )
+		key_requests[uid] = compose_results(key_requests[uid], result);
 	else
 		key_requests[uid] = result;
 
-	# Mark that this worker is done.
+	# Mark that a worker is done.
 	++done_with[uid];
 
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
@@ -232,7 +249,7 @@ event Measurement::cluster_key_intermediate_response(mid: string, key: Key)
 	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting key data for %s", key2str(key));
 
-	if ( [mid] in outstanding_global_views &&
+	if ( mid in outstanding_global_views &&
 	     |outstanding_global_views[mid]| > max_outstanding_global_views )
 		{
 		# Don't do this intermediate update.  Perhaps at some point in the future 
@@ -261,7 +278,7 @@ event Measurement::cluster_measurement_response(uid: string, mid: string, data:
 	for ( key in data )
 		{
 		if ( key in local_data )
-			local_data[key] = compose_resultvals(local_data[key], data[key]);
+			local_data[key] = compose_results(local_data[key], data[key]);
 		else
 			local_data[key] = data[key];
 
@@ -280,8 +297,8 @@ event Measurement::cluster_measurement_response(uid: string, mid: string, data:
 	# If the data has been collected from all peers, we are done and ready to finish.
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		if ( m?$period_finished )
-			m$period_finished(local_data);
+		if ( m?$epoch_finished )
+			m$epoch_finished(local_data);
 
 		# Clean up
 		delete measurement_results[uid];
@@ -291,4 +308,9 @@ event Measurement::cluster_measurement_response(uid: string, mid: string, data:
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer) &priority=5
+	{
+	send_id(p, "Measurement::measurement_store");
+	send_id(p, "Measurement::reducer_store");
+	}
 @endif
diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
index a7f22ed3b7..a8e2950f5a 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -19,21 +19,21 @@ export {
 		## value in a Host header.  This is an example of a non-host based
 		## metric since multiple IP addresses could respond for the same Host
 		## header value.
-		str:          string &optional;
+		str:  string &optional;
 	
 		## Host is the value to which this metric applies.
-		host:         addr &optional;
-	} &log;
+		host: addr &optional;
+	};
 	
 	## Represents data being added for a single metric data point.
 	## Only supply a single value here at a time.
 	type DataPoint: record {
 		## Count value.
-		num:       count       &optional;
+		num:  count  &optional;
 		## Double value.
-		dbl:       double      &optional;
+		dbl:  double &optional;
 		## String value.
-		str:       string      &optional;
+		str:  string &optional;
 	};
 
 	type Reducer: record {
@@ -45,7 +45,7 @@ export {
 		
 		## A predicate so that you can decide per key if you would like
 		## to accept the data being inserted.
-		pred:           function(key: Measurement::Key, data: Measurement::DataPoint): bool &optional;
+		pred:           function(key: Measurement::Key, point: Measurement::DataPoint): bool &optional;
 		
 		## A function to normalize the key.  This can be used to aggregate or
 		## normalize the entire key.
@@ -54,44 +54,45 @@ export {
 
 	## Value calculated for a data point stream fed into a reducer.
 	## Most of the fields are added by plugins.
-	type Result: record {
+	type ResultVal: record {
 		## The time when the first data point was added to this result value.
-		begin:    time          &log;
+		begin:  time;
 
 		## The time when the last data point was added to this result value.
-		end:      time          &log;
+		end:    time;
 
 		## The number of measurements received.
-		num:      count         &log &default=0;
+		num:    count &default=0;
 	};
 
-	## Type to store a table of measurement results.  First table is
-	## indexed on the measurement Key and the enclosed table is 
-	## indexed on the data id that the Key was relevant for.
-	type ResultTable: table[Key] of table[string] of Result;
+	## Type to store results for multiple reducers.
+	type Result: table[string] of ResultVal;
 
-	## Filters define how the data from a metric is aggregated and handled.  
-	## Filters can be used to set how often the measurements are cut 
-	## and logged or how the data within them is aggregated.
+	## Type to store a table of measurement results indexed by the measurement key.
+	type ResultTable: table[Key] of Result;
+
+	## Measurements represent an aggregation of reducers along with 
+	## mechanisms to handle various situations like the epoch ending
+	## or thresholds being crossed.
 	type Measurement: record {
 		## The interval at which this filter should be "broken" and the
-		## callback called.  The counters are also reset to zero at 
-		## this time so any threshold based detection needs to be set to a 
-		## number that should be expected to happen within this period.
+		## '$epoch_finished' callback called.  The results are also reset 
+		## at this time so any threshold based detection needs to be set to a 
+		## number that should be expected to happen within this epoch.
 		epoch:              interval;
 
 		## The reducers for the measurement indexed by data id.
 		reducers:           set[Reducer];
 
-		## Optionally provide a function to calculate a value from the Result 
+		## Provide a function to calculate a value from the :bro:see:`Result`
 		## structure which will be used for thresholding.
-		threshold_val:      function(result: Measurement::Result): count  &optional;
+		threshold_val:      function(key: Measurement::Key, result: Measurement::Result): count &optional;
 
 		## The threshold value for calling the $threshold_crossed callback.
-		threshold:          count                    &optional;
+		threshold:          count             &optional;
 		
 		## A series of thresholds for calling the $threshold_crossed callback.
-		threshold_series:   vector of count          &optional;
+		threshold_series:   vector of count   &optional;
 
 		## A callback that is called when a threshold is crossed.
 		threshold_crossed:  function(key: Measurement::Key, result: Measurement::Result) &optional;
@@ -100,22 +101,21 @@ export {
 		## It's best to not access any global state outside of the variables
 		## given to the callback because there is no assurance provided as to
 		## where the callback will be executed on clusters.
-		period_finished:    function(data: Measurement::ResultTable) &optional;
+		epoch_finished:    function(rt: Measurement::ResultTable) &optional;
 	};
 	
 	## Create a measurement.
 	global create: function(m: Measurement::Measurement);
 
-	## Add data into a metric.  This should be called when
-	## a script has measured some point value and is ready to increment the
-	## counters.
+	## Add data into a data point stream.  This should be called when
+	## a script has measured some point value.
 	##
-	## id: The metric identifier that the data represents.
+	## id: The stream identifier that the data point represents.
 	##
-	## key: The metric key that the value is to be added to.
+	## key: The measurement key that the value is to be added to.
 	##
-	## data: The data point to send into the stream.
-	global add_data: function(id: string, key: Measurement::Key, data: Measurement::DataPoint);
+	## point: The data point to send into the stream.
+	global add_data: function(id: string, key: Measurement::Key, point: Measurement::DataPoint);
 
 	## Helper function to represent a :bro:type:`Measurement::Key` value as 
 	## a simple string.
@@ -124,15 +124,19 @@ export {
 	##
 	## Returns: A string representation of the metric key.
 	global key2str: function(key: Measurement::Key): string;
-	
+
+	## This event is generated for each new measurement that is created.
+	##
+	## m: The record which describes a measurement.
+	global new_measurement: event(m: Measurement);
 }
 
 redef record Reducer += {
-	# Internal use only.  Measurement ID.
+	# Internal use only.  Provides a reference back to the related Measurement by it's ID.
 	mid: string &optional;
 };
 
-redef record Result += {
+type Thresholding: record {
 	# Internal use only.  Indicates if a simple threshold was already crossed.
 	is_threshold_crossed: bool &default=F;
 
@@ -143,16 +147,22 @@ redef record Result += {
 redef record Measurement += {
 	# Internal use only (mostly for cluster coherency).
 	id: string &optional;
+
+	# Internal use only.  For tracking tresholds per key.
+	threshold_tracker: table[Key] of Thresholding &optional;
 };
 
-# Store of reducers indexed on the data id.
+# Store of measurements indexed on the measurement id.
+global measurement_store: table[string] of Measurement = table();
+
+# Store of reducers indexed on the data point stream id.
 global reducer_store: table[string] of set[Reducer] = table();
 
 # Store of results indexed on the measurement id.
 global result_store: table[string] of ResultTable = table();
 
-# Store of measurements indexed on the measurement id.
-global measurement_store: table[string] of Measurement = table();
+# Store of threshold information.
+global thresholds_store: table[string, Key] of bool = table();
 
 # This is called whenever
 # key values are updated and the new val is given as the `val` argument.
@@ -161,13 +171,15 @@ global measurement_store: table[string] of Measurement = table();
 global data_added: function(m: Measurement, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
-global add_to_reducer: hook(r: Reducer, val: double, data: DataPoint, result: Result);
+global add_to_reducer_hook: hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal);
+# Prototype the hook point for plugins to initialize any result values.
+global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
 # Prototype the hook point for plugins to merge Results.
-global compose_resultvals_hook: hook(result: Result, rv1: Result, rv2: Result);
+global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
 
 # Event that is used to "finish" measurements and adapt the measurement
 # framework for clustered or non-clustered usage.
-global finish_period: event(m: Measurement);
+global finish_epoch: event(m: Measurement);
 
 function key2str(key: Key): string
 	{
@@ -176,12 +188,19 @@ function key2str(key: Key): string
 		out = fmt("%shost=%s", out, key$host);
 	if ( key?$str )
 		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", key$str);
-	return fmt("metric_key(%s)", out);
+	return fmt("measurement_key(%s)", out);
 	}
 
-function compose_resultvals(rv1: Result, rv2: Result): Result
+function init_resultval(r: Reducer): ResultVal
 	{
-	local result: Result;
+	local rv: ResultVal = [$begin=network_time(), $end=network_time()];
+	hook init_resultval_hook(r, rv);
+	return rv;
+	}
+
+function compose_resultvals(rv1: ResultVal, rv2: ResultVal): ResultVal
+	{
+	local result: ResultVal;
 
 	# Merge $begin (take the earliest one)
 	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
@@ -192,18 +211,40 @@ function compose_resultvals(rv1: Result, rv2: Result): Result
 	# Merge $num
 	result$num = rv1$num + rv2$num;
 
-	# Merge $threshold_series_index
-	result$threshold_series_index = (rv1$threshold_series_index > rv2$threshold_series_index) ? rv1$threshold_series_index : rv2$threshold_series_index;
-
-	# Merge $is_threshold_crossed
-	if ( rv1$is_threshold_crossed || rv2$is_threshold_crossed )
-		result$is_threshold_crossed = T;
-
 	hook compose_resultvals_hook(result, rv1, rv2);
 
 	return result;
 	}
+
+function compose_results(r1: Result, r2: Result): Result
+	{
+	local result: Result = table();
+
+	if ( |r1| > |r2| )
+		{
+		for ( data_id in r1 )
+			{
+			if ( data_id in r2 )
+				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
+			else
+				result[data_id] = r1[data_id];
+			}
+		}
+	else
+		{
+		for ( data_id in r2 )
+			{
+			if ( data_id in r1 )
+				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
+			else
+				result[data_id] = r2[data_id];
+			}
+		}
 	
+	return result;
+	}
+
+
 function reset(m: Measurement)
 	{
 	if ( m$id in result_store )
@@ -214,7 +255,10 @@ function reset(m: Measurement)
 
 function create(m: Measurement)
 	{
-	m$id=unique_id("");
+	if ( ! m?$id )
+		m$id=unique_id("");
+	local tmp: table[Key] of Thresholding = table();
+	m$threshold_tracker = tmp;
 	measurement_store[m$id] = m;
 
 	for ( reducer in m$reducers )
@@ -226,20 +270,20 @@ function create(m: Measurement)
 		}
 
 	reset(m);
-	schedule m$epoch { Measurement::finish_period(m) };
+	schedule m$epoch { Measurement::finish_epoch(m) };
 	}
 
-function add_data(data_id: string, key: Key, data: DataPoint)
+function add_data(id: string, key: Key, point: DataPoint)
 	{
 	# Try to add the data to all of the defined reducers.
-	if ( data_id !in reducer_store )
+	if ( id !in reducer_store )
 		return;
 
-	for ( r in reducer_store[data_id] )
+	for ( r in reducer_store[id] )
 		{
 		# If this reducer has a predicate, run the predicate 
 		# and skip this key if the predicate return false.
-		if ( r?$pred && ! r$pred(key, data) )
+		if ( r?$pred && ! r$pred(key, point) )
 			next;
 		
 		if ( r?$normalize_key )
@@ -249,20 +293,21 @@ function add_data(data_id: string, key: Key, data: DataPoint)
 		local results = result_store[m$id];
 		if ( key !in results )
 			results[key] = table();
-		if ( data_id !in results[key] )
-			results[key][data_id] = [$begin=network_time(), $end=network_time()];
+		if ( id !in results[key] )
+			results[key][id] = init_resultval(r);
 
-		local result = results[key][data_id];
-		++result$num;
+		local result = results[key];
+		local result_val = result[id];
+		++result_val$num;
 		# Continually update the $end field.
-		result$end=network_time();
+		result_val$end=network_time();
 
 		# If a string was given, fall back to 1.0 as the value.
 		local val = 1.0;
-		if ( data?$num || data?$dbl )
-			val = data?$dbl ? data$dbl : data$num;
+		if ( point?$num || point?$dbl )
+			val = point?$dbl ? point$dbl : point$num;
 
-		hook add_to_reducer(r, val, data, result);
+		hook add_to_reducer_hook(r, val, point, result_val);
 		data_added(m, key, result);
 		}
 	}
@@ -274,28 +319,37 @@ function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct:
 	if ( ! (m?$threshold || m?$threshold_series) )
 		return F;
 
-	local watch = 0.0;
-	#if ( val?$unique )
-	#	watch = val$unique;
-	#else if ( val?$sum )
-	#	watch = val$sum;
+	if ( key !in m$threshold_tracker )
+		{
+		local tmp: Thresholding;
+		m$threshold_tracker[key] = tmp;
+		}
 
-	if ( m?$threshold_val )
-		watch = m$threshold_val(result);
+	# Add in the extra ResultVals to make threshold_vals easier to write.
+	if ( |m$reducers| != |result| )
+		{
+		for ( reducer in m$reducers )
+			{
+			if ( reducer$stream !in result )
+				result[reducer$stream] = init_resultval(reducer);
+			}
+		}
+
+	local watch = m$threshold_val(key, result);
 
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
-		watch = watch/modify_pct;
+		watch = double_to_count(floor(watch/modify_pct));
 
-	if ( ! result$is_threshold_crossed &&
-	     m?$threshold && watch >= m$threshold )
+	local tt = m$threshold_tracker[key];
+	if ( m?$threshold && ! tt$is_threshold_crossed && watch >= m$threshold )
 		{
-		# A default threshold was given and the value crossed it.
+		# Value crossed the threshold.
 		return T;
 		}
 
 	if ( m?$threshold_series &&
-	     |m$threshold_series| >= result$threshold_series_index &&
-	     watch >= m$threshold_series[result$threshold_series_index] )
+	     |m$threshold_series| >= tt$threshold_series_index &&
+	     watch >= m$threshold_series[tt$threshold_series_index] )
 		{
 		# A threshold series was given and the value crossed the next 
 		# value in the series.
@@ -307,17 +361,29 @@ function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct:
 
 function threshold_crossed(m: Measurement, key: Key, result: Result)
 	{
+	# If there is no callback, there is no point in any of this.
 	if ( ! m?$threshold_crossed )
 		return;
 
 	#if ( val?$sample_queue )
 	#	val$samples = Queue::get_str_vector(val$sample_queue);
 
+	# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
+	if ( |m$reducers| != |result| )
+		{
+		for ( reducer in m$reducers )
+			{
+			if ( reducer$stream !in result )
+				result[reducer$stream] = init_resultval(reducer);
+			}
+		}
+
 	m$threshold_crossed(key, result);
-	result$is_threshold_crossed = T;
+	local tt = m$threshold_tracker[key];
+	tt$is_threshold_crossed = T;
 
 	# Bump up to the next threshold series index if a threshold series is being used.
 	if ( m?$threshold_series )
-		++result$threshold_series_index;
+		++tt$threshold_series_index;
 	}
 
diff --git a/scripts/base/frameworks/measurement/non-cluster.bro b/scripts/base/frameworks/measurement/non-cluster.bro
index 7a0a2a2c3e..35ff9dc935 100644
--- a/scripts/base/frameworks/measurement/non-cluster.bro
+++ b/scripts/base/frameworks/measurement/non-cluster.bro
@@ -2,18 +2,18 @@
 
 module Measurement;
 
-event Measurement::finish_period(m: Measurement)
+event Measurement::finish_epoch(m: Measurement)
 	{
 	if ( m$id in result_store )
 		{
 		local data = result_store[m$id];
-		if ( m?$period_finished )
-			m$period_finished(data);
+		if ( m?$epoch_finished )
+			m$epoch_finished(data);
 
 		reset(m);
 		}
-	
-	schedule m$epoch { Measurement::finish_period(m) };
+
+	schedule m$epoch { Measurement::finish_epoch(m) };
 	}
 	
 	
diff --git a/scripts/base/frameworks/measurement/plugins/average.bro b/scripts/base/frameworks/measurement/plugins/average.bro
index 629cb2fc7b..172e8c788d 100644
--- a/scripts/base/frameworks/measurement/plugins/average.bro
+++ b/scripts/base/frameworks/measurement/plugins/average.bro
@@ -7,24 +7,24 @@ export {
 		AVERAGE
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## For numeric data, this calculates the average of all values.
-		average: double &log &optional;
+		average: double &optional;
 	};
 }
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( AVERAGE in r$apply )
 		{
-		if ( ! result?$average ) 
-			result$average = val;
+		if ( ! rv?$average )
+			rv$average = val;
 		else
-			result$average += (val - result$average) / result$num;
+			rv$average += (val - rv$average) / rv$num;
 		}
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$average && rv2?$average )
 		result$average = ((rv1$average*rv1$num) + (rv2$average*rv2$num))/(rv1$num+rv2$num);
diff --git a/scripts/base/frameworks/measurement/plugins/max.bro b/scripts/base/frameworks/measurement/plugins/max.bro
index 5138e3f684..02b536f849 100644
--- a/scripts/base/frameworks/measurement/plugins/max.bro
+++ b/scripts/base/frameworks/measurement/plugins/max.bro
@@ -7,24 +7,24 @@ export {
 		MAX
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## For numeric data, this tracks the maximum value given.
-		max:      double        &log &optional;
+		max: double &optional;
 	};
 }
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( MAX in r$apply )
 		{
-		if ( ! result?$max ) 
-			result$max = val;
-		else if ( val > result$max )
-			result$max = val;
+		if ( ! rv?$max ) 
+			rv$max = val;
+		else if ( val > rv$max )
+			rv$max = val;
 		}
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$max && rv2?$max )
 		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
diff --git a/scripts/base/frameworks/measurement/plugins/min.bro b/scripts/base/frameworks/measurement/plugins/min.bro
index ebdbb39373..944ee9fcb4 100644
--- a/scripts/base/frameworks/measurement/plugins/min.bro
+++ b/scripts/base/frameworks/measurement/plugins/min.bro
@@ -7,24 +7,24 @@ export {
 		MIN
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## For numeric data, this tracks the minimum value given.
-		min:      double        &log &optional;
+		min: double &optional;
 	};
 }
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( MIN in r$apply )
 		{
-		if ( ! result?$min ) 
-			result$min = val;
-		else if ( val < result$min )
-			result$min = val;
+		if ( ! rv?$min ) 
+			rv$min = val;
+		else if ( val < rv$min )
+			rv$min = val;
 		}
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$min && rv2?$min )
 		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
diff --git a/scripts/base/frameworks/measurement/plugins/sample.bro b/scripts/base/frameworks/measurement/plugins/sample.bro
index 3edd92ce13..e0084e88d1 100644
--- a/scripts/base/frameworks/measurement/plugins/sample.bro
+++ b/scripts/base/frameworks/measurement/plugins/sample.bro
@@ -8,7 +8,7 @@ export {
 		samples: count &default=0;
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## A sample of something being measured.  This is helpful in 
 		## some cases for collecting information to do further detection
 		## or better logging for forensic purposes.
@@ -16,24 +16,24 @@ export {
 	};
 }
 
-redef record Result += {
+redef record ResultVal += {
 	# Internal use only.  This is the queue where samples
 	# are maintained since the queue is self managing for
 	# the number of samples requested.
 	sample_queue: Queue::Queue &optional;
 };
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( r$samples > 0 )
 		{
-		if ( ! result?$sample_queue )
-			result$sample_queue = Queue::init([$max_len=r$samples]);
-		Queue::push(result$sample_queue, data$str);
+		if ( ! rv?$sample_queue )
+			rv$sample_queue = Queue::init([$max_len=r$samples]);
+		Queue::push(rv$sample_queue, data$str);
 		}
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	# Merge $sample_queue
 	if ( rv1?$sample_queue && rv2?$sample_queue )
diff --git a/scripts/base/frameworks/measurement/plugins/std-dev.bro b/scripts/base/frameworks/measurement/plugins/std-dev.bro
index 6d13d7fc51..bcf2cdcb00 100644
--- a/scripts/base/frameworks/measurement/plugins/std-dev.bro
+++ b/scripts/base/frameworks/measurement/plugins/std-dev.bro
@@ -9,28 +9,31 @@ export {
 		STD_DEV
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## For numeric data, this calculates the standard deviation.
-		std_dev:  double &log &optional;
+		std_dev: double &optional;
 	};
 }
 
+function calc_std_dev(rv: ResultVal)
+	{
+	if ( rv?$variance )
+		rv$std_dev = sqrt(rv$variance);
+	}
+
 # This depends on the variance plugin which uses priority -5
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal) &priority=-10
 	{
 	if ( STD_DEV in r$apply )
 		{
-		if ( result?$variance )
-			result$std_dev = sqrt(result$variance);
+		if ( rv?$variance )
+			calc_std_dev(rv);
+		else
+			rv$std_dev = 0.0;
 		}
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result) &priority=-10
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
 	{
-	if ( rv1?$sum || rv2?$sum )
-		{
-		result$sum = rv1?$sum ? rv1$sum : 0;
-		if ( rv2?$sum )
-			result$sum += rv2$sum;
-		}
-	}
\ No newline at end of file
+	calc_std_dev(result);
+	}
diff --git a/scripts/base/frameworks/measurement/plugins/sum.bro b/scripts/base/frameworks/measurement/plugins/sum.bro
index 7e8b6ff692..8f989317d8 100644
--- a/scripts/base/frameworks/measurement/plugins/sum.bro
+++ b/scripts/base/frameworks/measurement/plugins/sum.bro
@@ -8,23 +8,32 @@ export {
 		SUM
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## For numeric data, this tracks the sum of all values.
-		sum:      double        &log &optional;
+		sum: double &default=0.0;
 	};
+
+	type threshold_function: function(key: Measurement::Key, result: Measurement::Result): count;
+	global sum_threshold: function(data_id: string): threshold_function;
 }
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+function sum_threshold(data_id: string): threshold_function
 	{
-	if ( SUM in r$apply )
+	return function(key: Measurement::Key, result: Measurement::Result): count
 		{
-		if ( ! result?$sum ) 
-			result$sum = 0;
-		result$sum += val;
-		}
+		print fmt("data_id: %s", data_id);
+		print result;
+		return double_to_count(result[data_id]$sum);
+		};
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+	{
+	if ( SUM in r$apply )
+		rv$sum += val;
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$sum || rv2?$sum )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/unique.bro b/scripts/base/frameworks/measurement/plugins/unique.bro
index 4f30206a4e..5160f0df91 100644
--- a/scripts/base/frameworks/measurement/plugins/unique.bro
+++ b/scripts/base/frameworks/measurement/plugins/unique.bro
@@ -7,14 +7,14 @@ export {
 		UNIQUE
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## If cardinality is being tracked, the number of unique
 		## items is tracked here.
-		unique: count &log &optional;
+		unique: count &optional;
 	};
 }
 
-redef record Result += {
+redef record ResultVal += {
 	# Internal use only.  This is not meant to be publically available 
 	# because we don't want to trust that we can inspect the values 
 	# since we will like move to a probalistic data structure in the future.
@@ -22,18 +22,18 @@ redef record Result += {
 	unique_vals: set[DataPoint] &optional;
 };
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( UNIQUE in r$apply )
 		{
-		if ( ! result?$unique_vals ) 
-			result$unique_vals=set();
-		add result$unique_vals[data];
-		result$unique = |result$unique_vals|;
+		if ( ! rv?$unique_vals ) 
+			rv$unique_vals=set();
+		add rv$unique_vals[data];
+		rv$unique = |rv$unique_vals|;
 		}
 	}
 
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result)
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$unique_vals || rv2?$unique_vals )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/variance.bro b/scripts/base/frameworks/measurement/plugins/variance.bro
index 07a7293539..dc94f39840 100644
--- a/scripts/base/frameworks/measurement/plugins/variance.bro
+++ b/scripts/base/frameworks/measurement/plugins/variance.bro
@@ -8,40 +8,40 @@ export {
 		VARIANCE
 	};
 
-	redef record Result += {
+	redef record ResultVal += {
 		## For numeric data, this calculates the variance.
-		variance: double &log &optional;
+		variance: double &optional;
 	};
 }
 
-redef record Result += {
+redef record ResultVal += {
 	# Internal use only.  Used for incrementally calculating variance.
 	prev_avg: double &optional;
 
 	# Internal use only.  For calculating incremental variance.
-	var_s: double &optional;
+	var_s: double &default=0.0;
 };
 
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result)
+function calc_variance(rv: ResultVal)
 	{
-	if ( VARIANCE in r$apply )
-		result$prev_avg = result$average;
+	rv$variance = (rv$num > 1) ? rv$var_s/(rv$num-1) : 0.0;
 	}
 
 # Reduced priority since this depends on the average
-hook add_to_reducer(r: Reducer, val: double, data: DataPoint, result: Result) &priority=-5
+hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal) &priority=-5
 	{
 	if ( VARIANCE in r$apply )
 		{
-		if ( ! result?$var_s )
-			result$var_s = 0.0;
-		result$var_s += (val - result$prev_avg) * (val - result$average);
-		result$variance = (val > 0) ? result$var_s/val : 0.0;
+		if ( rv$num > 1 )
+			rv$var_s += ((val - rv$prev_avg) * (val - rv$average));
+
+		calc_variance(rv);
+		rv$prev_avg = rv$average;
 		}
 	}
 
 # Reduced priority since this depends on the average
-hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result) &priority=-5
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-5
 	{
 	if ( rv1?$var_s && rv2?$var_s )
 		{
@@ -62,4 +62,6 @@ hook compose_resultvals_hook(result: Result, rv1: Result, rv2: Result) &priority
 		result$prev_avg = rv1$prev_avg;
 	else if ( rv2?$prev_avg )
 		result$prev_avg = rv2$prev_avg;
+
+	calc_variance(result);
 	}
\ No newline at end of file
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.basic-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.measurement.basic-cluster/manager-1..stdout
new file mode 100644
index 0000000000..ea8904d2e6
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.measurement.basic-cluster/manager-1..stdout
@@ -0,0 +1,4 @@
+Host: 6.5.4.3 - num:2 - sum:6.0 - avg:3.0 - max:5.0 - min:1.0 - var:8.0 - std_dev:2.8 - unique:2
+Host: 10.10.10.10 - num:1 - sum:5.0 - avg:5.0 - max:5.0 - min:5.0 - var:0.0 - std_dev:0.0 - unique:1
+Host: 1.2.3.4 - num:9 - sum:437.0 - avg:48.6 - max:95.0 - min:3.0 - var:758.8 - std_dev:27.5 - unique:8
+Host: 7.2.1.5 - num:2 - sum:145.0 - avg:72.5 - max:91.0 - min:54.0 - var:684.5 - std_dev:26.2 - unique:2
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.basic/.stdout b/testing/btest/Baseline/scripts.base.frameworks.measurement.basic/.stdout
new file mode 100644
index 0000000000..208b6103b7
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.measurement.basic/.stdout
@@ -0,0 +1,3 @@
+Host: 6.5.4.3 - num:1 - sum:2.0 - var:0.0 - avg:2.0 - max:2.0 - min:2.0 - std_dev:0.0 - unique:1
+Host: 1.2.3.4 - num:5 - sum:221.0 - var:1144.2 - avg:44.2 - max:94.0 - min:5.0 - std_dev:33.8 - unique:4
+Host: 7.2.1.5 - num:1 - sum:1.0 - var:0.0 - avg:1.0 - max:1.0 - min:1.0 - std_dev:0.0 - unique:1
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.cluster-intermediate-update/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.measurement.cluster-intermediate-update/manager-1..stdout
new file mode 100644
index 0000000000..2a53389dc3
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.measurement.cluster-intermediate-update/manager-1..stdout
@@ -0,0 +1 @@
+A test metric threshold was crossed with a value of: 100.0
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
new file mode 100644
index 0000000000..09c65c3864
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
@@ -0,0 +1,6 @@
+THRESHOLD_SERIES: hit a threshold series value at 3 for measurement_key(host=1.2.3.4)
+THRESHOLD: hit a threshold value at 6 for measurement_key(host=1.2.3.4)
+THRESHOLD_SERIES: hit a threshold series value at 6 for measurement_key(host=1.2.3.4)
+THRESHOLD: hit a threshold value at 1001 for measurement_key(host=7.2.1.5)
+THRESHOLD_SERIES: hit a threshold series value at 1001 for measurement_key(host=7.2.1.5)
+THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at 55x for measurement_key(host=7.2.1.5)
diff --git a/testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro b/testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro
new file mode 100644
index 0000000000..e2f5e4e7d5
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro
@@ -0,0 +1,83 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait 15
+
+# @TEST-EXEC: btest-diff manager-1/.stdout
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+redef Log::default_rotation_interval = 0secs;
+
+global n = 0;
+
+event bro_init() &priority=5
+	{
+	local r1: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM, Measurement::MIN, Measurement::MAX, Measurement::AVERAGE, Measurement::STD_DEV, Measurement::VARIANCE, Measurement::UNIQUE)];
+	Measurement::create([$epoch=5secs,
+	                     $reducers=set(r1),
+	                     $epoch_finished(rt: Measurement::ResultTable) =
+	                     	{
+	                     	for ( key in rt )
+	                     		{
+	                     		local r = rt[key]["test.metric"];
+	                     		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
+	                     		}
+
+	                     	terminate();
+	                     	}
+	                     ]);
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready_for_data: event();
+redef Cluster::manager2worker_events += /^ready_for_data$/;
+
+event ready_for_data()
+	{
+	if ( Cluster::node == "worker-1" )
+		{
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=34]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
+		Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=1]);
+		Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=54]);
+		}
+	if ( Cluster::node == "worker-2" )
+		{
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=75]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=57]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=52]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=61]);
+		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=95]);
+		Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=5]);
+		Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=91]);
+		Measurement::add_data("test.metric", [$host=10.10.10.10], [$num=5]);
+		}
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+event remote_connection_handshake_done(p: event_peer) &priority=-5
+	{
+	++peer_count;
+	if ( peer_count == 2 )
+		event ready_for_data();
+	}
+
+@endif
diff --git a/testing/btest/scripts/base/frameworks/measurement/basic.bro b/testing/btest/scripts/base/frameworks/measurement/basic.bro
new file mode 100644
index 0000000000..e9dd21e0ef
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/measurement/basic.bro
@@ -0,0 +1,34 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event bro_init() &priority=5
+	{
+	local r1: Measurement::Reducer = [$stream="test.metric", 
+	                                  $apply=set(Measurement::SUM, 
+	                                             Measurement::VARIANCE, 
+	                                             Measurement::AVERAGE, 
+	                                             Measurement::MAX, 
+	                                             Measurement::MIN, 
+	                                             Measurement::STD_DEV,
+	                                             Measurement::UNIQUE)];
+	Measurement::create([$epoch=3secs,
+	                     $reducers=set(r1),
+	                     $epoch_finished(data: Measurement::ResultTable) = 
+	                     	{
+	                     	for ( key in data )
+	                     		{
+	                     		local r = data[key]["test.metric"];
+	                     		print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
+	                     		}
+	                     	}
+		 ]);
+
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=5]);
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=22]);
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=94]);
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
+
+	Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
+	Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
+	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/measurement/cluster-intermediate-update.bro
similarity index 56%
rename from testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
rename to testing/btest/scripts/base/frameworks/measurement/cluster-intermediate-update.bro
index b16645dbe6..56f44db2eb 100644
--- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/measurement/cluster-intermediate-update.bro
@@ -19,14 +19,20 @@ redef Log::default_rotation_interval = 0secs;
 
 event bro_init() &priority=5
 	{
-	Metrics::add_filter("test.metric",
-	                    [$every=1hr,
-	                     $measure=set(Metrics::SUM),
+	local r1: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
+	Measurement::create([$epoch=1hr,
+	                     $reducers=set(r1),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	return double_to_count(result["test.metric"]$sum);
+	                     	},
 	                     $threshold=100,
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	print "A test metric threshold was crossed!";
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);
 	                     	terminate();
-	                     }]);
+	                     	}
+	                    ]);
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -39,13 +45,16 @@ event do_metrics(i: count)
 	# Worker-1 will trigger an intermediate update and then if everything
 	# works correctly, the data from worker-2 will hit the threshold and
 	# should trigger the notice.
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=i]);
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=i]);
 	}
 
-event bro_init()
+event remote_connection_handshake_done(p: event_peer)
 	{
-	if ( Cluster::node == "worker-1" )
-		schedule 2sec { do_metrics(99) };
-	if ( Cluster::node == "worker-2" )
-		event do_metrics(1);
+	if ( p$descr == "manager-1" )
+		{
+		if ( Cluster::node == "worker-1" )
+			schedule 0.1sec { do_metrics(1) };
+		if ( Cluster::node == "worker-2" )
+			schedule 0.5sec { do_metrics(99) };
+		}
 	}
diff --git a/testing/btest/scripts/base/frameworks/measurement/thresholding.bro b/testing/btest/scripts/base/frameworks/measurement/thresholding.bro
new file mode 100644
index 0000000000..d25350930e
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/measurement/thresholding.bro
@@ -0,0 +1,73 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+event bro_init() &priority=5
+	{
+	local r1: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
+	Measurement::create([$epoch=3secs,
+	                     $reducers=set(r1),
+	                     #$threshold_val = Measurement::sum_threshold("test.metric"),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return double_to_count(result["test.metric"]$sum);
+	                     	},
+	                     $threshold=5,
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
+	                     	{
+	                     	local r = result["test.metric"];
+	                     	print fmt("THRESHOLD: hit a threshold value at %.0f for %s", r$sum, Measurement::key2str(key));
+	                     	}
+	                     ]);
+
+	local r2: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
+	Measurement::create([$epoch=3secs,
+	                     $reducers=set(r2),
+	                     #$threshold_val = Measurement::sum_threshold("test.metric"),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return double_to_count(result["test.metric"]$sum); 
+	                     	},
+	                     $threshold_series=vector(3,6,800),
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
+	                     	{
+	                     	local r = result["test.metric"];
+	                     	print fmt("THRESHOLD_SERIES: hit a threshold series value at %.0f for %s", r$sum, Measurement::key2str(key));
+	                     	}
+	                     ]);
+
+	local r3: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
+	local r4: Measurement::Reducer = [$stream="test.metric2", $apply=set(Measurement::SUM)];
+	Measurement::create([$epoch=3secs,
+	                     $reducers=set(r3, r4),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	# Calculate a ratio between sums of two reducers.
+	                     	if ( "test.metric2" in result && "test.metric" in result &&
+	                     	     result["test.metric"]$sum > 0 )
+	                     		return double_to_count(result["test.metric2"]$sum / result["test.metric"]$sum);
+	                     	else
+	                     		return 0;
+	                     	},
+	                     # Looking for metric2 sum to be 5 times the sum of metric
+	                     $threshold=5, 
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	local thold = result["test.metric2"]$sum / result["test.metric"]$sum;
+	                     	print fmt("THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at %.0fx for %s", thold, Measurement::key2str(key));
+	                     	}
+	                     ]);
+
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
+	Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
+	Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
+	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
+	Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=1000]);
+	Measurement::add_data("test.metric2", [$host=7.2.1.5], [$num=10]);
+	Measurement::add_data("test.metric2", [$host=7.2.1.5], [$num=1000]);
+	Measurement::add_data("test.metric2", [$host=7.2.1.5], [$num=54321]);
+
+	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
deleted file mode 100644
index c68a4f7beb..0000000000
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ /dev/null
@@ -1,88 +0,0 @@
-# @TEST-SERIALIZE: comm
-#
-# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 15
-# @TEST-EXEC: btest-diff manager-1/metrics.log
-
-@TEST-START-FILE cluster-layout.bro
-redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["proxy-1"]   = [$node_type=Cluster::PROXY,   $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
-	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
-	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
-};
-@TEST-END-FILE
-
-redef Log::default_rotation_interval = 0secs;
-
-global n = 0;
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("test.metric", 
-	                    [$every=3secs,
-	                     $measure=set(Metrics::SUM, Metrics::MIN, Metrics::MAX, Metrics::AVG, Metrics::STD_DEV, Metrics::VARIANCE, Metrics::UNIQUE),
-	                     $period_finished(ts: time, metric_name: string, filter_name: string, data: Metrics::MetricTable) =
-	                     	{
-	                     	Metrics::write_log(ts, metric_name, filter_name, data);
-	                     	if ( ++n == 3 )
-	                     		{
-	                     		terminate_communication();
-	                     		terminate();
-	                     		}
-	                     	}
-	                     	]);
-	}
-
-event remote_connection_closed(p: event_peer)
-	{
-	terminate();
-	}
-
-global ready_for_data: event();
-
-redef Cluster::manager2worker_events += /ready_for_data/;
-
-@if ( Cluster::local_node_type() == Cluster::WORKER )
-
-event ready_for_data()
-	{
-	if ( Cluster::node == "worker-1" )
-		{
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=34]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
-		Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=1]);
-		Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=54]);
-		}
-	if ( Cluster::node == "worker-2" )
-		{
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=75]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=57]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=52]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=61]);
-		Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=95]);
-		Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=5]);
-		Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=91]);
-		}
-	}
-
-@endif
-
-@if ( Cluster::local_node_type() == Cluster::MANAGER )
-
-global peer_count = 0;
-
-event remote_connection_handshake_done(p: event_peer)
-	{
-	++peer_count;
-	if ( peer_count == 3 )
-		event ready_for_data();
-	}
-
-@endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic.bro b/testing/btest/scripts/base/frameworks/metrics/basic.bro
deleted file mode 100644
index e665f2ea5c..0000000000
--- a/testing/btest/scripts/base/frameworks/metrics/basic.bro
+++ /dev/null
@@ -1,20 +0,0 @@
-# @TEST-EXEC: bro %INPUT
-# @TEST-EXEC: btest-diff metrics.log
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("test.metric", 
-		[$name="foo-bar",
-		 $every=3secs,
-		 $measure=set(Metrics::SUM, Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
-		 $period_finished=Metrics::write_log]);
-
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=5]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=22]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=94]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
-
-	Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
-	Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
-	}
diff --git a/testing/btest/scripts/base/frameworks/metrics/thresholding.bro b/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
deleted file mode 100644
index f39443fc2a..0000000000
--- a/testing/btest/scripts/base/frameworks/metrics/thresholding.bro
+++ /dev/null
@@ -1,44 +0,0 @@
-# @TEST-EXEC: bro %INPUT
-# @TEST-EXEC: btest-diff .stdout
-
-
-redef enum Notice::Type += {
-	Test_Notice,
-};
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("test.metric", 
-	                    [$name="foobar",
-	                     $every=3secs,
-	                     $measure=set(Metrics::SUM),
-	                     $threshold=5,
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	print fmt("THRESHOLD: hit a threshold value at %.0f for %s", val$sum, Metrics::index2str(index));
-	                     }]);
-
-	Metrics::add_filter("test.metric", 
-	                    [$name="foobar2",
-	                     $every=3secs,
-	                     $measure=set(Metrics::SUM),
-	                     $threshold_series=vector(3,6,800),
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	print fmt("THRESHOLD_SERIES: hit a threshold series value at %.0f for %s", val$sum, Metrics::index2str(index));
-		                 }]);
-	Metrics::add_filter("test.metric", 
-	                    [$every=3secs,
-	                     $measure=set(Metrics::SUM),
-	                     $threshold_func(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	# This causes any data added to be cross the threshold.
-	                     	return T;
-	                     },
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	print fmt("THRESHOLD_FUNC: hit a threshold function value at %.0f for %s", val$sum, Metrics::index2str(index));
-	                     }]);
-
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
-	Metrics::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
-	Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
-	Metrics::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
-	Metrics::add_data("test.metric", [$host=7.2.1.5], [$num=1000]);
-	}

From b477d2b02d80e0d449724e4fb812c36b480f98ab Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 1 Apr 2013 17:04:15 -0400
Subject: [PATCH 51/70] Measurement framework is ready for testing.

 - New, expanded API.
 - Calculations moved into plugins.
 - Scripts using measurement framework ported.
 - Updated the script-land queue implementation to make it more generic.
 -
---
 scripts/base/frameworks/measurement/main.bro  |   8 +-
 .../frameworks/measurement/plugins/sample.bro |   6 +-
 .../frameworks/measurement/plugins/sum.bro    |   6 +
 .../frameworks/measurement/plugins/unique.bro |   2 +-
 scripts/base/utils/queue.bro                  |  71 +++---------
 scripts/policy/misc/app-metrics.bro           |  19 +++-
 .../policy/misc/detect-traceroute/main.bro    |  56 ++++------
 scripts/policy/misc/scan.bro                  | 105 +++++++++---------
 .../protocols/ftp/detect-bruteforcing.bro     |   4 +-
 scripts/policy/protocols/http/detect-sqli.bro |  54 +++++----
 .../protocols/ssh/detect-bruteforcing.bro     |  38 ++++---
 11 files changed, 183 insertions(+), 186 deletions(-)

diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
index a8e2950f5a..f649dbe1f2 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -255,6 +255,11 @@ function reset(m: Measurement)
 
 function create(m: Measurement)
 	{
+	if ( (m?$threshold || m?$threshold_series) && ! m?$threshold_val )
+		{
+		Reporter::error("Measurement given a threshold with no $threshold_val function");
+		}
+
 	if ( ! m?$id )
 		m$id=unique_id("");
 	local tmp: table[Key] of Thresholding = table();
@@ -365,9 +370,6 @@ function threshold_crossed(m: Measurement, key: Key, result: Result)
 	if ( ! m?$threshold_crossed )
 		return;
 
-	#if ( val?$sample_queue )
-	#	val$samples = Queue::get_str_vector(val$sample_queue);
-
 	# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
 	if ( |m$reducers| != |result| )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/sample.bro b/scripts/base/frameworks/measurement/plugins/sample.bro
index e0084e88d1..018b7c9652 100644
--- a/scripts/base/frameworks/measurement/plugins/sample.bro
+++ b/scripts/base/frameworks/measurement/plugins/sample.bro
@@ -1,3 +1,4 @@
+@load base/utils/queue
 
 module Measurement;
 
@@ -29,7 +30,10 @@ hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal
 		{
 		if ( ! rv?$sample_queue )
 			rv$sample_queue = Queue::init([$max_len=r$samples]);
-		Queue::push(rv$sample_queue, data$str);
+		if ( ! rv?$samples )
+			rv$samples = vector();
+		Queue::put(rv$sample_queue, data);
+		Queue::get_vector(rv$sample_queue, rv$samples);
 		}
 	}
 
diff --git a/scripts/base/frameworks/measurement/plugins/sum.bro b/scripts/base/frameworks/measurement/plugins/sum.bro
index 8f989317d8..5a25573870 100644
--- a/scripts/base/frameworks/measurement/plugins/sum.bro
+++ b/scripts/base/frameworks/measurement/plugins/sum.bro
@@ -27,6 +27,12 @@ function sum_threshold(data_id: string): threshold_function
 		};
 	}
 
+hook init_resultval_hook(r: Reducer, rv: ResultVal)
+	{
+	if ( SUM in r$apply && ! rv?$sum )
+		rv$sum = 0;
+	}
+
 hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( SUM in r$apply )
diff --git a/scripts/base/frameworks/measurement/plugins/unique.bro b/scripts/base/frameworks/measurement/plugins/unique.bro
index 5160f0df91..7664663d29 100644
--- a/scripts/base/frameworks/measurement/plugins/unique.bro
+++ b/scripts/base/frameworks/measurement/plugins/unique.bro
@@ -10,7 +10,7 @@ export {
 	redef record ResultVal += {
 		## If cardinality is being tracked, the number of unique
 		## items is tracked here.
-		unique: count &optional;
+		unique: count &default=0;
 	};
 }
 
diff --git a/scripts/base/utils/queue.bro b/scripts/base/utils/queue.bro
index 438529f579..1e7a293e17 100644
--- a/scripts/base/utils/queue.bro
+++ b/scripts/base/utils/queue.bro
@@ -1,4 +1,4 @@
-##! A FIFO string queue.
+##! A FIFO queue.
 
 module Queue;
 
@@ -23,17 +23,17 @@ export {
 
 	## Push a string onto the top of a queue.
 	## 
-	## q: The queue to push the string into.
+	## q: The queue to put the value into.
 	## 
-	## val: The string to push 
-	global push:       function(q: Queue, val: any);
+	## val: The value to insert into the queue. 
+	global put:       function(q: Queue, val: any);
 
 	## Pop a string from the bottom of a queue.
 	##
-	## q: The queue to pop the string from.
+	## q: The queue to get the string from.
 	##
-	## Returns: The string popped from the queue.
-	global pop:        function(q: Queue): any;
+	## Returns: The value gotten from the queue.
+	global get:        function(q: Queue): any;
 
 	## Merge two queue's together.  If any settings are applied 
 	## to the queues, the settings from q1 are used for the new
@@ -53,23 +53,14 @@ export {
 	## Returns: The length of the queue.
 	global len:     function(q: Queue): count;
 	
-	## Get the contents of the queue as a string vector.
+	## Get the contents of the queue as a vector.
 	## 
 	## q: The queue.
 	##
-	## Returns: A :bro:type:`vector of string` containing the 
-	##          current contents of q.
-	global get_str_vector: function(q: Queue): vector of string;
+	## ret: A vector containing the 
+	##      current contents of q as the type of ret.
+	global get_vector: function(q: Queue, ret: vector of any);
 
-	## Get the contents of the queue as a count vector.  Use care
-	## with this function.  If the data put into the queue wasn't 
-	## integers you will get conversion errors.
-	## 
-	## q: The queue.
-	##
-	## Returns: A :bro:type:`vector of count` containing the 
-	##          current contents of q.
-	global get_cnt_vector: function(q: Queue): vector of count;
 }
 
 redef record Queue += {
@@ -96,15 +87,15 @@ function init(s: Settings): Queue
 	return q;
 	}
 
-function push(q: Queue, val: any)
+function put(q: Queue, val: any)
 	{
 	if ( q$settings?$max_len && len(q) >= q$settings$max_len )
-		pop(q);
+		get(q);
 	q$vals[q$top] = val;
 	++q$top;
 	}
 
-function pop(q: Queue): any
+function get(q: Queue): any
 	{
 	local ret = q$vals[q$bottom];
 	delete q$vals[q$bottom];
@@ -120,9 +111,9 @@ function merge(q1: Queue, q2: Queue): Queue
 	for ( ignored_val in q1$vals )
 		{
 		if ( i in q1$vals )
-			push(ret, q1$vals[i]);
+			put(ret, q1$vals[i]);
 		if ( j in q2$vals )
-			push(ret, q2$vals[j]);
+			put(ret, q2$vals[j]);
 		++i;
 		++j;
 		}
@@ -134,9 +125,8 @@ function len(q: Queue): count
 	return |q$vals|;
 	}
 
-function get_str_vector(q: Queue): vector of string
+function get_vector(q: Queue, ret: vector of any)
 	{
-	local ret: vector of string;
 	local i = q$bottom;
 	local j = 0;
 	# Really dumb hack, this is only to provide
@@ -147,32 +137,7 @@ function get_str_vector(q: Queue): vector of string
 		if ( i >= q$top )
 			break;
 
-		ret[j] = cat(q$vals[i]);
+		ret[j] = q$vals[i];
 		++j; ++i;
 		}
-	return ret;
 	}
-
-function get_cnt_vector(q: Queue): vector of count
-	{
-	local ret: vector of count;
-	local i = q$bottom;
-	local j = 0;
-	# Really dumb hack, this is only to provide
-	# the iteration for the correct number of 
-	# values in q$vals.
-	for ( ignored_val in q$vals )
-		{
-		if ( i >= q$top )
-			break;
-
-		# TODO: this is terrible and should be replaced by 
-		#       a more generic version of the various 
-		#       functions to get vectors of values.
-		#       (the way "any" works right now makes this impossible though)
-		ret[j] = to_count(cat(q$vals[i]));
-		++j; ++i;
-		}
-	return ret;
-	}
-
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index d76511fe98..967d5eb88f 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -8,22 +8,28 @@ export {
 	redef enum Log::ID += { LOG };
 
 	type Info: record {
+		## Timestamp when the log line was finished and written.
 		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The name of the "app", like "facebook" or "netflix".
 		app:        string &log;
+		## The number of unique local hosts using the app.
 		uniq_hosts: count  &log;
+		## The number of hits to the app in total.
 		hits:       count  &log;
+		## The total number of bytes received by users of the app.
 		bytes:      count  &log;
 	};
 
 	## The frequency of logging the stats collected by this script.
-	const break_interval = 1min &redef;
+	const break_interval = 15mins &redef;
 }
 
 redef record connection += {
 	resp_hostname: string &optional;
 };
 
-
 event bro_init() &priority=3
 	{
 	Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
@@ -32,10 +38,11 @@ event bro_init() &priority=3
 	local r2: Measurement::Reducer = [$stream="apps.hits",  $apply=set(Measurement::UNIQUE)];
 	Measurement::create([$epoch=break_interval, 
 	                     $reducers=set(r1, r2),
-	                     $period_finished(data: Measurement::ResultTable) = 
+	                     $epoch_finished(data: Measurement::ResultTable) = 
 	                     	{
 	                     	local l: Info;
 	                     	l$ts = network_time();
+	                     	l$ts_delta = break_interval;
 	                     	for ( key in data )
 	                     		{
 	                     		local result = data[key];
@@ -48,7 +55,7 @@ event bro_init() &priority=3
 	                     	}]);
 	}
 
-function do_metric(id: conn_id, hostname: string, size: count)
+function do_measurement(id: conn_id, hostname: string, size: count)
 	{
 	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
 		{
@@ -92,11 +99,11 @@ event ssl_established(c: connection)
 event connection_finished(c: connection)
 	{
 	if ( c?$resp_hostname )
-		do_metric(c$id, c$resp_hostname, c$resp$size);
+		do_measurement(c$id, c$resp_hostname, c$resp$size);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if( rec?$host )
-		do_metric(rec$id, rec$host, rec$response_body_len);
+		do_measurement(rec$id, rec$host, rec$response_body_len);
 	}
diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index 7656ed8d03..1b9f369ca5 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -49,53 +49,45 @@ export {
 	global log_traceroute: event(rec: Traceroute::Info);
 }
 
-# Track hosts that have sent low TTL packets and which hosts they 
-# sent them to.
-global low_ttlers: set[addr, addr] = {} &create_expire=2min &synchronized;
-
-function traceroute_detected(src: addr, dst: addr)
-	{
-	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
-	NOTICE([$note=Traceroute::Detected,
-	        $msg=fmt("%s seems to be running traceroute", src),
-	        $src=src, $dst=dst,
-	        $identifier=cat(src)]);
-	}
-
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
 
-	Metrics::add_filter("traceroute.time_exceeded", 
-	                    [$log=F,
-	                     $every=icmp_time_exceeded_interval,
-	                     $measure=set(Metrics::UNIQUE),
+	local r1: Measurement::Reducer = [$stream="traceroute.time_exceeded", $apply=set(Measurement::UNIQUE)];
+	local r2: Measurement::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(Measurement::SUM)];
+	Measurement::create([$epoch=icmp_time_exceeded_interval,
+	                     $reducers=set(r1, r2),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	# Give a threshold value of zero depending on if the host 
+	                     	# sends a low ttl packet.
+	                     	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
+	                     		return 0;
+	                     	else
+	                     		return result["traceroute.time_exceeded"]$unique;
+	                     	},
 	                     $threshold=icmp_time_exceeded_threshold,
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                     	local parts = split1(index$str, /-/);
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	local parts = split1(key$str, /-/);
 	                     	local src = to_addr(parts[1]);
 	                     	local dst = to_addr(parts[2]);
-	                     	if ( require_low_ttl_packets )
-	                     		{
-	                     		when ( [src, dst] in low_ttlers )
-	                     			{
-	                     			traceroute_detected(src, dst);
-	                     			}
-	                     		}
-	                     	else
-	                     		traceroute_detected(src, dst);
-	                     }]);
+	                     	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
+	                     	NOTICE([$note=Traceroute::Detected,
+	                     	        $msg=fmt("%s seems to be running traceroute", src),
+	                     	        $src=src, $dst=dst,
+	                     	        $identifier=cat(src)]);
+	                     	}]);
 	}
 
 # Low TTL packets are detected with a signature.
 event signature_match(state: signature_state, msg: string, data: string)
 	{
 	if ( state$sig_id == /traceroute-detector.*/ )
-		add low_ttlers[state$conn$id$orig_h, state$conn$id$resp_h];
+		Measurement::add_data("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
 	}
 
 event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
 	{
-	Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
+	Measurement::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
 	}
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 570dbfe6b0..2ea1e9c0fe 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -52,59 +52,64 @@ export {
 }
 
 
-function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
-	{
-	# We don't need to do this if no custom thresholds are defined.
-	if ( |addr_scan_custom_thresholds| == 0 )
-		return F;
-
-	local service = to_port(index$str);
-	return ( service in addr_scan_custom_thresholds &&
-	         val$sum > addr_scan_custom_thresholds[service] );
-	}
-
-function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
-	{
-	local side = Site::is_local_addr(index$host) ? "local" : "remote";
-	local dur = duration_to_mins_secs(val$end-val$begin);
-	local message=fmt("%s scanned at least %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
-
-	NOTICE([$note=Address_Scan,
-	        $src=index$host,
-	        $p=to_port(index$str),
-	        $sub=side,
-	        $msg=message,
-	        $identifier=cat(index$host)]);
-	}
-
-function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
-	{
-	local side = Site::is_local_addr(index$host) ? "local" : "remote";
-	local dur = duration_to_mins_secs(val$end-val$begin);
-	local message = fmt("%s scanned at least %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
-
-	NOTICE([$note=Port_Scan, 
-	        $src=index$host,
-	        $dst=to_addr(index$str),
-	        $sub=side,
-	        $msg=message,
-	        $identifier=cat(index$host)]);
-	}
+#function check_addr_scan_threshold(key: Measurement::Key, val: Measurement::Result): bool
+#	{
+#	# We don't need to do this if no custom thresholds are defined.
+#	if ( |addr_scan_custom_thresholds| == 0 )
+#		return F;
+#
+#	local service = to_port(key$str);
+#	return ( service in addr_scan_custom_thresholds &&
+#	         val$sum > addr_scan_custom_thresholds[service] );
+#	}
 
 event bro_init() &priority=5
 	{
-	# Note: addr scans are trcked similar to:  table[src_ip, port] of set(dst);	
-	Metrics::add_filter("scan.addr.fail", [$every=addr_scan_interval,
-	                                       $measure=set(Metrics::UNIQUE),
-	                                       $threshold_func=check_addr_scan_threshold,
-	                                       $threshold=addr_scan_threshold,
-	                                       $threshold_crossed=addr_scan_threshold_crossed]); 
+	local r1: Measurement::Reducer = [$stream="scan.addr.fail", $apply=set(Measurement::UNIQUE)];
+	Measurement::create([$epoch=addr_scan_interval,
+	                     $reducers=set(r1),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	return double_to_count(result["scan.addr.fail"]$unique);
+	                     	},
+	                     #$threshold_func=check_addr_scan_threshold,
+	                     $threshold=addr_scan_threshold,
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
+	                     	{
+	                     	local r = result["scan.addr.fail"];
+	                     	local side = Site::is_local_addr(key$host) ? "local" : "remote";
+	                     	local dur = duration_to_mins_secs(r$end-r$begin);
+	                     	local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
+	                     	NOTICE([$note=Address_Scan,
+	                     	        $src=key$host,
+	                     	        $p=to_port(key$str),
+	                     	        $sub=side,
+	                     	        $msg=message,
+	                     	        $identifier=cat(key$host)]);
+	                     	}]); 
 
 	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
-	Metrics::add_filter("scan.port.fail", [$every=port_scan_interval,
-	                                       $measure=set(Metrics::UNIQUE),
-	                                       $threshold=port_scan_threshold,
-	                                       $threshold_crossed=port_scan_threshold_crossed]); 
+	local r2: Measurement::Reducer = [$stream="scan.port.fail", $apply=set(Measurement::UNIQUE)];
+	Measurement::create([$epoch=port_scan_interval,
+	                     $reducers=set(r2),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return double_to_count(result["scan.port.fail"]$unique);
+	                     	},
+	                     $threshold=port_scan_threshold,
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
+	                          {
+	                          local r = result["scan.port.fail"];
+	                          local side = Site::is_local_addr(key$host) ? "local" : "remote";
+	                          local dur = duration_to_mins_secs(r$end-r$begin);
+	                          local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
+	                          NOTICE([$note=Port_Scan, 
+	                                  $src=key$host,
+	                                  $dst=to_addr(key$str),
+	                                  $sub=side,
+	                                  $msg=message,
+	                                  $identifier=cat(key$host)]);
+	                          }]); 
 	}
 
 function add_metrics(id: conn_id, reverse: bool)
@@ -145,10 +150,10 @@ function add_metrics(id: conn_id, reverse: bool)
 	#	return F;
 	
 	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
-		Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
+		Measurement::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
 
 	if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
-		Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
+		Measurement::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 
 function is_failed_conn(c: connection): bool
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 59c8525c7e..286cc95979 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -27,9 +27,9 @@ event bro_init()
 	{
 	Metrics::add_filter("ftp.failed_auth", [$every=bruteforce_measurement_interval,
 	                                        $measure=set(Metrics::UNIQUE),
-	                                        $threshold_val_func(val: Metrics::ResultVal) = { return val$num; },
+	                                        $threshold_val_func(val: Metrics::Result) = { return val$num; },
 	                                        $threshold=bruteforce_threshold,
-	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = 
+	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::Result) = 
 	                                        	{
 	                                        	local dur = duration_to_mins_secs(val$end-val$begin);
 	                                        	local plural = val$unique>1 ? "s" : "";
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index 410d3fde31..bb47ec2f47 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -50,11 +50,11 @@ export {
 		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
-function format_sqli_samples(samples: vector of string): string
+function format_sqli_samples(samples: vector of Measurement::DataPoint): string
 	{
 	local ret = "SQL Injection samples\n---------------------";
 	for ( i in samples )
-		ret += "\n" + samples[i];
+		ret += "\n" + samples[i]$str;
 	return ret;
 	}
 
@@ -63,31 +63,41 @@ event bro_init() &priority=3
 	# Add filters to the metrics so that the metrics framework knows how to 
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
-	Metrics::add_filter("http.sqli.attacker",
-	                    [$every=sqli_requests_interval,
-	                     $measure=set(Metrics::SUM),
+	local r1: Measurement::Reducer = [$stream="http.sqli.attacker", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
+	Measurement::create([$epoch=sqli_requests_interval,
+	                     $reducers=set(r1),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return double_to_count(result["http.sqli.attacker"]$sum);
+	                     	},
 	                     $threshold=sqli_requests_threshold,
-	                     $samples=collect_SQLi_samples,
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
+	                     	{
+	                     	local r = result["http.sqli.attacker"];
 	                     	NOTICE([$note=SQL_Injection_Attacker,
 	                     	        $msg="An SQL injection attacker was discovered!",
-	                     	        $email_body_sections=vector(format_sqli_samples(val$samples)),
-	                     	        $src=index$host,
-	                     	        $identifier=cat(index$host)]);
-	                     }]);
+	                     	        $email_body_sections=vector(format_sqli_samples(r$samples)),
+	                     	        $src=key$host,
+	                     	        $identifier=cat(key$host)]);
+	                     	}]);
 
-	Metrics::add_filter("http.sqli.victim",
-	                    [$every=sqli_requests_interval,
-	                     $measure=set(Metrics::SUM),
+	local r2: Measurement::Reducer = [$stream="http.sqli.victim", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
+	Measurement::create([$epoch=sqli_requests_interval,
+	                     $reducers=set(r2),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return double_to_count(result["http.sqli.victim"]$sum);
+	                     	},
 	                     $threshold=sqli_requests_threshold,
-	                     $samples=collect_SQLi_samples,
-	                     $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
+	                     	{
+	                     	local r = result["http.sqli.victim"];
 	                     	NOTICE([$note=SQL_Injection_Victim,
 	                     	        $msg="An SQL injection victim was discovered!",
-	                     	        $email_body_sections=vector(format_sqli_samples(val$samples)),
-	                     	        $src=index$host,
-	                     	        $identifier=cat(index$host)]);
-	                     }]);
+	                     	        $email_body_sections=vector(format_sqli_samples(r$samples)),
+	                     	        $src=key$host,
+	                     	        $identifier=cat(key$host)]);
+	                     	}]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -97,7 +107,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
-		Metrics::add_data("http.sqli.victim",   [$host=c$id$resp_h], [$str=original_URI]);
+		Measurement::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
+		Measurement::add_data("http.sqli.victim",   [$host=c$id$resp_h], [$str=original_URI]);
 		}
 	}
diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index 44e94eb361..cf2d4030fd 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -42,21 +42,27 @@ export {
 
 event bro_init()
 	{
-	Metrics::add_filter("ssh.login.failure", [$name="detect-bruteforcing", $log=F,
-	                                          $every=guessing_timeout,
-	                                          $measure=set(Metrics::SUM),
-	                                          $threshold=password_guesses_limit,
-	                                          $threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
-	                                          	# Generate the notice.
-	                                          	NOTICE([$note=Password_Guessing, 
-	                                          	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %.0f connections).", index$host, val$sum),
-	                                          	        $src=index$host,
-	                                          	        $identifier=cat(index$host)]);
-	                                          	# Insert the guesser into the intel framework.
-	                                          	Intel::insert([$host=index$host,
-	                                          	               $meta=[$source="local", 
-	                                          	                      $desc=fmt("Bro observed %0.f apparently failed SSH connections.", val$sum)]]);
-	                                          }]);
+	local r1: Measurement::Reducer = [$stream="ssh.login.failure", $apply=set(Measurement::SUM)];
+	Measurement::create([$epoch=guessing_timeout,
+	                     $reducers=set(r1),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return double_to_count(result["ssh.login.failure"]$sum);
+	                     	},
+	                     $threshold=password_guesses_limit,
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
+	                     	{
+	                     	local r = result["ssh.login.failure"];
+	                     	# Generate the notice.
+	                     	NOTICE([$note=Password_Guessing, 
+	                     	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
+	                     	        $src=key$host,
+	                     	        $identifier=cat(key$host)]);
+	                     	# Insert the guesser into the intel framework.
+	                     	Intel::insert([$host=key$host,
+	                     	               $meta=[$source="local", 
+	                     	                      $desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]);
+	                     	}]);
 	}
 
 event SSH::heuristic_successful_login(c: connection)
@@ -76,5 +82,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		Metrics::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
+		Measurement::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
 	}

From d11a1dab73fc6c34585c478b08787951293fc582 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 2 Apr 2013 00:15:55 -0400
Subject: [PATCH 52/70] Removed the example metrics scripts. Better real world
 examples exist now.

---
 .../frameworks/metrics/conn-example.bro       | 26 -----------------
 .../frameworks/metrics/http-example.bro       | 29 -------------------
 .../policy/frameworks/metrics/ssl-example.bro | 23 ---------------
 3 files changed, 78 deletions(-)
 delete mode 100644 scripts/policy/frameworks/metrics/conn-example.bro
 delete mode 100644 scripts/policy/frameworks/metrics/http-example.bro
 delete mode 100644 scripts/policy/frameworks/metrics/ssl-example.bro

diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro
deleted file mode 100644
index 3f87ecb283..0000000000
--- a/scripts/policy/frameworks/metrics/conn-example.bro
+++ /dev/null
@@ -1,26 +0,0 @@
-##! An example of using the metrics framework to collect connection metrics 
-##! aggregated into /24 CIDR ranges.
-
-@load base/frameworks/measurement
-@load base/utils/site
-
-event bro_init()
-	{
-	#Metrics::add_filter("conns.originated", [$aggregation_mask=24, $break_interval=1mins]);
-	Metrics::add_filter("conns.originated", [$every=1mins, $measure=set(Metrics::SUM), 
-	                                         $aggregation_table=Site::local_nets_table, 
-	                                         $period_finished=Metrics::write_log]);
-	
-	
-	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter("conns.responded",  [$every=1mins, $measure=set(Metrics::SUM),
-	                                         $aggregation_table=Site::local_nets_table, 
-	                                         $period_finished=Metrics::write_log]);
-
-	}
-
-event connection_established(c: connection)
-	{
-	Metrics::add_data("conns.originated", [$host=c$id$orig_h], [$num=1]);
-	Metrics::add_data("conns.responded",  [$host=c$id$resp_h], [$num=1]);
-	}
diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro
deleted file mode 100644
index d7aa304754..0000000000
--- a/scripts/policy/frameworks/metrics/http-example.bro
+++ /dev/null
@@ -1,29 +0,0 @@
-##! Provides an example of aggregating and limiting collection down to 
-##! only local networks.  Additionally, the status code for the response from
-##! the request is added into the metric.
-
-@load base/frameworks/measurement
-@load base/protocols/http
-@load base/utils/site
-
-event bro_init()
-	{
-	Metrics::add_filter("http.request.by_host_header", 
-	                    [$every=1min, $measure=set(Metrics::SUM), 
-	                     $pred(index: Metrics::Index, data: Metrics::DataPoint) = { return T; return Site::is_local_addr(index$host); },
-	                     $aggregation_mask=24,
-	                     $period_finished=Metrics::write_log]);
-	
-	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter("http.request.by_status_code", [$every=1min, $measure=set(Metrics::SUM),
-	                                                    $aggregation_table=Site::local_nets_table,
-	                                                    $period_finished=Metrics::write_log]);
-	}
-
-event HTTP::log_http(rec: HTTP::Info)
-	{
-	if ( rec?$host )
-		Metrics::add_data("http.request.by_host_header", [$str=rec$host], [$num=1]);
-	if ( rec?$status_code )
-		Metrics::add_data("http.request.by_status_code", [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], [$num=1]);
-	}
diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro
deleted file mode 100644
index 400373c06c..0000000000
--- a/scripts/policy/frameworks/metrics/ssl-example.bro
+++ /dev/null
@@ -1,23 +0,0 @@
-##! Provides an example of using the metrics framework to collect the number
-##! of times a specific server name indicator value is seen in SSL session
-##! establishments.  Names ending in google.com are being filtered out as an
-##! example of the predicate based filtering in metrics filters.
-
-@load base/frameworks/measurement
-@load base/protocols/ssl
-
-event bro_init()
-	{
-	Metrics::add_filter("ssl.by_servername", 
-		[$name="no-google-ssl-servers",
-		 $every=10secs, $measure=set(Metrics::SUM),
-		 $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
-		    return (/google\.com$/ !in index$str); 
-		 }]);
-	}
-
-event SSL::log_ssl(rec: SSL::Info)
-	{
-	if ( rec?$server_name )
-		Metrics::add_data("ssl.by_servername", [$str=rec$server_name], [$num=1]);
-	}

From f1d165956a7e3f61f54a9294d369d30e7e028ea8 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 2 Apr 2013 00:16:56 -0400
Subject: [PATCH 53/70] Fix path compression to include removing "/./".

 - This involved a fix to the FTP scripts that relied on the old behavior.
---
 scripts/base/protocols/ftp/main.bro    | 11 ++++++++---
 scripts/base/utils/paths.bro           |  4 ++--
 scripts/policy/misc/loaded-scripts.bro |  3 ++-
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index 3d7b1fe61a..69e7c331ae 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -56,10 +56,10 @@ export {
 		tags:             set[string] &log &default=set();
 		
 		## Current working directory that this session is in.  By making
-		## the default value '/.', we can indicate that unless something
+		## the default value '.', we can indicate that unless something
 		## more concrete is discovered that the existing but unknown
 		## directory is ok to use.
-		cwd:                string  &default="/.";
+		cwd:                string  &default=".";
 		
 		## Command that is currently waiting for a response.
 		cmdarg:             CmdArg  &optional;
@@ -172,7 +172,12 @@ function ftp_message(s: Info)
 		
 		local arg = s$cmdarg$arg;
 		if ( s$cmdarg$cmd in file_cmds )
-			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), build_path_compressed(s$cwd, arg));
+			{
+			local comp_path = build_path_compressed(s$cwd, arg);
+			if ( s$cwd[0] != "/" )
+				comp_path = cat("/", comp_path);
+			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path);
+			}
 		
 		s$ts=s$cmdarg$ts;
 		s$command=s$cmdarg$cmd;
diff --git a/scripts/base/utils/paths.bro b/scripts/base/utils/paths.bro
index aa083ddf5b..f8ad384ea7 100644
--- a/scripts/base/utils/paths.bro
+++ b/scripts/base/utils/paths.bro
@@ -19,7 +19,7 @@ function extract_path(input: string): string
 	}
 
 ## Compresses a given path by removing '..'s and the parent directory it
-## references and also removing '/'s.
+## references and also removing dual '/'s and extraneous '/./'s.
 ## dir: a path string, either relative or absolute
 ## Returns: a compressed version of the input path
 function compress_path(dir: string): string
@@ -41,7 +41,7 @@ function compress_path(dir: string): string
 		return compress_path(dir);
 		}
 
-	const multislash_sep = /(\/){2,}/;
+	const multislash_sep = /(\/\.?){2,}/;
 	parts = split_all(dir, multislash_sep);
 	for ( i in parts )
 		if ( i % 2 == 0 )
diff --git a/scripts/policy/misc/loaded-scripts.bro b/scripts/policy/misc/loaded-scripts.bro
index 468478e682..516826aa7e 100644
--- a/scripts/policy/misc/loaded-scripts.bro
+++ b/scripts/policy/misc/loaded-scripts.bro
@@ -1,4 +1,5 @@
 ##! Log the loaded scripts.
+@load base/utils/paths
 
 module LoadedScripts;
 
@@ -34,5 +35,5 @@ event bro_init() &priority=5
 
 event bro_script_loaded(path: string, level: count)
 	{
-	Log::write(LoadedScripts::LOG, [$name=cat(depth[level], path)]);
+	Log::write(LoadedScripts::LOG, [$name=cat(depth[level], compress_path(path))]);
 	}
\ No newline at end of file

From 0e3c84e863b285e45e11c937f7b77ace76373c96 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 2 Apr 2013 00:19:06 -0400
Subject: [PATCH 54/70] Fixed the measurement "sample" plugin.

---
 .../frameworks/measurement/plugins/sample.bro | 46 ++++++++++---------
 scripts/policy/protocols/http/detect-sqli.bro |  4 +-
 .../Baseline/scripts.base.utils.queue/output  | 10 ++--
 testing/btest/scripts/base/utils/queue.test   | 34 +++++++-------
 4 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/scripts/base/frameworks/measurement/plugins/sample.bro b/scripts/base/frameworks/measurement/plugins/sample.bro
index 018b7c9652..399f572490 100644
--- a/scripts/base/frameworks/measurement/plugins/sample.bro
+++ b/scripts/base/frameworks/measurement/plugins/sample.bro
@@ -1,3 +1,4 @@
+@load base/frameworks/measurement
 @load base/utils/queue
 
 module Measurement;
@@ -10,40 +11,41 @@ export {
 	};
 
 	redef record ResultVal += {
-		## A sample of something being measured.  This is helpful in 
-		## some cases for collecting information to do further detection
-		## or better logging for forensic purposes.
-		samples: vector of Measurement::DataPoint &optional;
+		# This is the queue where samples
+		# are maintained.  Use the :bro:see:`Measurement::get_samples`
+		## function to get a vector of the samples.
+		samples: Queue::Queue &optional;
 	};
+
+	## Get a vector of sample DataPoint values from a ResultVal.
+	global get_samples: function(rv: ResultVal): vector of DataPoint;
 }
 
-redef record ResultVal += {
-	# Internal use only.  This is the queue where samples
-	# are maintained since the queue is self managing for
-	# the number of samples requested.
-	sample_queue: Queue::Queue &optional;
-};
+function get_samples(rv: ResultVal): vector of DataPoint
+	{
+	local s: vector of DataPoint = vector();
+	if ( rv?$samples )
+		Queue::get_vector(rv$samples, s);
+	return s;
+	}
 
 hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
 	{
 	if ( r$samples > 0 )
 		{
-		if ( ! rv?$sample_queue )
-			rv$sample_queue = Queue::init([$max_len=r$samples]);
 		if ( ! rv?$samples )
-			rv$samples = vector();
-		Queue::put(rv$sample_queue, data);
-		Queue::get_vector(rv$sample_queue, rv$samples);
+			rv$samples = Queue::init([$max_len=r$samples]);
+		Queue::put(rv$samples, data);
 		}
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
-	# Merge $sample_queue
-	if ( rv1?$sample_queue && rv2?$sample_queue )
-		result$sample_queue = Queue::merge(rv1$sample_queue, rv2$sample_queue);
-	else if ( rv1?$sample_queue )
-		result$sample_queue = rv1$sample_queue;
-	else if ( rv2?$sample_queue )
-		result$sample_queue = rv2$sample_queue;
+	# Merge $samples
+	if ( rv1?$samples && rv2?$samples )
+		result$samples = Queue::merge(rv1$samples, rv2$samples);
+	else if ( rv1?$samples )
+		result$samples = rv1$samples;
+	else if ( rv2?$samples )
+		result$samples = rv2$samples;
 	}
\ No newline at end of file
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index bb47ec2f47..f5e15c5505 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -76,7 +76,7 @@ event bro_init() &priority=3
 	                     	local r = result["http.sqli.attacker"];
 	                     	NOTICE([$note=SQL_Injection_Attacker,
 	                     	        $msg="An SQL injection attacker was discovered!",
-	                     	        $email_body_sections=vector(format_sqli_samples(r$samples)),
+	                     	        $email_body_sections=vector(format_sqli_samples(Measurement::get_samples(r))),
 	                     	        $src=key$host,
 	                     	        $identifier=cat(key$host)]);
 	                     	}]);
@@ -94,7 +94,7 @@ event bro_init() &priority=3
 	                     	local r = result["http.sqli.victim"];
 	                     	NOTICE([$note=SQL_Injection_Victim,
 	                     	        $msg="An SQL injection victim was discovered!",
-	                     	        $email_body_sections=vector(format_sqli_samples(r$samples)),
+	                     	        $email_body_sections=vector(format_sqli_samples(Measurement::get_samples(r))),
 	                     	        $src=key$host,
 	                     	        $identifier=cat(key$host)]);
 	                     	}]);
diff --git a/testing/btest/Baseline/scripts.base.utils.queue/output b/testing/btest/Baseline/scripts.base.utils.queue/output
index b878006310..e54dd89f7a 100644
--- a/testing/btest/Baseline/scripts.base.utils.queue/output
+++ b/testing/btest/Baseline/scripts.base.utils.queue/output
@@ -1,9 +1,7 @@
-This is a get_cnt_vector test: 3
-This is a get_cnt_vector test: 4
-This is a get_str_vector test: 3
-This is a get_str_vector test: 4
-Testing pop: 3
-Length after pop: 1
+This is a get_vector test: 3
+This is a get_vector test: 4
+Testing get: 3
+Length after get: 1
 Size of q2: 4
 String queue value: test 1
 String queue value: test 2
diff --git a/testing/btest/scripts/base/utils/queue.test b/testing/btest/scripts/base/utils/queue.test
index 50f541a25f..344ea73f45 100644
--- a/testing/btest/scripts/base/utils/queue.test
+++ b/testing/btest/scripts/base/utils/queue.test
@@ -7,29 +7,27 @@
 event bro_init()
 	{
 	local q = Queue::init([$max_len=2]);
-	Queue::push(q, 1);
-	Queue::push(q, 2);
-	Queue::push(q, 3);
-	Queue::push(q, 4);
-	local test1 =  Queue::get_cnt_vector(q);
+	Queue::put(q, 1);
+	Queue::put(q, 2);
+	Queue::put(q, 3);
+	Queue::put(q, 4);
+	local test1: vector of count = vector();
+	Queue::get_vector(q, test1);
 	for ( i in test1 )
-		print fmt("This is a get_cnt_vector test: %d", test1[i]);
+		print fmt("This is a get_vector test: %d", test1[i]);
 
-	local test2 =  Queue::get_str_vector(q);
-	for ( i in test2 )
-		print fmt("This is a get_str_vector test: %s", test2[i]);
-
-	local test_val = Queue::pop(q);
-	print fmt("Testing pop: %s", test_val);
-	print fmt("Length after pop: %d", Queue::len(q));
+	local test_val = Queue::get(q);
+	print fmt("Testing get: %s", test_val);
+	print fmt("Length after get: %d", Queue::len(q));
 
 	local q2 = Queue::init([]);
-	Queue::push(q2, "test 1");
-	Queue::push(q2, "test 2");
-	Queue::push(q2, "test 2");
-	Queue::push(q2, "test 1");
+	Queue::put(q2, "test 1");
+	Queue::put(q2, "test 2");
+	Queue::put(q2, "test 2");
+	Queue::put(q2, "test 1");
 	print fmt("Size of q2: %d", Queue::len(q2));
-	local test3: vector of string = Queue::get_str_vector(q2);
+	local test3: vector of string = vector();
+	Queue::get_vector(q2, test3);
 	for ( i in test3 )
 		print fmt("String queue value: %s", test3[i]);
 	}
\ No newline at end of file

From 423bf3b3bff4721f75424492c845c64c11403b91 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 2 Apr 2013 00:30:14 -0400
Subject: [PATCH 55/70] Test updates and cleanup.

---
 doc/scripts/DocSourcesList.cmake              |  15 +-
 scripts/base/frameworks/measurement/main.bro  |   6 +-
 .../measurement/plugins/average.bro           |   1 +
 .../frameworks/measurement/plugins/max.bro    |   1 +
 .../frameworks/measurement/plugins/min.bro    |   1 +
 .../measurement/plugins/std-dev.bro           |   2 +-
 .../frameworks/measurement/plugins/sum.bro    |   1 +
 .../frameworks/measurement/plugins/unique.bro |   1 +
 .../measurement/plugins/variance.bro          |   1 +
 scripts/test-all-policy.bro                   |   3 -
 .../canonified_loaded_scripts.log             |  29 ++--
 .../canonified_loaded_scripts.log             | 148 ++++++++++--------
 .../coverage.init-default/missing_loads       |   2 +-
 .../manager-1.metrics.log                     |  12 --
 .../metrics.log                               |  12 --
 .../manager-1..stdout                         |   1 -
 .../manager-1.notice.log                      |  10 --
 .../notice.log                                |  11 --
 .../.stdout                                   |   8 -
 .../manager-1.notice.log                      |  10 +-
 .../manager-1.notice.log                      |  10 +-
 .../notice.log                                |  10 +-
 22 files changed, 130 insertions(+), 165 deletions(-)
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index 4e957d03a0..d4498b2fe3 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -49,6 +49,14 @@ rest_target(${psd} base/frameworks/logging/writers/none.bro)
 rest_target(${psd} base/frameworks/measurement/cluster.bro)
 rest_target(${psd} base/frameworks/measurement/main.bro)
 rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/average.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/max.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/min.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/sample.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/sum.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/unique.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/variance.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -107,6 +115,7 @@ rest_target(${psd} base/utils/queue.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} base/utils/time.bro)
 rest_target(${psd} base/utils/urls.bro)
 rest_target(${psd} policy/frameworks/communication/listen.bro)
 rest_target(${psd} policy/frameworks/control/controllee.bro)
@@ -122,9 +131,6 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/ssl.bro)
 rest_target(${psd} policy/frameworks/intel/where-locations.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
@@ -136,16 +142,17 @@ rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/detect-traceroute/main.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
+rest_target(${psd} policy/misc/scan.bro)
 rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/conn-stats-per-host.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
 rest_target(${psd} policy/protocols/conn/metrics.bro)
-rest_target(${psd} policy/protocols/conn/scan.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
+rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)
diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
index f649dbe1f2..5e33ff7a25 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -1,6 +1,4 @@
-##! The metrics framework provides a way to count and measure data.  
-
-@load base/utils/queue
+##! The measurement framework provides a way to count and measure data.  
 
 module Measurement;
 
@@ -12,7 +10,7 @@ export {
 
 	## Represents a thing which is having measurement results collected for it.
 	type Key: record {
-		## A non-address related metric or a sub-key for an address based metric.
+		## A non-address related measurement or a sub-key for an address based measurement.
 		## An example might be successful SSH connections by client IP address
 		## where the client string would be the key value.
 		## Another example might be number of HTTP requests to a particular
diff --git a/scripts/base/frameworks/measurement/plugins/average.bro b/scripts/base/frameworks/measurement/plugins/average.bro
index 172e8c788d..9a3938640e 100644
--- a/scripts/base/frameworks/measurement/plugins/average.bro
+++ b/scripts/base/frameworks/measurement/plugins/average.bro
@@ -1,3 +1,4 @@
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/base/frameworks/measurement/plugins/max.bro b/scripts/base/frameworks/measurement/plugins/max.bro
index 02b536f849..816d249de3 100644
--- a/scripts/base/frameworks/measurement/plugins/max.bro
+++ b/scripts/base/frameworks/measurement/plugins/max.bro
@@ -1,3 +1,4 @@
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/base/frameworks/measurement/plugins/min.bro b/scripts/base/frameworks/measurement/plugins/min.bro
index 944ee9fcb4..910d2c76d7 100644
--- a/scripts/base/frameworks/measurement/plugins/min.bro
+++ b/scripts/base/frameworks/measurement/plugins/min.bro
@@ -1,3 +1,4 @@
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/base/frameworks/measurement/plugins/std-dev.bro b/scripts/base/frameworks/measurement/plugins/std-dev.bro
index bcf2cdcb00..bfcaa67910 100644
--- a/scripts/base/frameworks/measurement/plugins/std-dev.bro
+++ b/scripts/base/frameworks/measurement/plugins/std-dev.bro
@@ -1,5 +1,5 @@
-@load ./sum
 @load ./variance
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/base/frameworks/measurement/plugins/sum.bro b/scripts/base/frameworks/measurement/plugins/sum.bro
index 5a25573870..2ada26e1d0 100644
--- a/scripts/base/frameworks/measurement/plugins/sum.bro
+++ b/scripts/base/frameworks/measurement/plugins/sum.bro
@@ -1,3 +1,4 @@
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/base/frameworks/measurement/plugins/unique.bro b/scripts/base/frameworks/measurement/plugins/unique.bro
index 7664663d29..f1027157a7 100644
--- a/scripts/base/frameworks/measurement/plugins/unique.bro
+++ b/scripts/base/frameworks/measurement/plugins/unique.bro
@@ -1,3 +1,4 @@
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/base/frameworks/measurement/plugins/variance.bro b/scripts/base/frameworks/measurement/plugins/variance.bro
index dc94f39840..2868a8a3ad 100644
--- a/scripts/base/frameworks/measurement/plugins/variance.bro
+++ b/scripts/base/frameworks/measurement/plugins/variance.bro
@@ -1,4 +1,5 @@
 @load ./average
+@load base/frameworks/measurement
 
 module Measurement;
 
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index a213031f4c..2fe32a4788 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -24,9 +24,6 @@
 @load frameworks/intel/smtp.bro
 @load frameworks/intel/ssl.bro
 @load frameworks/intel/where-locations.bro
-@load frameworks/metrics/conn-example.bro
-@load frameworks/metrics/http-example.bro
-@load frameworks/metrics/ssl-example.bro
 @load frameworks/software/version-changes.bro
 @load frameworks/software/vulnerable.bro
 @load integration/barnyard2/__load__.bro
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index 41209a4084..d521c151db 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2012-07-20-14-34-11
+#open	2013-04-02-04-24-03
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -14,20 +14,21 @@ scripts/base/init-bare.bro
   build/src/base/reporter.bif.bro
   build/src/base/event.bif.bro
   scripts/base/frameworks/logging/__load__.bro
-    scripts/base/frameworks/logging/./main.bro
+    scripts/base/frameworks/logging/main.bro
       build/src/base/logging.bif.bro
-    scripts/base/frameworks/logging/./postprocessors/__load__.bro
-      scripts/base/frameworks/logging/./postprocessors/./scp.bro
-      scripts/base/frameworks/logging/./postprocessors/./sftp.bro
-    scripts/base/frameworks/logging/./writers/ascii.bro
-    scripts/base/frameworks/logging/./writers/dataseries.bro
-    scripts/base/frameworks/logging/./writers/elasticsearch.bro
-    scripts/base/frameworks/logging/./writers/none.bro
+    scripts/base/frameworks/logging/postprocessors/__load__.bro
+      scripts/base/frameworks/logging/postprocessors/scp.bro
+      scripts/base/frameworks/logging/postprocessors/sftp.bro
+    scripts/base/frameworks/logging/writers/ascii.bro
+    scripts/base/frameworks/logging/writers/dataseries.bro
+    scripts/base/frameworks/logging/writers/elasticsearch.bro
+    scripts/base/frameworks/logging/writers/none.bro
   scripts/base/frameworks/input/__load__.bro
-    scripts/base/frameworks/input/./main.bro
+    scripts/base/frameworks/input/main.bro
       build/src/base/input.bif.bro
-    scripts/base/frameworks/input/./readers/ascii.bro
-    scripts/base/frameworks/input/./readers/raw.bro
-    scripts/base/frameworks/input/./readers/benchmark.bro
+    scripts/base/frameworks/input/readers/ascii.bro
+    scripts/base/frameworks/input/readers/raw.bro
+    scripts/base/frameworks/input/readers/benchmark.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2012-07-20-14-34-11
+  scripts/base/utils/paths.bro
+#close	2013-04-02-04-24-03
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 097fc1f2ca..e691a906c2 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-02-11-18-44-43
+#open	2013-04-02-04-22-32
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -14,24 +14,24 @@ scripts/base/init-bare.bro
   build/src/base/reporter.bif.bro
   build/src/base/event.bif.bro
   scripts/base/frameworks/logging/__load__.bro
-    scripts/base/frameworks/logging/./main.bro
+    scripts/base/frameworks/logging/main.bro
       build/src/base/logging.bif.bro
-    scripts/base/frameworks/logging/./postprocessors/__load__.bro
-      scripts/base/frameworks/logging/./postprocessors/./scp.bro
-      scripts/base/frameworks/logging/./postprocessors/./sftp.bro
-    scripts/base/frameworks/logging/./writers/ascii.bro
-    scripts/base/frameworks/logging/./writers/dataseries.bro
-    scripts/base/frameworks/logging/./writers/elasticsearch.bro
-    scripts/base/frameworks/logging/./writers/none.bro
+    scripts/base/frameworks/logging/postprocessors/__load__.bro
+      scripts/base/frameworks/logging/postprocessors/scp.bro
+      scripts/base/frameworks/logging/postprocessors/sftp.bro
+    scripts/base/frameworks/logging/writers/ascii.bro
+    scripts/base/frameworks/logging/writers/dataseries.bro
+    scripts/base/frameworks/logging/writers/elasticsearch.bro
+    scripts/base/frameworks/logging/writers/none.bro
   scripts/base/frameworks/input/__load__.bro
-    scripts/base/frameworks/input/./main.bro
+    scripts/base/frameworks/input/main.bro
       build/src/base/input.bif.bro
-    scripts/base/frameworks/input/./readers/ascii.bro
-    scripts/base/frameworks/input/./readers/raw.bro
-    scripts/base/frameworks/input/./readers/benchmark.bro
+    scripts/base/frameworks/input/readers/ascii.bro
+    scripts/base/frameworks/input/readers/raw.bro
+    scripts/base/frameworks/input/readers/benchmark.bro
 scripts/base/init-default.bro
   scripts/base/utils/site.bro
-    scripts/base/utils/./patterns.bro
+    scripts/base/utils/patterns.bro
   scripts/base/utils/addrs.bro
   scripts/base/utils/conn-ids.bro
   scripts/base/utils/directions-and-hosts.bro
@@ -41,83 +41,93 @@ scripts/base/init-default.bro
   scripts/base/utils/queue.bro
   scripts/base/utils/strings.bro
   scripts/base/utils/thresholds.bro
+  scripts/base/utils/time.bro
   scripts/base/utils/urls.bro
   scripts/base/frameworks/notice/__load__.bro
-    scripts/base/frameworks/notice/./main.bro
-    scripts/base/frameworks/notice/./weird.bro
-    scripts/base/frameworks/notice/./actions/drop.bro
-    scripts/base/frameworks/notice/./actions/email_admin.bro
-    scripts/base/frameworks/notice/./actions/page.bro
-    scripts/base/frameworks/notice/./actions/add-geodata.bro
-    scripts/base/frameworks/notice/./extend-email/hostnames.bro
+    scripts/base/frameworks/notice/main.bro
+    scripts/base/frameworks/notice/weird.bro
+    scripts/base/frameworks/notice/actions/drop.bro
+    scripts/base/frameworks/notice/actions/email_admin.bro
+    scripts/base/frameworks/notice/actions/page.bro
+    scripts/base/frameworks/notice/actions/add-geodata.bro
+    scripts/base/frameworks/notice/extend-email/hostnames.bro
     scripts/base/frameworks/cluster/__load__.bro
-      scripts/base/frameworks/cluster/./main.bro
+      scripts/base/frameworks/cluster/main.bro
         scripts/base/frameworks/control/__load__.bro
-          scripts/base/frameworks/control/./main.bro
-    scripts/base/frameworks/notice/./non-cluster.bro
-    scripts/base/frameworks/notice/./actions/pp-alarms.bro
+          scripts/base/frameworks/control/main.bro
+    scripts/base/frameworks/notice/non-cluster.bro
+    scripts/base/frameworks/notice/actions/pp-alarms.bro
   scripts/base/frameworks/dpd/__load__.bro
-    scripts/base/frameworks/dpd/./main.bro
+    scripts/base/frameworks/dpd/main.bro
   scripts/base/frameworks/signatures/__load__.bro
-    scripts/base/frameworks/signatures/./main.bro
+    scripts/base/frameworks/signatures/main.bro
   scripts/base/frameworks/packet-filter/__load__.bro
-    scripts/base/frameworks/packet-filter/./main.bro
-    scripts/base/frameworks/packet-filter/./netstats.bro
+    scripts/base/frameworks/packet-filter/main.bro
+    scripts/base/frameworks/packet-filter/netstats.bro
   scripts/base/frameworks/software/__load__.bro
-    scripts/base/frameworks/software/./main.bro
+    scripts/base/frameworks/software/main.bro
   scripts/base/frameworks/communication/__load__.bro
-    scripts/base/frameworks/communication/./main.bro
-  scripts/base/frameworks/metrics/__load__.bro
-    scripts/base/frameworks/metrics/./main.bro
-    scripts/base/frameworks/metrics/./non-cluster.bro
+    scripts/base/frameworks/communication/main.bro
+  scripts/base/frameworks/measurement/__load__.bro
+    scripts/base/frameworks/measurement/main.bro
+    scripts/base/frameworks/measurement/plugins/__load__.bro
+      scripts/base/frameworks/measurement/plugins/average.bro
+      scripts/base/frameworks/measurement/plugins/max.bro
+      scripts/base/frameworks/measurement/plugins/min.bro
+      scripts/base/frameworks/measurement/plugins/sample.bro
+      scripts/base/frameworks/measurement/plugins/std-dev.bro
+        scripts/base/frameworks/measurement/plugins/variance.bro
+      scripts/base/frameworks/measurement/plugins/sum.bro
+      scripts/base/frameworks/measurement/plugins/unique.bro
+    scripts/base/frameworks/measurement/non-cluster.bro
   scripts/base/frameworks/intel/__load__.bro
-    scripts/base/frameworks/intel/./main.bro
-    scripts/base/frameworks/intel/./input.bro
+    scripts/base/frameworks/intel/main.bro
+    scripts/base/frameworks/intel/input.bro
   scripts/base/frameworks/reporter/__load__.bro
-    scripts/base/frameworks/reporter/./main.bro
+    scripts/base/frameworks/reporter/main.bro
   scripts/base/frameworks/tunnels/__load__.bro
-    scripts/base/frameworks/tunnels/./main.bro
+    scripts/base/frameworks/tunnels/main.bro
   scripts/base/protocols/conn/__load__.bro
-    scripts/base/protocols/conn/./main.bro
-    scripts/base/protocols/conn/./contents.bro
-    scripts/base/protocols/conn/./inactivity.bro
-    scripts/base/protocols/conn/./polling.bro
+    scripts/base/protocols/conn/main.bro
+    scripts/base/protocols/conn/contents.bro
+    scripts/base/protocols/conn/inactivity.bro
+    scripts/base/protocols/conn/polling.bro
   scripts/base/protocols/dns/__load__.bro
-    scripts/base/protocols/dns/./consts.bro
-    scripts/base/protocols/dns/./main.bro
+    scripts/base/protocols/dns/consts.bro
+    scripts/base/protocols/dns/main.bro
   scripts/base/protocols/ftp/__load__.bro
-    scripts/base/protocols/ftp/./utils-commands.bro
-    scripts/base/protocols/ftp/./main.bro
-    scripts/base/protocols/ftp/./file-extract.bro
-    scripts/base/protocols/ftp/./gridftp.bro
+    scripts/base/protocols/ftp/utils-commands.bro
+    scripts/base/protocols/ftp/main.bro
+    scripts/base/protocols/ftp/file-extract.bro
+    scripts/base/protocols/ftp/gridftp.bro
       scripts/base/protocols/ssl/__load__.bro
-        scripts/base/protocols/ssl/./consts.bro
-        scripts/base/protocols/ssl/./main.bro
-        scripts/base/protocols/ssl/./mozilla-ca-list.bro
+        scripts/base/protocols/ssl/consts.bro
+        scripts/base/protocols/ssl/main.bro
+        scripts/base/protocols/ssl/mozilla-ca-list.bro
   scripts/base/protocols/http/__load__.bro
-    scripts/base/protocols/http/./main.bro
-    scripts/base/protocols/http/./utils.bro
-    scripts/base/protocols/http/./file-ident.bro
-    scripts/base/protocols/http/./file-hash.bro
-    scripts/base/protocols/http/./file-extract.bro
+    scripts/base/protocols/http/main.bro
+    scripts/base/protocols/http/utils.bro
+    scripts/base/protocols/http/file-ident.bro
+    scripts/base/protocols/http/file-hash.bro
+    scripts/base/protocols/http/file-extract.bro
   scripts/base/protocols/irc/__load__.bro
-    scripts/base/protocols/irc/./main.bro
-    scripts/base/protocols/irc/./dcc-send.bro
+    scripts/base/protocols/irc/main.bro
+    scripts/base/protocols/irc/dcc-send.bro
   scripts/base/protocols/modbus/__load__.bro
-    scripts/base/protocols/modbus/./consts.bro
-    scripts/base/protocols/modbus/./main.bro
+    scripts/base/protocols/modbus/consts.bro
+    scripts/base/protocols/modbus/main.bro
   scripts/base/protocols/smtp/__load__.bro
-    scripts/base/protocols/smtp/./main.bro
-    scripts/base/protocols/smtp/./entities.bro
-    scripts/base/protocols/smtp/./entities-excerpt.bro
+    scripts/base/protocols/smtp/main.bro
+    scripts/base/protocols/smtp/entities.bro
+    scripts/base/protocols/smtp/entities-excerpt.bro
   scripts/base/protocols/socks/__load__.bro
-    scripts/base/protocols/socks/./consts.bro
-    scripts/base/protocols/socks/./main.bro
+    scripts/base/protocols/socks/consts.bro
+    scripts/base/protocols/socks/main.bro
   scripts/base/protocols/ssh/__load__.bro
-    scripts/base/protocols/ssh/./main.bro
+    scripts/base/protocols/ssh/main.bro
   scripts/base/protocols/syslog/__load__.bro
-    scripts/base/protocols/syslog/./consts.bro
-    scripts/base/protocols/syslog/./main.bro
+    scripts/base/protocols/syslog/consts.bro
+    scripts/base/protocols/syslog/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-02-11-18-44-43
+#close	2013-04-02-04-22-32
diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads
index 34ba654dec..554fcf012e 100644
--- a/testing/btest/Baseline/coverage.init-default/missing_loads
+++ b/testing/btest/Baseline/coverage.init-default/missing_loads
@@ -3,5 +3,5 @@
 -./frameworks/cluster/nodes/worker.bro
 -./frameworks/cluster/setup-connections.bro
 -./frameworks/intel/cluster.bro
--./frameworks/metrics/cluster.bro
+-./frameworks/measurement/cluster.bro
 -./frameworks/notice/cluster.bro
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
deleted file mode 100644
index bdc86c68bb..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ /dev/null
@@ -1,12 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	metrics
-#open	2012-12-17-18-43-15
-#fields	ts	ts_delta	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
-#types	time	interval	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
-1355769795.365325	3.000000	test.metric	-	6.5.4.3	-	1355769793.449322	1355769793.458467	2	6.0	1.0	5.0	3.0	4.0	2.0	2
-1355769795.365325	3.000000	test.metric	-	1.2.3.4	-	1355769793.449322	1355769793.458467	9	437.0	3.0	95.0	48.555556	674.469136	25.970544	8
-1355769795.365325	3.000000	test.metric	-	7.2.1.5	-	1355769793.449322	1355769793.458467	2	145.0	54.0	91.0	72.5	342.25	18.5	2
-#close	2012-12-17-18-43-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
deleted file mode 100644
index 51d892e8d5..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ /dev/null
@@ -1,12 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	metrics
-#open	2012-12-17-18-43-45
-#fields	ts	ts_delta	metric	index.str	index.host	index.network	result.begin	result.end	result.num	result.sum	result.min	result.max	result.avg	result.variance	result.std_dev	result.unique
-#types	time	interval	string	string	addr	subnet	time	time	count	double	double	double	double	double	double	count
-1355769825.947161	3.000000	test.metric	-	6.5.4.3	-	1355769825.947161	1355769825.947161	1	2.0	2.0	2.0	2.0	0.0	0.0	-
-1355769825.947161	3.000000	test.metric	-	1.2.3.4	-	1355769825.947161	1355769825.947161	5	221.0	5.0	94.0	44.2	915.36	30.254917	-
-1355769825.947161	3.000000	test.metric	-	7.2.1.5	-	1355769825.947161	1355769825.947161	1	1.0	1.0	1.0	1.0	0.0	0.0	-
-#close	2012-12-17-18-43-45
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout
deleted file mode 100644
index 2d0750ca18..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1..stdout
+++ /dev/null
@@ -1 +0,0 @@
-A test metric threshold was crossed!
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
deleted file mode 100644
index c87853e2b4..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
+++ /dev/null
@@ -1,10 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	notice
-#open	2013-02-11-18-41-03
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double	addr	string	subnet
-1360608063.517719	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
-#close	2013-02-11-18-41-03
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
deleted file mode 100644
index ba6c680e27..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
+++ /dev/null
@@ -1,11 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	notice
-#open	2012-07-20-01-49-23
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1342748963.085888	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
-1342748963.085888	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	6.5.4.3	-	-
-#close	2012-07-20-01-49-23
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
deleted file mode 100644
index da692f2fe2..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.thresholding/.stdout
+++ /dev/null
@@ -1,8 +0,0 @@
-THRESHOLD_SERIES: hit a threshold series value at 3 for metric_index(host=1.2.3.4)
-THRESHOLD_FUNC: hit a threshold function value at 3 for metric_index(host=1.2.3.4)
-THRESHOLD_FUNC: hit a threshold function value at 2 for metric_index(host=6.5.4.3)
-THRESHOLD_FUNC: hit a threshold function value at 1 for metric_index(host=7.2.1.5)
-THRESHOLD: hit a threshold value at 6 for metric_index(host=1.2.3.4)
-THRESHOLD_SERIES: hit a threshold series value at 6 for metric_index(host=1.2.3.4)
-THRESHOLD: hit a threshold value at 1001 for metric_index(host=7.2.1.5)
-THRESHOLD_SERIES: hit a threshold series value at 1001 for metric_index(host=7.2.1.5)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
index ddbb59c565..e17610d69e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2013-02-11-18-45-43
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double	addr	string	subnet
-1360608343.088948	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2013-02-11-18-45-43
+#open	2013-04-02-02-21-00
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+1364869260.950557	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2013-04-02-02-21-00
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
index 2f163a5491..c8b4306d22 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2013-02-11-18-45-14
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double	addr	string	subnet
-1360608314.794257	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2013-02-11-18-45-17
+#open	2013-04-02-02-21-29
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+1364869289.545369	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2013-04-02-02-21-32
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
index da5489e0b7..051f1c6266 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2013-02-11-18-33-41
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double	addr	string	subnet
-1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2013-02-11-18-33-41
+#open	2013-04-02-02-19-21
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+#close	2013-04-02-02-19-21

From e8b60d1ba85a23696926f028eea030b31b0c0cb9 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 2 Apr 2013 00:55:25 -0400
Subject: [PATCH 56/70] Updated FTP bruteforce detection and a few other small
 changes.

---
 .../base/frameworks/measurement/simple.bro    |  6 ----
 .../protocols/conn/conn-stats-per-host.bro    | 27 --------------
 .../protocols/ftp/detect-bruteforcing.bro     | 35 +++++++++++--------
 .../.stdout                                   |  4 +--
 4 files changed, 22 insertions(+), 50 deletions(-)
 delete mode 100644 scripts/base/frameworks/measurement/simple.bro
 delete mode 100644 scripts/policy/protocols/conn/conn-stats-per-host.bro

diff --git a/scripts/base/frameworks/measurement/simple.bro b/scripts/base/frameworks/measurement/simple.bro
deleted file mode 100644
index 51bf7e8c44..0000000000
--- a/scripts/base/frameworks/measurement/simple.bro
+++ /dev/null
@@ -1,6 +0,0 @@
-
-module Metrics;
-
-export {
-	
-}
\ No newline at end of file
diff --git a/scripts/policy/protocols/conn/conn-stats-per-host.bro b/scripts/policy/protocols/conn/conn-stats-per-host.bro
deleted file mode 100644
index d537d13b72..0000000000
--- a/scripts/policy/protocols/conn/conn-stats-per-host.bro
+++ /dev/null
@@ -1,27 +0,0 @@
-
-@load base/protocols/conn
-@load base/frameworks/measurement
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("conn.orig.data", 
-	                    [$every=5mins,
-	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
-	                     $period_finished=Metrics::write_log]);
-	Metrics::add_filter("conn.resp.data", 
-	                    [$every=5mins,
-	                     $measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
-	                     $period_finished=Metrics::write_log]);
-	}
-
-
-event connection_state_remove(c: connection)
-	{
-	if ( ! (c$conn$conn_state == "SF" && c$conn$proto == tcp) )
-		return;
-
-	if ( Site::is_local_addr(c$id$orig_h) )
-		Metrics::add_data("conn.orig.data", [$host=c$id$orig_h], [$num=c$orig$size]);
-	if ( Site::is_local_addr(c$id$resp_h) )
-		Metrics::add_data("conn.resp.data", [$host=c$id$resp_h], [$num=c$resp$size]);
-	}
\ No newline at end of file
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index 286cc95979..bcf7a59d06 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -25,20 +25,25 @@ export {
 
 event bro_init()
 	{
-	Metrics::add_filter("ftp.failed_auth", [$every=bruteforce_measurement_interval,
-	                                        $measure=set(Metrics::UNIQUE),
-	                                        $threshold_val_func(val: Metrics::Result) = { return val$num; },
-	                                        $threshold=bruteforce_threshold,
-	                                        $threshold_crossed(index: Metrics::Index, val: Metrics::Result) = 
-	                                        	{
-	                                        	local dur = duration_to_mins_secs(val$end-val$begin);
-	                                        	local plural = val$unique>1 ? "s" : "";
-	                                        	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", index$host, val$num, val$unique, plural, dur);
-	                                        	NOTICE([$note=FTP::Bruteforcing, 
-	                                        	        $src=index$host,
-	                                        	        $msg=message,
-	                                        	        $identifier=cat(index$host)]);
-	                                        	}]);
+	local r1: Measurement::Reducer = [$stream="ftp.failed_auth", $apply=set(Measurement::UNIQUE)];
+	Measurement::create([$epoch=bruteforce_measurement_interval,
+	                     $reducers=set(r1),
+	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
+	                     	{ 
+	                     	return result["ftp.failed_auth"]$num;
+	                     	},
+	                     $threshold=bruteforce_threshold,
+	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
+	                     	{
+	                     	local r = result["ftp.failed_auth"];
+	                     	local dur = duration_to_mins_secs(r$end-r$begin);
+	                     	local plural = r$unique>1 ? "s" : "";
+	                     	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                     	NOTICE([$note=FTP::Bruteforcing, 
+	                     	        $src=key$host,
+	                     	        $msg=message,
+	                     	        $identifier=cat(key$host)]);
+	                     	}]);
 	}
 
 event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
@@ -47,6 +52,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
 	if ( cmd == "USER" || cmd == "PASS" )
 		{
 		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
-			Metrics::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+			Measurement::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
 		}
 	}
\ No newline at end of file
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
index 09c65c3864..ac8785d182 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
@@ -1,6 +1,6 @@
 THRESHOLD_SERIES: hit a threshold series value at 3 for measurement_key(host=1.2.3.4)
-THRESHOLD: hit a threshold value at 6 for measurement_key(host=1.2.3.4)
 THRESHOLD_SERIES: hit a threshold series value at 6 for measurement_key(host=1.2.3.4)
-THRESHOLD: hit a threshold value at 1001 for measurement_key(host=7.2.1.5)
+THRESHOLD: hit a threshold value at 6 for measurement_key(host=1.2.3.4)
 THRESHOLD_SERIES: hit a threshold series value at 1001 for measurement_key(host=7.2.1.5)
+THRESHOLD: hit a threshold value at 1001 for measurement_key(host=7.2.1.5)
 THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at 55x for measurement_key(host=7.2.1.5)

From 94f39fee2a67db955a90339129dba9725e692cd2 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 2 Apr 2013 01:04:40 -0400
Subject: [PATCH 57/70] Updating DocSourcesList

---
 DocSourcesList.cmake | 59 ++++++++++++++++++++++++++++++++++++++------
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/DocSourcesList.cmake b/DocSourcesList.cmake
index 1743b0258f..5ac87a6305 100644
--- a/DocSourcesList.cmake
+++ b/DocSourcesList.cmake
@@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal)
 rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
@@ -31,15 +32,31 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
 rest_target(${psd} base/frameworks/communication/main.bro)
 rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
+rest_target(${psd} base/frameworks/input/main.bro)
+rest_target(${psd} base/frameworks/input/readers/ascii.bro)
+rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
+rest_target(${psd} base/frameworks/input/readers/raw.bro)
+rest_target(${psd} base/frameworks/intel/cluster.bro)
+rest_target(${psd} base/frameworks/intel/input.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
+rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
+rest_target(${psd} base/frameworks/logging/writers/none.bro)
+rest_target(${psd} base/frameworks/measurement/cluster.bro)
+rest_target(${psd} base/frameworks/measurement/main.bro)
+rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/average.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/max.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/min.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/sample.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/sum.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/unique.bro)
+rest_target(${psd} base/frameworks/measurement/plugins/variance.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -48,18 +65,23 @@ rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
 rest_target(${psd} base/frameworks/notice/cluster.bro)
 rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
 rest_target(${psd} base/frameworks/notice/main.bro)
+rest_target(${psd} base/frameworks/notice/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/weird.bro)
 rest_target(${psd} base/frameworks/packet-filter/main.bro)
 rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/tunnels/main.bro)
+rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
+rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/gridftp.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
 rest_target(${psd} base/protocols/http/file-extract.bro)
@@ -69,9 +91,13 @@ rest_target(${psd} base/protocols/http/main.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
+rest_target(${psd} base/protocols/modbus/consts.bro)
+rest_target(${psd} base/protocols/modbus/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
+rest_target(${psd} base/protocols/socks/consts.bro)
+rest_target(${psd} base/protocols/socks/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
 rest_target(${psd} base/protocols/ssl/main.bro)
@@ -85,36 +111,50 @@ rest_target(${psd} base/utils/files.bro)
 rest_target(${psd} base/utils/numbers.bro)
 rest_target(${psd} base/utils/paths.bro)
 rest_target(${psd} base/utils/patterns.bro)
+rest_target(${psd} base/utils/queue.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} base/utils/time.bro)
+rest_target(${psd} base/utils/urls.bro)
 rest_target(${psd} policy/frameworks/communication/listen.bro)
 rest_target(${psd} policy/frameworks/control/controllee.bro)
 rest_target(${psd} policy/frameworks/control/controller.bro)
 rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
 rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
+rest_target(${psd} policy/frameworks/intel/conn-established.bro)
+rest_target(${psd} policy/frameworks/intel/dns.bro)
+rest_target(${psd} policy/frameworks/intel/http-host-header.bro)
+rest_target(${psd} policy/frameworks/intel/http-url.bro)
+rest_target(${psd} policy/frameworks/intel/http-user-agents.bro)
+rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
+rest_target(${psd} policy/frameworks/intel/smtp.bro)
+rest_target(${psd} policy/frameworks/intel/ssl.bro)
+rest_target(${psd} policy/frameworks/intel/where-locations.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
+rest_target(${psd} policy/integration/collective-intel/main.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
+rest_target(${psd} policy/misc/app-metrics.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
+rest_target(${psd} policy/misc/detect-traceroute/main.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
+rest_target(${psd} policy/misc/scan.bro)
 rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
+rest_target(${psd} policy/protocols/conn/metrics.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
+rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)
-rest_target(${psd} policy/protocols/http/detect-intel.bro)
 rest_target(${psd} policy/protocols/http/detect-sqli.bro)
 rest_target(${psd} policy/protocols/http/detect-webapps.bro)
 rest_target(${psd} policy/protocols/http/header-names.bro)
@@ -122,8 +162,11 @@ rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
+rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
+rest_target(${psd} policy/protocols/smtp/metrics.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
 rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)
@@ -133,9 +176,11 @@ rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
 rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
 rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
+rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
+rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
 rest_target(${psd} site/local-manager.bro)
 rest_target(${psd} site/local-proxy.bro)

From a615601269dd2b1e3d7cdf1e65c5257afa8c2952 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 11 Apr 2013 09:42:46 -0400
Subject: [PATCH 58/70] Trying to fix a state maintenance issue.

---
 scripts/base/frameworks/measurement/main.bro | 30 +++++++++++---------
 scripts/base/utils/queue.bro                 |  4 +--
 2 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
index 5e33ff7a25..7685099068 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -142,12 +142,12 @@ type Thresholding: record {
 	threshold_series_index: count &default=0;
 };
 
+# Internal use only.  For tracking thresholds per measurement and key.
+global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
+
 redef record Measurement += {
 	# Internal use only (mostly for cluster coherency).
 	id: string &optional;
-
-	# Internal use only.  For tracking tresholds per key.
-	threshold_tracker: table[Key] of Thresholding &optional;
 };
 
 # Store of measurements indexed on the measurement id.
@@ -249,6 +249,7 @@ function reset(m: Measurement)
 		delete result_store[m$id];
 
 	result_store[m$id] = table();
+	threshold_tracker[m$id] = table();
 	}
 
 function create(m: Measurement)
@@ -260,8 +261,7 @@ function create(m: Measurement)
 
 	if ( ! m?$id )
 		m$id=unique_id("");
-	local tmp: table[Key] of Thresholding = table();
-	m$threshold_tracker = tmp;
+	threshold_tracker[m$id] = table();
 	measurement_store[m$id] = m;
 
 	for ( reducer in m$reducers )
@@ -322,12 +322,6 @@ function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct:
 	if ( ! (m?$threshold || m?$threshold_series) )
 		return F;
 
-	if ( key !in m$threshold_tracker )
-		{
-		local tmp: Thresholding;
-		m$threshold_tracker[key] = tmp;
-		}
-
 	# Add in the extra ResultVals to make threshold_vals easier to write.
 	if ( |m$reducers| != |result| )
 		{
@@ -343,7 +337,17 @@ function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct:
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
 		watch = double_to_count(floor(watch/modify_pct));
 
-	local tt = m$threshold_tracker[key];
+	if ( m$id !in threshold_tracker )
+		threshold_tracker[m$id] = table();
+	local t_tracker = threshold_tracker[m$id];
+
+	if ( key !in t_tracker )
+		{
+		local ttmp: Thresholding;
+		t_tracker[key] = ttmp;
+		}
+	local tt = threshold_tracker[m$id][key];
+
 	if ( m?$threshold && ! tt$is_threshold_crossed && watch >= m$threshold )
 		{
 		# Value crossed the threshold.
@@ -379,7 +383,7 @@ function threshold_crossed(m: Measurement, key: Key, result: Result)
 		}
 
 	m$threshold_crossed(key, result);
-	local tt = m$threshold_tracker[key];
+	local tt = threshold_tracker[m$id][key];
 	tt$is_threshold_crossed = T;
 
 	# Bump up to the next threshold series index if a threshold series is being used.
diff --git a/scripts/base/utils/queue.bro b/scripts/base/utils/queue.bro
index 1e7a293e17..ed45b034f5 100644
--- a/scripts/base/utils/queue.bro
+++ b/scripts/base/utils/queue.bro
@@ -21,14 +21,14 @@ export {
 	## Returns: An opaque queue record.
 	global init:       function(s: Settings): Queue;
 
-	## Push a string onto the top of a queue.
+	## Put a string onto the beginning of a queue.
 	## 
 	## q: The queue to put the value into.
 	## 
 	## val: The value to insert into the queue. 
 	global put:       function(q: Queue, val: any);
 
-	## Pop a string from the bottom of a queue.
+	## Get a string from the end of a queue.
 	##
 	## q: The queue to get the string from.
 	##

From e93fd69cf20c41a28b44c18796793ac93c0df6c2 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 12 Apr 2013 09:28:38 -0400
Subject: [PATCH 59/70] Small updates to hopefully correct reporter errors
 leading to lost memory.

---
 scripts/base/frameworks/measurement/main.bro        | 13 +++++++++----
 .../base/frameworks/measurement/plugins/sample.bro  |  8 ++++----
 .../base/frameworks/measurement/plugins/std-dev.bro |  4 +---
 .../frameworks/measurement/plugins/variance.bro     |  3 ++-
 4 files changed, 16 insertions(+), 12 deletions(-)

diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/measurement/main.bro
index 7685099068..db59a7ba85 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/measurement/main.bro
@@ -293,14 +293,19 @@ function add_data(id: string, key: Key, point: DataPoint)
 			key = r$normalize_key(copy(key));
 		
 		local m = measurement_store[r$mid];
-		local results = result_store[m$id];
+		
+		if ( r$mid !in result_store )
+			result_store[m$id] = table();
+		local results = result_store[r$mid];
+
 		if ( key !in results )
 			results[key] = table();
-		if ( id !in results[key] )
-			results[key][id] = init_resultval(r);
-
 		local result = results[key];
+
+		if ( id !in result )
+			result[id] = init_resultval(r);
 		local result_val = result[id];
+
 		++result_val$num;
 		# Continually update the $end field.
 		result_val$end=network_time();
diff --git a/scripts/base/frameworks/measurement/plugins/sample.bro b/scripts/base/frameworks/measurement/plugins/sample.bro
index 399f572490..0187946d61 100644
--- a/scripts/base/frameworks/measurement/plugins/sample.bro
+++ b/scripts/base/frameworks/measurement/plugins/sample.bro
@@ -4,16 +4,16 @@
 module Measurement;
 
 export {
-	
 	redef record Reducer += {
 		## A number of sample DataPoints to collect.
 		samples: count &default=0;
 	};
 
 	redef record ResultVal += {
-		# This is the queue where samples
-		# are maintained.  Use the :bro:see:`Measurement::get_samples`
-		## function to get a vector of the samples.
+		## This is the queue where samples
+		## are maintained.  Use the 
+		## :bro:see:`Measurement::get_samples` function
+		## to get a vector of the samples.
 		samples: Queue::Queue &optional;
 	};
 
diff --git a/scripts/base/frameworks/measurement/plugins/std-dev.bro b/scripts/base/frameworks/measurement/plugins/std-dev.bro
index bfcaa67910..63dcc62d4b 100644
--- a/scripts/base/frameworks/measurement/plugins/std-dev.bro
+++ b/scripts/base/frameworks/measurement/plugins/std-dev.bro
@@ -11,7 +11,7 @@ export {
 
 	redef record ResultVal += {
 		## For numeric data, this calculates the standard deviation.
-		std_dev: double &optional;
+		std_dev: double &default=0.0;
 	};
 }
 
@@ -28,8 +28,6 @@ hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal
 		{
 		if ( rv?$variance )
 			calc_std_dev(rv);
-		else
-			rv$std_dev = 0.0;
 		}
 	}
 
diff --git a/scripts/base/frameworks/measurement/plugins/variance.bro b/scripts/base/frameworks/measurement/plugins/variance.bro
index 2868a8a3ad..3c2223430d 100644
--- a/scripts/base/frameworks/measurement/plugins/variance.bro
+++ b/scripts/base/frameworks/measurement/plugins/variance.bro
@@ -44,7 +44,8 @@ hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal
 # Reduced priority since this depends on the average
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-5
 	{
-	if ( rv1?$var_s && rv2?$var_s )
+	if ( rv1?$var_s && rv1?$average &&
+	     rv2?$var_s && rv2?$average )
 		{
 		local rv1_avg_sq = (rv1$average - result$average);
 		rv1_avg_sq = rv1_avg_sq*rv1_avg_sq;

From 8165d6077d4935ada81b9e57ce88f314eaea3ce8 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 12 Apr 2013 11:20:45 -0400
Subject: [PATCH 60/70] Fix another occasional reporter error.

---
 scripts/base/frameworks/measurement/cluster.bro | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/base/frameworks/measurement/cluster.bro b/scripts/base/frameworks/measurement/cluster.bro
index 481b306417..fa5d58e5f6 100644
--- a/scripts/base/frameworks/measurement/cluster.bro
+++ b/scripts/base/frameworks/measurement/cluster.bro
@@ -132,8 +132,11 @@ event Measurement::cluster_measurement_request(uid: string, mid: string)
 	#print fmt("WORKER %s: received the cluster_measurement_request event for %s.", Cluster::node, id);
 	
 	# Initiate sending all of the data for the requested measurement.
-	event Measurement::send_data(uid, mid, result_store[mid]);
-	
+	if ( mid in result_store )
+		event Measurement::send_data(uid, mid, result_store[mid]);
+	else
+		event Measurement::send_data(uid, mid, table());
+
 	# Lookup the actual measurement and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
 	if ( mid in measurement_store )

From fbe967e16a473b7768e0608b164c674d4f9fdbb1 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 15 Apr 2013 15:12:28 -0400
Subject: [PATCH 61/70] Checkpoint for SumStats rename.

---
 .../{measurement => sumstats}/__load__.bro    |   0
 .../{measurement => sumstats}/cluster.bro     | 111 ++++----
 .../{measurement => sumstats}/main.bro        | 236 +++++++++---------
 .../{measurement => sumstats}/non-cluster.bro |   8 +-
 .../plugins/__load__.bro                      |   2 +-
 .../plugins/average.bro                       |   6 +-
 .../{measurement => sumstats}/plugins/max.bro |   6 +-
 .../{measurement => sumstats}/plugins/min.bro |   6 +-
 .../plugins/sample.bro                        |  18 +-
 .../plugins/std-dev.bro                       |   6 +-
 .../{measurement => sumstats}/plugins/sum.bro |  10 +-
 .../plugins/unique.bro                        |   8 +-
 .../plugins/variance.bro                      |   6 +-
 scripts/base/protocols/ssh/main.bro           |   3 +-
 scripts/policy/misc/app-metrics.bro           |  74 +++---
 .../policy/misc/detect-traceroute/main.bro    |  56 ++---
 scripts/policy/misc/scan.bro                  | 108 ++++----
 .../protocols/ftp/detect-bruteforcing.bro     |  42 ++--
 scripts/policy/protocols/http/detect-sqli.bro |  76 +++---
 scripts/policy/protocols/smtp/metrics.bro     |   4 +-
 .../protocols/ssh/detect-bruteforcing.bro     |  46 ++--
 .../manager-1..stdout                         |   0
 .../.stdout                                   |   0
 .../manager-1..stdout                         |   0
 .../.stdout                                   |   0
 .../frameworks/measurement/basic-cluster.bro  |  83 ------
 .../base/frameworks/measurement/basic.bro     |  34 ---
 .../frameworks/measurement/thresholding.bro   |  73 ------
 .../frameworks/sumstats/basic-cluster.bro     |  82 ++++++
 .../base/frameworks/sumstats/basic.bro        |  34 +++
 .../cluster-intermediate-update.bro           |  35 ++-
 .../base/frameworks/sumstats/thresholding.bro |  73 ++++++
 32 files changed, 626 insertions(+), 620 deletions(-)
 rename scripts/base/frameworks/{measurement => sumstats}/__load__.bro (100%)
 rename scripts/base/frameworks/{measurement => sumstats}/cluster.bro (74%)
 rename scripts/base/frameworks/{measurement => sumstats}/main.bro (52%)
 rename scripts/base/frameworks/{measurement => sumstats}/non-cluster.bro (58%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/__load__.bro (85%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/average.bro (83%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/max.bro (81%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/min.bro (81%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/sample.bro (63%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/std-dev.bro (80%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/sum.bro (74%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/unique.bro (86%)
 rename scripts/base/frameworks/{measurement => sumstats}/plugins/variance.bro (91%)
 rename testing/btest/Baseline/{scripts.base.frameworks.measurement.basic-cluster => scripts.base.frameworks.sumstats.basic-cluster}/manager-1..stdout (100%)
 rename testing/btest/Baseline/{scripts.base.frameworks.measurement.basic => scripts.base.frameworks.sumstats.basic}/.stdout (100%)
 rename testing/btest/Baseline/{scripts.base.frameworks.measurement.cluster-intermediate-update => scripts.base.frameworks.sumstats.cluster-intermediate-update}/manager-1..stdout (100%)
 rename testing/btest/Baseline/{scripts.base.frameworks.measurement.thresholding => scripts.base.frameworks.sumstats.thresholding}/.stdout (100%)
 delete mode 100644 testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro
 delete mode 100644 testing/btest/scripts/base/frameworks/measurement/basic.bro
 delete mode 100644 testing/btest/scripts/base/frameworks/measurement/thresholding.bro
 create mode 100644 testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
 create mode 100644 testing/btest/scripts/base/frameworks/sumstats/basic.bro
 rename testing/btest/scripts/base/frameworks/{measurement => sumstats}/cluster-intermediate-update.bro (59%)
 create mode 100644 testing/btest/scripts/base/frameworks/sumstats/thresholding.bro

diff --git a/scripts/base/frameworks/measurement/__load__.bro b/scripts/base/frameworks/sumstats/__load__.bro
similarity index 100%
rename from scripts/base/frameworks/measurement/__load__.bro
rename to scripts/base/frameworks/sumstats/__load__.bro
diff --git a/scripts/base/frameworks/measurement/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
similarity index 74%
rename from scripts/base/frameworks/measurement/cluster.bro
rename to scripts/base/frameworks/sumstats/cluster.bro
index fa5d58e5f6..098c047961 100644
--- a/scripts/base/frameworks/measurement/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -7,7 +7,7 @@
 @load base/frameworks/cluster
 @load ./main
 
-module Measurement;
+module SumStats;
 
 export {
 	## Allows a user to decide how large of result groups the 
@@ -48,22 +48,21 @@ export {
 	global cluster_key_request: event(uid: string, mid: string, key: Key);
 
 	## This event is sent by nodes in response to a 
-	## :bro:id:`Measurement::cluster_key_request` event.
+	## :bro:id:`SumStats::cluster_key_request` event.
 	global cluster_key_response: event(uid: string, mid: string, key: Key, result: Result);
 
 	## This is sent by workers to indicate that they crossed the percent of the 
 	## current threshold by the percentage defined globally in 
-	## :bro:id:`Measurement::cluster_request_global_view_percent`
-	global cluster_key_intermediate_response: event(mid: string, key: Measurement::Key);
+	## :bro:id:`SumStats::cluster_request_global_view_percent`
+	global cluster_key_intermediate_response: event(mid: string, key: SumStats::Key);
 
 	## This event is scheduled internally on workers to send result chunks.
 	global send_data: event(uid: string, mid: string, data: ResultTable);
 }
 
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /Measurement::cluster_(measurement_request|key_request)/;
-redef Cluster::manager2worker_events += /Measurement::new_measurement/;
-redef Cluster::worker2manager_events += /Measurement::cluster_(measurement_response|key_response|key_intermediate_response)/;
+redef Cluster::manager2worker_events += /SumStats::cluster_(measurement_request|key_request)/;
+redef Cluster::worker2manager_events += /SumStats::cluster_(measurement_response|key_response|key_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # This variable is maintained to know what keys have recently sent as 
@@ -75,32 +74,32 @@ global recent_global_view_keys: table[string, Key] of count &create_expire=1min
 event bro_init() &priority=-100
 	{
 	# The manager is the only host allowed to track these.
-	measurement_store = table();
+	stats_store = table();
 	reducer_store = table();
 	}
 
 # This is done on all non-manager node types in the event that a metric is 
 # being collected somewhere other than a worker.
-function data_added(m: Measurement, key: Key, result: Result)
+function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	# If an intermediate update for this value was sent recently, don't send
 	# it again.
-	if ( [m$id, key] in recent_global_view_keys )
+	if ( [ss$id, key] in recent_global_view_keys )
 		return;
 
 	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
 	# crosses the full threshold then it's a candidate to send as an 
 	# intermediate update.
 	if ( enable_intermediate_updates && 
-	     check_thresholds(m, key, result, cluster_request_global_view_percent) )
+	     check_thresholds(ss, key, result, cluster_request_global_view_percent) )
 		{
 		# kick off intermediate update
-		event Measurement::cluster_key_intermediate_response(m$id, key);
-		++recent_global_view_keys[m$id, key];
+		event SumStats::cluster_key_intermediate_response(ss$id, key);
+		++recent_global_view_keys[ss$id, key];
 		}
 	}
 
-event Measurement::send_data(uid: string, mid: string, data: ResultTable)
+event SumStats::send_data(uid: string, mid: string, data: ResultTable)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 
@@ -122,39 +121,39 @@ event Measurement::send_data(uid: string, mid: string, data: ResultTable)
 	if ( |data| == 0 )
 		done = T;
 	
-	event Measurement::cluster_measurement_response(uid, mid, local_data, done);
+	event SumStats::cluster_measurement_response(uid, mid, local_data, done);
 	if ( ! done )
-		schedule 0.01 sec { Measurement::send_data(uid, mid, data) };
+		schedule 0.01 sec { SumStats::send_data(uid, mid, data) };
 	}
 
-event Measurement::cluster_measurement_request(uid: string, mid: string)
+event SumStats::cluster_measurement_request(uid: string, mid: string)
 	{
 	#print fmt("WORKER %s: received the cluster_measurement_request event for %s.", Cluster::node, id);
 	
 	# Initiate sending all of the data for the requested measurement.
 	if ( mid in result_store )
-		event Measurement::send_data(uid, mid, result_store[mid]);
+		event SumStats::send_data(uid, mid, result_store[mid]);
 	else
-		event Measurement::send_data(uid, mid, table());
+		event SumStats::send_data(uid, mid, table());
 
 	# Lookup the actual measurement and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
-	if ( mid in measurement_store )
-		reset(measurement_store[mid]);
+	if ( mid in stats_store )
+		reset(stats_store[mid]);
 	}
 	
-event Measurement::cluster_key_request(uid: string, mid: string, key: Key)
+event SumStats::cluster_key_request(uid: string, mid: string, key: Key)
 	{
 	if ( mid in result_store && key in result_store[mid] )
 		{
 		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
-		event Measurement::cluster_key_response(uid, mid, key, result_store[mid][key]);
+		event SumStats::cluster_key_response(uid, mid, key, result_store[mid][key]);
 		}
 	else
 		{
 		# We need to send an empty response if we don't have the data so that the manager
 		# can know that it heard back from all of the workers.
-		event Measurement::cluster_key_response(uid, mid, key, table());
+		event SumStats::cluster_key_response(uid, mid, key, table());
 		}
 	}
 
@@ -166,7 +165,7 @@ event Measurement::cluster_key_request(uid: string, mid: string, key: Key)
 # This variable is maintained by manager nodes as they collect and aggregate 
 # results.  
 # Index on a uid.
-global measurement_results: table[string] of ResultTable &read_expire=1min;
+global stats_results: table[string] of ResultTable &read_expire=1min;
 
 # This variable is maintained by manager nodes to track how many "dones" they
 # collected per collection unique id.  Once the number of results for a uid 
@@ -189,7 +188,7 @@ global outstanding_global_views: table[string] of count &default=0;
 
 const zero_time = double_to_time(0.0);
 # Managers handle logging.
-event Measurement::finish_epoch(m: Measurement)
+event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( network_time() > zero_time )
 		{
@@ -198,25 +197,25 @@ event Measurement::finish_epoch(m: Measurement)
 		
 		if ( uid in measurement_results )
 			delete measurement_results[uid];
-		measurement_results[uid] = table();
+		stats_results[uid] = table();
 		
 		# Request data from peers.
-		event Measurement::cluster_measurement_request(uid, m$id);
+		event SumStats::cluster_measurement_request(uid, ss$id);
 		}
 
 	# Schedule the next finish_epoch event.
-	schedule m$epoch { Measurement::finish_epoch(m) };
+	schedule m$epoch { SumStats::finish_epoch(m) };
 	}
 
 # This is unlikely to be called often, but it's here in case there are measurements
 # being collected by managers.
-function data_added(m: Measurement, key: Key, result: Result)
+function data_added(ss: SumStat, key: Key, result: Result)
 	{
-	if ( check_thresholds(m, key, result, 1.0) )
-		threshold_crossed(m, key, result);
+	if ( check_thresholds(ss, key, result, 1.0) )
+		threshold_crossed(ss, key, result);
 	}
 	
-event Measurement::cluster_key_response(uid: string, mid: string, key: Key, result: Result)
+event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
 	{
 	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
 
@@ -233,26 +232,26 @@ event Measurement::cluster_key_response(uid: string, mid: string, key: Key, resu
 	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		local m = measurement_store[mid];
+		local ss = stats_store[ssid];
 		local ir = key_requests[uid];
-		if ( check_thresholds(m, key, ir, 1.0) )
-			threshold_crossed(m, key, ir);
+		if ( check_thresholds(ss, key, ir, 1.0) )
+			threshold_crossed(ss, key, ir);
 
 		delete done_with[uid];
 		delete key_requests[uid];
 		# Check that there is an outstanding view before subtracting.
-		if ( outstanding_global_views[mid] > 0 )
-			--outstanding_global_views[mid];
+		if ( outstanding_global_views[ssid] > 0 )
+			--outstanding_global_views[ssid];
 		}
 	}
 
 # Managers handle intermediate updates here.
-event Measurement::cluster_key_intermediate_response(mid: string, key: Key)
+event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
 	{
 	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
 	#print fmt("MANAGER: requesting key data for %s", key2str(key));
 
-	if ( mid in outstanding_global_views &&
+	if ( ssid in outstanding_global_views &&
 	     |outstanding_global_views[mid]| > max_outstanding_global_views )
 		{
 		# Don't do this intermediate update.  Perhaps at some point in the future 
@@ -261,13 +260,13 @@ event Measurement::cluster_key_intermediate_response(mid: string, key: Key)
 		return;
 		}
 
-	++outstanding_global_views[mid];
+	++outstanding_global_views[ssid];
 
 	local uid = unique_id("");
-	event Measurement::cluster_key_request(uid, mid, key);
+	event SumStats::cluster_key_request(uid, ssid, key);
 	}
 
-event Measurement::cluster_measurement_response(uid: string, mid: string, data: ResultTable, done: bool)
+event SumStats::cluster_measurement_response(uid: string, ssid: string, data: ResultTable, done: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 
@@ -275,8 +274,8 @@ event Measurement::cluster_measurement_response(uid: string, mid: string, data:
 	if ( done )
 		++done_with[uid];
 
-	local local_data = measurement_results[uid];
-	local m = measurement_store[mid];
+	local local_data = stats_results[uid];
+	local ss = stats_store[ssid];
 
 	for ( key in data )
 		{
@@ -285,14 +284,14 @@ event Measurement::cluster_measurement_response(uid: string, mid: string, data:
 		else
 			local_data[key] = data[key];
 
-		# If a measurement is done being collected, thresholds for each key
-		# need to be checked so we're doing it here to avoid doubly iterating 
-		# over each key.
+		# If a stat is done being collected, thresholds for each key
+		# need to be checked so we're doing it here to avoid doubly 
+		# iterating over each key.
 		if ( Cluster::worker_count == done_with[uid] )
 			{
-			if ( check_thresholds(m, key, local_data[key], 1.0) )
+			if ( check_thresholds(ss, key, local_data[key], 1.0) )
 				{
-				threshold_crossed(m, key, local_data[key]);
+				threshold_crossed(ss, key, local_data[key]);
 				}
 			}
 		}
@@ -300,20 +299,20 @@ event Measurement::cluster_measurement_response(uid: string, mid: string, data:
 	# If the data has been collected from all peers, we are done and ready to finish.
 	if ( Cluster::worker_count == done_with[uid] )
 		{
-		if ( m?$epoch_finished )
-			m$epoch_finished(local_data);
+		if ( ss?$epoch_finished )
+			ss$epoch_finished(local_data);
 
 		# Clean up
-		delete measurement_results[uid];
+		delete stats_results[uid];
 		delete done_with[uid];
 		# Not sure I need to reset the measurement on the manager.
-		reset(m);
+		reset(ss);
 		}
 	}
 
 event remote_connection_handshake_done(p: event_peer) &priority=5
 	{
-	send_id(p, "Measurement::measurement_store");
-	send_id(p, "Measurement::reducer_store");
+	send_id(p, "SumStats::stats_store");
+	send_id(p, "SumStats::reducer_store");
 	}
 @endif
diff --git a/scripts/base/frameworks/measurement/main.bro b/scripts/base/frameworks/sumstats/main.bro
similarity index 52%
rename from scripts/base/frameworks/measurement/main.bro
rename to scripts/base/frameworks/sumstats/main.bro
index db59a7ba85..a5c41ba1f3 100644
--- a/scripts/base/frameworks/measurement/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -1,6 +1,8 @@
-##! The measurement framework provides a way to count and measure data.  
+##! The summary statistics framework provides a way to 
+##! summarize large streams of data into simple reduced 
+##! measurements.
 
-module Measurement;
+module SumStats;
 
 export {
 	## The various calculations are all defined as plugins.
@@ -8,14 +10,17 @@ export {
 		PLACEHOLDER
 	};
 
-	## Represents a thing which is having measurement results collected for it.
+	## Represents a thing which is having summarization 
+	## results collected for it.
 	type Key: record {
-		## A non-address related measurement or a sub-key for an address based measurement.
-		## An example might be successful SSH connections by client IP address
+		## A non-address related summarization or a sub-key for 
+		## an address based summarization. An example might be 
+		## successful SSH connections by client IP address
 		## where the client string would be the key value.
-		## Another example might be number of HTTP requests to a particular
-		## value in a Host header.  This is an example of a non-host based
-		## metric since multiple IP addresses could respond for the same Host
+		## Another example might be number of HTTP requests to 
+		## a particular value in a Host header.  This is an 
+		## example of a non-host based metric since multiple 
+		## IP addresses could respond for the same Host 
 		## header value.
 		str:  string &optional;
 	
@@ -23,9 +28,9 @@ export {
 		host: addr &optional;
 	};
 	
-	## Represents data being added for a single metric data point.
-	## Only supply a single value here at a time.
-	type DataPoint: record {
+	## Represents data being added for a single observation.
+	## Only supply a single field at a time!
+	type Observation: record {
 		## Count value.
 		num:  count  &optional;
 		## Double value.
@@ -35,102 +40,110 @@ export {
 	};
 
 	type Reducer: record {
-		## Data stream identifier for the reducer to attach to.
+		## Observation stream identifier for the reducer 
+		## to attach to.
 		stream:         string;
 
 		## The calculations to perform on the data points.
 		apply:          set[Calculation];
 		
-		## A predicate so that you can decide per key if you would like
-		## to accept the data being inserted.
-		pred:           function(key: Measurement::Key, point: Measurement::DataPoint): bool &optional;
+		## A predicate so that you can decide per key if you 
+		## would like to accept the data being inserted.
+		pred:           function(key: SumStats::Key, obs: SumStats::Observation): bool &optional;
 		
 		## A function to normalize the key.  This can be used to aggregate or
 		## normalize the entire key.
-		normalize_key:  function(key: Measurement::Key): Key &optional;
+		normalize_key:  function(key: SumStats::Key): Key &optional;
 	};
 
-	## Value calculated for a data point stream fed into a reducer.
+	## Value calculated for an observation stream fed into a reducer.
 	## Most of the fields are added by plugins.
 	type ResultVal: record {
-		## The time when the first data point was added to this result value.
+		## The time when the first observation was added to 
+		## this result value.
 		begin:  time;
 
-		## The time when the last data point was added to this result value.
+		## The time when the last observation was added to 
+		## this result value.
 		end:    time;
 
-		## The number of measurements received.
+		## The number of observations received.
 		num:    count &default=0;
 	};
 
 	## Type to store results for multiple reducers.
 	type Result: table[string] of ResultVal;
 
-	## Type to store a table of measurement results indexed by the measurement key.
+	## Type to store a table of sumstats results indexed 
+	## by keys.
 	type ResultTable: table[Key] of Result;
 
-	## Measurements represent an aggregation of reducers along with 
+	## SumStats represent an aggregation of reducers along with 
 	## mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
-	type Measurement: record {
-		## The interval at which this filter should be "broken" and the
-		## '$epoch_finished' callback called.  The results are also reset 
-		## at this time so any threshold based detection needs to be set to a 
-		## number that should be expected to happen within this epoch.
+	## It's best to not access any global state outside 
+	## of the variables given to the callbacks because there 
+	## is no assurance provided as to where the callbacks 
+	## will be executed on clusters.
+	type SumStat: record {
+		## The interval at which this filter should be "broken" 
+		## and the '$epoch_finished' callback called.  The 
+		## results are also reset at this time so any threshold
+		## based detection needs to be set to a 
+		## value that should be expected to happen within 
+		## this epoch.
 		epoch:              interval;
 
-		## The reducers for the measurement indexed by data id.
+		## The reducers for the SumStat
 		reducers:           set[Reducer];
 
-		## Provide a function to calculate a value from the :bro:see:`Result`
-		## structure which will be used for thresholding.
-		threshold_val:      function(key: Measurement::Key, result: Measurement::Result): count &optional;
+		## Provide a function to calculate a value from the 
+		## :bro:see:`Result` structure which will be used 
+		## for thresholding.  
+		## This is required if a $threshold value is given.
+		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): count &optional;
 
-		## The threshold value for calling the $threshold_crossed callback.
+		## The threshold value for calling the 
+		## $threshold_crossed callback.
 		threshold:          count             &optional;
 		
-		## A series of thresholds for calling the $threshold_crossed callback.
+		## A series of thresholds for calling the 
+		## $threshold_crossed callback.
 		threshold_series:   vector of count   &optional;
 
 		## A callback that is called when a threshold is crossed.
-		threshold_crossed:  function(key: Measurement::Key, result: Measurement::Result) &optional;
+		threshold_crossed:  function(key: SumStats::Key, result: SumStats::Result) &optional;
 		
-		## A callback with the full collection of Results for this filter.
-		## It's best to not access any global state outside of the variables
-		## given to the callback because there is no assurance provided as to
-		## where the callback will be executed on clusters.
-		epoch_finished:    function(rt: Measurement::ResultTable) &optional;
+		## A callback with the full collection of Results for 
+		## this SumStat.
+		epoch_finished:    function(rt: SumStats::ResultTable) &optional;
 	};
 	
-	## Create a measurement.
-	global create: function(m: Measurement::Measurement);
+	## Create a summary statistic.
+	global create: function(m: SumStats::SumStat);
 
-	## Add data into a data point stream.  This should be called when
-	## a script has measured some point value.
+	## Add data into an observation stream. This should be 
+	## called when a script has measured some point value.
 	##
-	## id: The stream identifier that the data point represents.
+	## id: The observation stream identifier that the data
+	##     point represents.
 	##
-	## key: The measurement key that the value is to be added to.
+	## key: The key that the value is related to.
 	##
-	## point: The data point to send into the stream.
-	global add_data: function(id: string, key: Measurement::Key, point: Measurement::DataPoint);
+	## obs: The data point to send into the stream.
+	global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
 
-	## Helper function to represent a :bro:type:`Measurement::Key` value as 
+	## Helper function to represent a :bro:type:`SumStats::Key` value as 
 	## a simple string.
 	## 
 	## key: The metric key that is to be converted into a string.
 	##
 	## Returns: A string representation of the metric key.
-	global key2str: function(key: Measurement::Key): string;
-
-	## This event is generated for each new measurement that is created.
-	##
-	## m: The record which describes a measurement.
-	global new_measurement: event(m: Measurement);
+	global key2str: function(key: SumStats::Key): string;
 }
 
 redef record Reducer += {
-	# Internal use only.  Provides a reference back to the related Measurement by it's ID.
+	# Internal use only.  Provides a reference back to the related SumStats by it's ID.
 	mid: string &optional;
 };
 
@@ -142,16 +155,16 @@ type Thresholding: record {
 	threshold_series_index: count &default=0;
 };
 
-# Internal use only.  For tracking thresholds per measurement and key.
+# Internal use only.  For tracking thresholds per sumstat and key.
 global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
 
-redef record Measurement += {
+redef record SumStats += {
 	# Internal use only (mostly for cluster coherency).
 	id: string &optional;
 };
 
-# Store of measurements indexed on the measurement id.
-global measurement_store: table[string] of Measurement = table();
+# Store of sumstats indexed on the sumstat id.
+global stats_store: table[string] of SumStats = table();
 
 # Store of reducers indexed on the data point stream id.
 global reducer_store: table[string] of set[Reducer] = table();
@@ -166,10 +179,10 @@ global thresholds_store: table[string, Key] of bool = table();
 # key values are updated and the new val is given as the `val` argument.
 # It's only prototyped here because cluster and non-cluster have separate 
 # implementations.
-global data_added: function(m: Measurement, key: Key, result: Result);
+global data_added: function(m: SumStats, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
-global add_to_reducer_hook: hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal);
+global add_to_reducer_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
 # Prototype the hook point for plugins to initialize any result values.
 global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
 # Prototype the hook point for plugins to merge Results.
@@ -177,7 +190,7 @@ global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: Res
 
 # Event that is used to "finish" measurements and adapt the measurement
 # framework for clustered or non-clustered usage.
-global finish_epoch: event(m: Measurement);
+global finish_epoch: event(m: SumStats);
 
 function key2str(key: Key): string
 	{
@@ -186,7 +199,7 @@ function key2str(key: Key): string
 		out = fmt("%shost=%s", out, key$host);
 	if ( key?$str )
 		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", key$str);
-	return fmt("measurement_key(%s)", out);
+	return fmt("sumstats_key(%s)", out);
 	}
 
 function init_resultval(r: Reducer): ResultVal
@@ -200,17 +213,12 @@ function compose_resultvals(rv1: ResultVal, rv2: ResultVal): ResultVal
 	{
 	local result: ResultVal;
 
-	# Merge $begin (take the earliest one)
 	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
-
-	# Merge $end (take the latest one)
 	result$end = (rv1$end > rv2$end) ? rv1$end : rv2$end;
-
-	# Merge $num
 	result$num = rv1$num + rv2$num;
 
+	# Run the plugin composition hooks.
 	hook compose_resultvals_hook(result, rv1, rv2);
-
 	return result;
 	}
 
@@ -243,59 +251,59 @@ function compose_results(r1: Result, r2: Result): Result
 	}
 
 
-function reset(m: Measurement)
+function reset(ss: SumStat)
 	{
-	if ( m$id in result_store )
-		delete result_store[m$id];
+	if ( ss$id in result_store )
+		delete result_store[ss$id];
 
-	result_store[m$id] = table();
-	threshold_tracker[m$id] = table();
+	result_store[ss$id] = table();
+	threshold_tracker[ss$id] = table();
 	}
 
-function create(m: Measurement)
+function create(ss: SumStat)
 	{
-	if ( (m?$threshold || m?$threshold_series) && ! m?$threshold_val )
+	if ( (ss?$threshold || ss?$threshold_series) && ! ss?$threshold_val )
 		{
-		Reporter::error("Measurement given a threshold with no $threshold_val function");
+		Reporter::error("SumStats given a threshold with no $threshold_val function");
 		}
 
-	if ( ! m?$id )
-		m$id=unique_id("");
-	threshold_tracker[m$id] = table();
-	measurement_store[m$id] = m;
+	if ( ! ss?$id )
+		ss$id=unique_id("");
+	threshold_tracker[ss$id] = table();
+	stats_store[ss$id] = ss;
 
-	for ( reducer in m$reducers )
+	for ( reducer in ss$reducers )
 		{
-		reducer$mid = m$id;
+		reducer$mid = ss$id;
 		if ( reducer$stream !in reducer_store )
 			reducer_store[reducer$stream] = set();
 		add reducer_store[reducer$stream][reducer];
 		}
 
-	reset(m);
-	schedule m$epoch { Measurement::finish_epoch(m) };
+	reset(ss);
+	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 
-function add_data(id: string, key: Key, point: DataPoint)
+function observe(id: string, key: Key, obs: Observation)
 	{
-	# Try to add the data to all of the defined reducers.
 	if ( id !in reducer_store )
 		return;
 
+	# Try to add the data to all of the defined reducers.
 	for ( r in reducer_store[id] )
 		{
 		# If this reducer has a predicate, run the predicate 
 		# and skip this key if the predicate return false.
-		if ( r?$pred && ! r$pred(key, point) )
+		if ( r?$pred && ! r$pred(key, obs) )
 			next;
 		
 		if ( r?$normalize_key )
 			key = r$normalize_key(copy(key));
 		
-		local m = measurement_store[r$mid];
+		local ss = stats_store[r$mid];
 		
 		if ( r$mid !in result_store )
-			result_store[m$id] = table();
+			result_store[ss$id] = table();
 		local results = result_store[r$mid];
 
 		if ( key !in results )
@@ -312,56 +320,56 @@ function add_data(id: string, key: Key, point: DataPoint)
 
 		# If a string was given, fall back to 1.0 as the value.
 		local val = 1.0;
-		if ( point?$num || point?$dbl )
-			val = point?$dbl ? point$dbl : point$num;
+		if ( obs?$num || obs?$dbl )
+			val = obs?$dbl ? obs$dbl : obs$num;
 
-		hook add_to_reducer_hook(r, val, point, result_val);
-		data_added(m, key, result);
+		hook add_to_reducer_hook(r, val, obs, result_val);
+		data_added(ss, key, result);
 		}
 	}
 
 # This function checks if a threshold has been crossed.  It is also used as a method to implement 
 # mid-break-interval threshold crossing detection for cluster deployments.
-function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct: double): bool
+function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: double): bool
 	{
-	if ( ! (m?$threshold || m?$threshold_series) )
+	if ( ! (ss?$threshold || ss?$threshold_series) )
 		return F;
 
 	# Add in the extra ResultVals to make threshold_vals easier to write.
-	if ( |m$reducers| != |result| )
+	if ( |ss$reducers| != |result| )
 		{
-		for ( reducer in m$reducers )
+		for ( reducer in ss$reducers )
 			{
 			if ( reducer$stream !in result )
 				result[reducer$stream] = init_resultval(reducer);
 			}
 		}
 
-	local watch = m$threshold_val(key, result);
+	local watch = ss$threshold_val(key, result);
 
 	if ( modify_pct < 1.0 && modify_pct > 0.0 )
 		watch = double_to_count(floor(watch/modify_pct));
 
-	if ( m$id !in threshold_tracker )
-		threshold_tracker[m$id] = table();
-	local t_tracker = threshold_tracker[m$id];
+	if ( ss$id !in threshold_tracker )
+		threshold_tracker[ss$id] = table();
+	local t_tracker = threshold_tracker[ss$id];
 
 	if ( key !in t_tracker )
 		{
 		local ttmp: Thresholding;
 		t_tracker[key] = ttmp;
 		}
-	local tt = threshold_tracker[m$id][key];
+	local tt = t_tracker[key];
 
-	if ( m?$threshold && ! tt$is_threshold_crossed && watch >= m$threshold )
+	if ( ss?$threshold && ! tt$is_threshold_crossed && watch >= ss$threshold )
 		{
 		# Value crossed the threshold.
 		return T;
 		}
 
-	if ( m?$threshold_series &&
-	     |m$threshold_series| >= tt$threshold_series_index &&
-	     watch >= m$threshold_series[tt$threshold_series_index] )
+	if ( ss?$threshold_series &&
+	     |ss$threshold_series| >= tt$threshold_series_index &&
+	     watch >= ss$threshold_series[tt$threshold_series_index] )
 		{
 		# A threshold series was given and the value crossed the next 
 		# value in the series.
@@ -371,28 +379,28 @@ function check_thresholds(m: Measurement, key: Key, result: Result, modify_pct:
 	return F;
 	}
 
-function threshold_crossed(m: Measurement, key: Key, result: Result)
+function threshold_crossed(ss: SumStat, key: Key, result: Result)
 	{
 	# If there is no callback, there is no point in any of this.
-	if ( ! m?$threshold_crossed )
+	if ( ! ss?$threshold_crossed )
 		return;
 
 	# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
-	if ( |m$reducers| != |result| )
+	if ( |ss$reducers| != |result| )
 		{
-		for ( reducer in m$reducers )
+		for ( reducer in ss$reducers )
 			{
 			if ( reducer$stream !in result )
 				result[reducer$stream] = init_resultval(reducer);
 			}
 		}
 
-	m$threshold_crossed(key, result);
-	local tt = threshold_tracker[m$id][key];
+	ss$threshold_crossed(key, result);
+	local tt = threshold_tracker[ss$id][key];
 	tt$is_threshold_crossed = T;
 
 	# Bump up to the next threshold series index if a threshold series is being used.
-	if ( m?$threshold_series )
+	if ( ss?$threshold_series )
 		++tt$threshold_series_index;
 	}
 
diff --git a/scripts/base/frameworks/measurement/non-cluster.bro b/scripts/base/frameworks/sumstats/non-cluster.bro
similarity index 58%
rename from scripts/base/frameworks/measurement/non-cluster.bro
rename to scripts/base/frameworks/sumstats/non-cluster.bro
index 35ff9dc935..6163548ec6 100644
--- a/scripts/base/frameworks/measurement/non-cluster.bro
+++ b/scripts/base/frameworks/sumstats/non-cluster.bro
@@ -1,8 +1,8 @@
 @load ./main
 
-module Measurement;
+module SumStats;
 
-event Measurement::finish_epoch(m: Measurement)
+event SumStats::finish_epoch(m: SumStats)
 	{
 	if ( m$id in result_store )
 		{
@@ -13,11 +13,11 @@ event Measurement::finish_epoch(m: Measurement)
 		reset(m);
 		}
 
-	schedule m$epoch { Measurement::finish_epoch(m) };
+	schedule m$epoch { SumStats::finish_epoch(m) };
 	}
 	
 	
-function data_added(m: Measurement, key: Key, result: Result)
+function data_added(m: SumStats, key: Key, result: Result)
 	{
 	if ( check_thresholds(m, key, result, 1.0) )
 		threshold_crossed(m, key, result);
diff --git a/scripts/base/frameworks/measurement/plugins/__load__.bro b/scripts/base/frameworks/sumstats/plugins/__load__.bro
similarity index 85%
rename from scripts/base/frameworks/measurement/plugins/__load__.bro
rename to scripts/base/frameworks/sumstats/plugins/__load__.bro
index 0d4c2ed302..d739af29a7 100644
--- a/scripts/base/frameworks/measurement/plugins/__load__.bro
+++ b/scripts/base/frameworks/sumstats/plugins/__load__.bro
@@ -2,7 +2,7 @@
 @load ./max
 @load ./min
 @load ./sample
+@load ./variance
 @load ./std-dev
 @load ./sum
 @load ./unique
-@load ./variance
\ No newline at end of file
diff --git a/scripts/base/frameworks/measurement/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro
similarity index 83%
rename from scripts/base/frameworks/measurement/plugins/average.bro
rename to scripts/base/frameworks/sumstats/plugins/average.bro
index 9a3938640e..002a0147ff 100644
--- a/scripts/base/frameworks/measurement/plugins/average.bro
+++ b/scripts/base/frameworks/sumstats/plugins/average.bro
@@ -1,6 +1,6 @@
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
 	{
 	if ( AVERAGE in r$apply )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro
similarity index 81%
rename from scripts/base/frameworks/measurement/plugins/max.bro
rename to scripts/base/frameworks/sumstats/plugins/max.bro
index 816d249de3..0e377ff320 100644
--- a/scripts/base/frameworks/measurement/plugins/max.bro
+++ b/scripts/base/frameworks/sumstats/plugins/max.bro
@@ -1,6 +1,6 @@
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
 	{
 	if ( MAX in r$apply )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro
similarity index 81%
rename from scripts/base/frameworks/measurement/plugins/min.bro
rename to scripts/base/frameworks/sumstats/plugins/min.bro
index 910d2c76d7..5e1e3fbbb7 100644
--- a/scripts/base/frameworks/measurement/plugins/min.bro
+++ b/scripts/base/frameworks/sumstats/plugins/min.bro
@@ -1,6 +1,6 @@
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
 	{
 	if ( MIN in r$apply )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro
similarity index 63%
rename from scripts/base/frameworks/measurement/plugins/sample.bro
rename to scripts/base/frameworks/sumstats/plugins/sample.bro
index 0187946d61..a694296727 100644
--- a/scripts/base/frameworks/measurement/plugins/sample.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sample.bro
@@ -1,35 +1,35 @@
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 @load base/utils/queue
 
-module Measurement;
+module SumStats;
 
 export {
 	redef record Reducer += {
-		## A number of sample DataPoints to collect.
+		## A number of sample Observations to collect.
 		samples: count &default=0;
 	};
 
 	redef record ResultVal += {
 		## This is the queue where samples
 		## are maintained.  Use the 
-		## :bro:see:`Measurement::get_samples` function
+		## :bro:see:`SumStats::get_samples` function
 		## to get a vector of the samples.
 		samples: Queue::Queue &optional;
 	};
 
-	## Get a vector of sample DataPoint values from a ResultVal.
-	global get_samples: function(rv: ResultVal): vector of DataPoint;
+	## Get a vector of sample Observation values from a ResultVal.
+	global get_samples: function(rv: ResultVal): vector of Observation;
 }
 
-function get_samples(rv: ResultVal): vector of DataPoint
+function get_samples(rv: ResultVal): vector of Observation
 	{
-	local s: vector of DataPoint = vector();
+	local s: vector of Observation = vector();
 	if ( rv?$samples )
 		Queue::get_vector(rv$samples, s);
 	return s;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
 	{
 	if ( r$samples > 0 )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
similarity index 80%
rename from scripts/base/frameworks/measurement/plugins/std-dev.bro
rename to scripts/base/frameworks/sumstats/plugins/std-dev.bro
index 63dcc62d4b..af6eea8cdc 100644
--- a/scripts/base/frameworks/measurement/plugins/std-dev.bro
+++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
@@ -1,7 +1,7 @@
 @load ./variance
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -22,7 +22,7 @@ function calc_std_dev(rv: ResultVal)
 	}
 
 # This depends on the variance plugin which uses priority -5
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal) &priority=-10
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-10
 	{
 	if ( STD_DEV in r$apply )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro
similarity index 74%
rename from scripts/base/frameworks/measurement/plugins/sum.bro
rename to scripts/base/frameworks/sumstats/plugins/sum.bro
index 2ada26e1d0..572402d6c5 100644
--- a/scripts/base/frameworks/measurement/plugins/sum.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sum.bro
@@ -1,6 +1,6 @@
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -14,13 +14,13 @@ export {
 		sum: double &default=0.0;
 	};
 
-	type threshold_function: function(key: Measurement::Key, result: Measurement::Result): count;
+	type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
 	global sum_threshold: function(data_id: string): threshold_function;
 }
 
 function sum_threshold(data_id: string): threshold_function
 	{
-	return function(key: Measurement::Key, result: Measurement::Result): count
+	return function(key: SumStats::Key, result: SumStats::Result): count
 		{
 		print fmt("data_id: %s", data_id);
 		print result;
@@ -34,7 +34,7 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
 	{
 	if ( SUM in r$apply )
 		rv$sum += val;
diff --git a/scripts/base/frameworks/measurement/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro
similarity index 86%
rename from scripts/base/frameworks/measurement/plugins/unique.bro
rename to scripts/base/frameworks/sumstats/plugins/unique.bro
index f1027157a7..f260148af4 100644
--- a/scripts/base/frameworks/measurement/plugins/unique.bro
+++ b/scripts/base/frameworks/sumstats/plugins/unique.bro
@@ -1,6 +1,6 @@
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -20,10 +20,10 @@ redef record ResultVal += {
 	# because we don't want to trust that we can inspect the values 
 	# since we will like move to a probalistic data structure in the future.
 	# TODO: in the future this will optionally be a hyperloglog structure
-	unique_vals: set[DataPoint] &optional;
+	unique_vals: set[Observation] &optional;
 };
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
 	{
 	if ( UNIQUE in r$apply )
 		{
diff --git a/scripts/base/frameworks/measurement/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro
similarity index 91%
rename from scripts/base/frameworks/measurement/plugins/variance.bro
rename to scripts/base/frameworks/sumstats/plugins/variance.bro
index 3c2223430d..a26a2d4095 100644
--- a/scripts/base/frameworks/measurement/plugins/variance.bro
+++ b/scripts/base/frameworks/sumstats/plugins/variance.bro
@@ -1,7 +1,7 @@
 @load ./average
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module Measurement;
+module SumStats;
 
 export {
 	redef enum Calculation += { 
@@ -29,7 +29,7 @@ function calc_variance(rv: ResultVal)
 	}
 
 # Reduced priority since this depends on the average
-hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal) &priority=-5
+hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-5
 	{
 	if ( VARIANCE in r$apply )
 		{
diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro
index d782c4fa37..f4112efde0 100644
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@@ -5,6 +5,7 @@
 ##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
 ##! is not attempted if the connection size analyzer isn't enabled.
 
+@load base/protocols/conn
 @load base/frameworks/notice
 @load base/utils/site
 @load base/utils/thresholds
@@ -115,7 +116,7 @@ function check_ssh_connection(c: connection, done: bool)
 		     # Responder must have sent fewer than 40 packets.
 		     c$resp$num_pkts < 40 &&
 		     # If there was a content gap we can't reliably do this heuristic.
-		     c$conn$missed_bytes == 0)# && 
+		     c?$conn && c$conn$missed_bytes == 0)# && 
 		     # Only "normal" connections can count.
 		     #c$conn?$conn_state && c$conn$conn_state in valid_states )
 			{
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index 967d5eb88f..53f210b46a 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -1,8 +1,8 @@
 @load base/protocols/http
 @load base/protocols/ssl
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
-module AppMeasurement;
+module AppStats;
 
 export {
 	redef enum Log::ID += { LOG };
@@ -32,60 +32,60 @@ redef record connection += {
 
 event bro_init() &priority=3
 	{
-	Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
+	Log::create_stream(AppSumStats::LOG, [$columns=Info]);
 
-	local r1: Measurement::Reducer = [$stream="apps.bytes", $apply=set(Measurement::SUM)];
-	local r2: Measurement::Reducer = [$stream="apps.hits",  $apply=set(Measurement::UNIQUE)];
-	Measurement::create([$epoch=break_interval, 
-	                     $reducers=set(r1, r2),
-	                     $epoch_finished(data: Measurement::ResultTable) = 
-	                     	{
-	                     	local l: Info;
-	                     	l$ts = network_time();
-	                     	l$ts_delta = break_interval;
-	                     	for ( key in data )
-	                     		{
-	                     		local result = data[key];
-	                     		l$app        = key$str;
-	                     		l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
-	                     		l$hits       = result["apps.hits"]$num;
-	                     		l$uniq_hosts = result["apps.hits"]$unique;
-	                     		Log::write(LOG, l);
-	                     		}
-	                     	}]);
+	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=break_interval, 
+	                  $reducers=set(r1, r2),
+	                  $epoch_finished(data: SumStats::ResultTable) = 
+	                  	{
+	                  	local l: Info;
+	                  	l$ts = network_time();
+	                  	l$ts_delta = break_interval;
+	                  	for ( key in data )
+	                  		{
+	                  		local result = data[key];
+	                  		l$app        = key$str;
+	                  		l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
+	                  		l$hits       = result["apps.hits"]$num;
+	                  		l$uniq_hosts = result["apps.hits"]$unique;
+	                  		Log::write(LOG, l);
+	                  		}
+	                  	}]);
 	}
 
-function do_measurement(id: conn_id, hostname: string, size: count)
+function add_sumstats(id: conn_id, hostname: string, size: count)
 	{
 	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
 		{
-		Measurement::add_data("apps.bytes", [$str="youtube"], [$num=size]);
-		Measurement::add_data("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
+		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
 		{
-		Measurement::add_data("apps.bytes", [$str="facebook"], [$num=size]);
-		Measurement::add_data("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
+		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /\.google\.com$/ in hostname && size > 20 ) 
 		{
-		Measurement::add_data("apps.bytes", [$str="google"], [$num=size]);
-		Measurement::add_data("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
+		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /\.nflximg\.com$/ in hostname && size > 200*1024 ) 
 		{
-		Measurement::add_data("apps.bytes", [$str="netflix"], [$num=size]);
-		Measurement::add_data("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
+		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
 		{
-		Measurement::add_data("apps.bytes", [$str="pandora"], [$num=size]);
-		Measurement::add_data("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
+		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
 		}
 	else if ( /\.gmail\.com$/ in hostname && size > 20 )
 		{
-		Measurement::add_data("apps.bytes", [$str="gmail"], [$num=size]);
-		Measurement::add_data("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
+		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
 		}
 }
 
@@ -99,11 +99,11 @@ event ssl_established(c: connection)
 event connection_finished(c: connection)
 	{
 	if ( c?$resp_hostname )
-		do_measurement(c$id, c$resp_hostname, c$resp$size);
+		add_sumstats(c$id, c$resp_hostname, c$resp$size);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if( rec?$host )
-		do_measurement(rec$id, rec$host, rec$response_body_len);
+		add_sumstats(rec$id, rec$host, rec$response_body_len);
 	}
diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index 1b9f369ca5..9ac0f5c2f9 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -2,7 +2,7 @@
 ##! toward hosts that have sent low TTL packets.
 ##! It generates a notice when the number of ICMP Time Exceeded 
 ##! messages for a source-destination pair exceeds threshold
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 @load base/frameworks/signatures
 @load-sigs ./detect-low-ttls.sig
 
@@ -53,41 +53,41 @@ event bro_init() &priority=5
 	{
 	Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
 
-	local r1: Measurement::Reducer = [$stream="traceroute.time_exceeded", $apply=set(Measurement::UNIQUE)];
-	local r2: Measurement::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(Measurement::SUM)];
-	Measurement::create([$epoch=icmp_time_exceeded_interval,
-	                     $reducers=set(r1, r2),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	# Give a threshold value of zero depending on if the host 
-	                     	# sends a low ttl packet.
-	                     	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
-	                     		return 0;
-	                     	else
-	                     		return result["traceroute.time_exceeded"]$unique;
-	                     	},
-	                     $threshold=icmp_time_exceeded_threshold,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	local parts = split1(key$str, /-/);
-	                     	local src = to_addr(parts[1]);
-	                     	local dst = to_addr(parts[2]);
-	                     	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
-	                     	NOTICE([$note=Traceroute::Detected,
-	                     	        $msg=fmt("%s seems to be running traceroute", src),
-	                     	        $src=src, $dst=dst,
-	                     	        $identifier=cat(src)]);
-	                     	}]);
+	local r1: SumStats::Reducer = [$stream="traceroute.time_exceeded", $apply=set(SumStats::UNIQUE)];
+	local r2: SumStats::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=icmp_time_exceeded_interval,
+	                  $reducers=set(r1, r2),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	# Give a threshold value of zero depending on if the host 
+	                  	# sends a low ttl packet.
+	                  	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
+	                  		return 0;
+	                  	else
+	                  		return result["traceroute.time_exceeded"]$unique;
+	                  	},
+	                  $threshold=icmp_time_exceeded_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local parts = split1(key$str, /-/);
+	                  	local src = to_addr(parts[1]);
+	                  	local dst = to_addr(parts[2]);
+	                  	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
+	                  	NOTICE([$note=Traceroute::Detected,
+	                  	        $msg=fmt("%s seems to be running traceroute", src),
+	                  	        $src=src, $dst=dst,
+	                  	        $identifier=cat(src)]);
+	                  	}]);
 	}
 
 # Low TTL packets are detected with a signature.
 event signature_match(state: signature_state, msg: string, data: string)
 	{
 	if ( state$sig_id == /traceroute-detector.*/ )
-		Measurement::add_data("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
+		SumStats::observe("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
 	}
 
 event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
 	{
-	Measurement::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
+	SumStats::observe("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
 	}
diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 2ea1e9c0fe..9a95cf9917 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -5,7 +5,7 @@
 ##!            All the authors of the old scan.bro
 
 @load base/frameworks/notice
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
 @load base/utils/time
 
@@ -52,7 +52,7 @@ export {
 }
 
 
-#function check_addr_scan_threshold(key: Measurement::Key, val: Measurement::Result): bool
+#function check_addr_scan_threshold(key: SumStats::Key, val: SumStats::Result): bool
 #	{
 #	# We don't need to do this if no custom thresholds are defined.
 #	if ( |addr_scan_custom_thresholds| == 0 )
@@ -65,54 +65,54 @@ export {
 
 event bro_init() &priority=5
 	{
-	local r1: Measurement::Reducer = [$stream="scan.addr.fail", $apply=set(Measurement::UNIQUE)];
-	Measurement::create([$epoch=addr_scan_interval,
-	                     $reducers=set(r1),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	return double_to_count(result["scan.addr.fail"]$unique);
-	                     	},
-	                     #$threshold_func=check_addr_scan_threshold,
-	                     $threshold=addr_scan_threshold,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	local r = result["scan.addr.fail"];
-	                     	local side = Site::is_local_addr(key$host) ? "local" : "remote";
-	                     	local dur = duration_to_mins_secs(r$end-r$begin);
-	                     	local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
-	                     	NOTICE([$note=Address_Scan,
-	                     	        $src=key$host,
-	                     	        $p=to_port(key$str),
-	                     	        $sub=side,
-	                     	        $msg=message,
-	                     	        $identifier=cat(key$host)]);
-	                     	}]); 
+	local r1: SumStats::Reducer = [$stream="scan.addr.fail", $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=addr_scan_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return double_to_count(result["scan.addr.fail"]$unique);
+	                  	},
+	                  #$threshold_func=check_addr_scan_threshold,
+	                  $threshold=addr_scan_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["scan.addr.fail"];
+	                  	local side = Site::is_local_addr(key$host) ? "local" : "remote";
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
+	                  	NOTICE([$note=Address_Scan,
+	                  	        $src=key$host,
+	                  	        $p=to_port(key$str),
+	                  	        $sub=side,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]); 
 
 	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
-	local r2: Measurement::Reducer = [$stream="scan.port.fail", $apply=set(Measurement::UNIQUE)];
-	Measurement::create([$epoch=port_scan_interval,
-	                     $reducers=set(r2),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return double_to_count(result["scan.port.fail"]$unique);
-	                     	},
-	                     $threshold=port_scan_threshold,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
-	                          {
-	                          local r = result["scan.port.fail"];
-	                          local side = Site::is_local_addr(key$host) ? "local" : "remote";
-	                          local dur = duration_to_mins_secs(r$end-r$begin);
-	                          local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
-	                          NOTICE([$note=Port_Scan, 
-	                                  $src=key$host,
-	                                  $dst=to_addr(key$str),
-	                                  $sub=side,
-	                                  $msg=message,
-	                                  $identifier=cat(key$host)]);
-	                          }]); 
+	local r2: SumStats::Reducer = [$stream="scan.port.fail", $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=port_scan_interval,
+	                  $reducers=set(r2),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return double_to_count(result["scan.port.fail"]$unique);
+	                  	},
+	                  $threshold=port_scan_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["scan.port.fail"];
+	                  	local side = Site::is_local_addr(key$host) ? "local" : "remote";
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
+	                  	NOTICE([$note=Port_Scan, 
+	                  	        $src=key$host,
+	                  	        $dst=to_addr(key$str),
+	                  	        $sub=side,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]); 
 	}
 
-function add_metrics(id: conn_id, reverse: bool)
+function add_sumstats(id: conn_id, reverse: bool)
 	{
 	local scanner      = id$orig_h;
 	local victim       = id$resp_h;
@@ -150,10 +150,10 @@ function add_metrics(id: conn_id, reverse: bool)
 	#	return F;
 	
 	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
-		Measurement::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
+		SumStats::observe("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
 
 	if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
-		Measurement::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
+		SumStats::observe("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
 	}
 
 function is_failed_conn(c: connection): bool
@@ -193,7 +193,7 @@ event connection_attempt(c: connection)
 	if ( "H" in c$history )
 		is_reverse_scan = T;
 	
-	add_metrics(c$id, is_reverse_scan);
+	add_sumstats(c$id, is_reverse_scan);
 	}
 
 ## Generated for a rejected TCP connection. This event 
@@ -206,7 +206,7 @@ event connection_rejected(c: connection)
 	if ( "s" in c$history )
 		is_reverse_scan = T;
 	
-	add_metrics(c$id, is_reverse_scan);
+	add_sumstats(c$id, is_reverse_scan);
 	}
 
 ## Generated when an endpoint aborted a TCP connection. 
@@ -215,16 +215,16 @@ event connection_rejected(c: connection)
 event connection_reset(c: connection)
 	{
 	if ( is_failed_conn(c) )
-		add_metrics(c$id, F);
+		add_sumstats(c$id, F);
 	else if ( is_reverse_failed_conn(c) )
-		add_metrics(c$id, T);
+		add_sumstats(c$id, T);
 	}
 
 ## Generated for each still-open connection when Bro terminates.
 event connection_pending(c: connection)
 	{
 	if ( is_failed_conn(c) )
-		add_metrics(c$id, F);
+		add_sumstats(c$id, F);
 	else if ( is_reverse_failed_conn(c) )
-		add_metrics(c$id, T);
+		add_sumstats(c$id, T);
 	}
diff --git a/scripts/policy/protocols/ftp/detect-bruteforcing.bro b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
index bcf7a59d06..e6c44ddb64 100644
--- a/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ftp/detect-bruteforcing.bro
@@ -1,6 +1,6 @@
 
 @load base/protocols/ftp
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 
 @load base/utils/time
 
@@ -25,25 +25,25 @@ export {
 
 event bro_init()
 	{
-	local r1: Measurement::Reducer = [$stream="ftp.failed_auth", $apply=set(Measurement::UNIQUE)];
-	Measurement::create([$epoch=bruteforce_measurement_interval,
-	                     $reducers=set(r1),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return result["ftp.failed_auth"]$num;
-	                     	},
-	                     $threshold=bruteforce_threshold,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
-	                     	{
-	                     	local r = result["ftp.failed_auth"];
-	                     	local dur = duration_to_mins_secs(r$end-r$begin);
-	                     	local plural = r$unique>1 ? "s" : "";
-	                     	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
-	                     	NOTICE([$note=FTP::Bruteforcing, 
-	                     	        $src=key$host,
-	                     	        $msg=message,
-	                     	        $identifier=cat(key$host)]);
-	                     	}]);
+	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=bruteforce_measurement_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return result["ftp.failed_auth"]$num;
+	                  	},
+	                  $threshold=bruteforce_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	local r = result["ftp.failed_auth"];
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local plural = r$unique>1 ? "s" : "";
+	                  	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                  	NOTICE([$note=FTP::Bruteforcing, 
+	                  	        $src=key$host,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
 	}
 
 event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
@@ -52,6 +52,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
 	if ( cmd == "USER" || cmd == "PASS" )
 		{
 		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
-			Measurement::add_data("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+			SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
 		}
 	}
\ No newline at end of file
diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro
index f5e15c5505..daec0b0fb0 100644
--- a/scripts/policy/protocols/http/detect-sqli.bro
+++ b/scripts/policy/protocols/http/detect-sqli.bro
@@ -1,7 +1,7 @@
 ##! SQL injection attack detection in HTTP.
 
 @load base/frameworks/notice
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 @load base/protocols/http
 
 module HTTP;
@@ -50,7 +50,7 @@ export {
 		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
-function format_sqli_samples(samples: vector of Measurement::DataPoint): string
+function format_sqli_samples(samples: vector of SumStats::Observation): string
 	{
 	local ret = "SQL Injection samples\n---------------------";
 	for ( i in samples )
@@ -63,41 +63,41 @@ event bro_init() &priority=3
 	# Add filters to the metrics so that the metrics framework knows how to 
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
-	local r1: Measurement::Reducer = [$stream="http.sqli.attacker", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
-	Measurement::create([$epoch=sqli_requests_interval,
-	                     $reducers=set(r1),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return double_to_count(result["http.sqli.attacker"]$sum);
-	                     	},
-	                     $threshold=sqli_requests_threshold,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
-	                     	{
-	                     	local r = result["http.sqli.attacker"];
-	                     	NOTICE([$note=SQL_Injection_Attacker,
-	                     	        $msg="An SQL injection attacker was discovered!",
-	                     	        $email_body_sections=vector(format_sqli_samples(Measurement::get_samples(r))),
-	                     	        $src=key$host,
-	                     	        $identifier=cat(key$host)]);
-	                     	}]);
+	local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples];
+	SumStats::create([$epoch=sqli_requests_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return double_to_count(result["http.sqli.attacker"]$sum);
+	                  	},
+	                  $threshold=sqli_requests_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	local r = result["http.sqli.attacker"];
+	                  	NOTICE([$note=SQL_Injection_Attacker,
+	                  	        $msg="An SQL injection attacker was discovered!",
+	                  	        $email_body_sections=vector(format_sqli_samples(SumStats::get_samples(r))),
+	                  	        $src=key$host,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
 
-	local r2: Measurement::Reducer = [$stream="http.sqli.victim", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
-	Measurement::create([$epoch=sqli_requests_interval,
-	                     $reducers=set(r2),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return double_to_count(result["http.sqli.victim"]$sum);
-	                     	},
-	                     $threshold=sqli_requests_threshold,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
-	                     	{
-	                     	local r = result["http.sqli.victim"];
-	                     	NOTICE([$note=SQL_Injection_Victim,
-	                     	        $msg="An SQL injection victim was discovered!",
-	                     	        $email_body_sections=vector(format_sqli_samples(Measurement::get_samples(r))),
-	                     	        $src=key$host,
-	                     	        $identifier=cat(key$host)]);
-	                     	}]);
+	local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples];
+	SumStats::create([$epoch=sqli_requests_interval,
+	                  $reducers=set(r2),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return double_to_count(result["http.sqli.victim"]$sum);
+	                  	},
+	                  $threshold=sqli_requests_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	local r = result["http.sqli.victim"];
+	                  	NOTICE([$note=SQL_Injection_Victim,
+	                  	        $msg="An SQL injection victim was discovered!",
+	                  	        $email_body_sections=vector(format_sqli_samples(SumStats::get_samples(r))),
+	                  	        $src=key$host,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -107,7 +107,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Measurement::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
-		Measurement::add_data("http.sqli.victim",   [$host=c$id$resp_h], [$str=original_URI]);
+		SumStats::observe("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
+		SumStats::observe("http.sqli.victim",   [$host=c$id$resp_h], [$str=original_URI]);
 		}
 	}
diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro
index 19a3220805..04e1185e25 100644
--- a/scripts/policy/protocols/smtp/metrics.bro
+++ b/scripts/policy/protocols/smtp/metrics.bro
@@ -18,12 +18,12 @@ event bro_init() &priority=5
 	{
 	Metrics::add_filter("smtp.mailfrom", [$every=breaks,
 	                                      $measure=set(Metrics::SUM),
-	                                      $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
+	                                      $pred(index: Metrics::Index, data: Metrics::Observation) = { 
 	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
 	                                      }]);
 	Metrics::add_filter("smtp.messages", [$every=breaks,
 	                                      $measure=set(Metrics::SUM),
-	                                      $pred(index: Metrics::Index, data: Metrics::DataPoint) = { 
+	                                      $pred(index: Metrics::Index, data: Metrics::Observation) = { 
 	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
 	                                      }]);
 	}
diff --git a/scripts/policy/protocols/ssh/detect-bruteforcing.bro b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
index cf2d4030fd..82c0bb0f08 100644
--- a/scripts/policy/protocols/ssh/detect-bruteforcing.bro
+++ b/scripts/policy/protocols/ssh/detect-bruteforcing.bro
@@ -2,7 +2,7 @@
 ##! bruteforcing over SSH.
 
 @load base/protocols/ssh
-@load base/frameworks/measurement
+@load base/frameworks/sumstats
 @load base/frameworks/notice
 @load base/frameworks/intel
 
@@ -42,27 +42,27 @@ export {
 
 event bro_init()
 	{
-	local r1: Measurement::Reducer = [$stream="ssh.login.failure", $apply=set(Measurement::SUM)];
-	Measurement::create([$epoch=guessing_timeout,
-	                     $reducers=set(r1),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return double_to_count(result["ssh.login.failure"]$sum);
-	                     	},
-	                     $threshold=password_guesses_limit,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
-	                     	{
-	                     	local r = result["ssh.login.failure"];
-	                     	# Generate the notice.
-	                     	NOTICE([$note=Password_Guessing, 
-	                     	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
-	                     	        $src=key$host,
-	                     	        $identifier=cat(key$host)]);
-	                     	# Insert the guesser into the intel framework.
-	                     	Intel::insert([$host=key$host,
-	                     	               $meta=[$source="local", 
-	                     	                      $desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]);
-	                     	}]);
+	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=guessing_timeout,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return double_to_count(result["ssh.login.failure"]$sum);
+	                  	},
+	                  $threshold=password_guesses_limit,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	local r = result["ssh.login.failure"];
+	                  	# Generate the notice.
+	                  	NOTICE([$note=Password_Guessing, 
+	                  	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
+	                  	        $src=key$host,
+	                  	        $identifier=cat(key$host)]);
+	                  	# Insert the guesser into the intel framework.
+	                  	Intel::insert([$host=key$host,
+	                  	               $meta=[$source="local", 
+	                  	                      $desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]);
+	                  	}]);
 	}
 
 event SSH::heuristic_successful_login(c: connection)
@@ -82,5 +82,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		Measurement::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
+		SumStats::observe("ssh.login.failure", [$host=id$orig_h], [$num=1]);
 	}
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.basic-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.basic-cluster/manager-1..stdout
similarity index 100%
rename from testing/btest/Baseline/scripts.base.frameworks.measurement.basic-cluster/manager-1..stdout
rename to testing/btest/Baseline/scripts.base.frameworks.sumstats.basic-cluster/manager-1..stdout
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.basic/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.basic/.stdout
similarity index 100%
rename from testing/btest/Baseline/scripts.base.frameworks.measurement.basic/.stdout
rename to testing/btest/Baseline/scripts.base.frameworks.sumstats.basic/.stdout
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.cluster-intermediate-update/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout
similarity index 100%
rename from testing/btest/Baseline/scripts.base.frameworks.measurement.cluster-intermediate-update/manager-1..stdout
rename to testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout
diff --git a/testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout
similarity index 100%
rename from testing/btest/Baseline/scripts.base.frameworks.measurement.thresholding/.stdout
rename to testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout
diff --git a/testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro b/testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro
deleted file mode 100644
index e2f5e4e7d5..0000000000
--- a/testing/btest/scripts/base/frameworks/measurement/basic-cluster.bro
+++ /dev/null
@@ -1,83 +0,0 @@
-# @TEST-SERIALIZE: comm
-#
-# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 15
-
-# @TEST-EXEC: btest-diff manager-1/.stdout
-
-@TEST-START-FILE cluster-layout.bro
-redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
-	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
-	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
-};
-@TEST-END-FILE
-
-redef Log::default_rotation_interval = 0secs;
-
-global n = 0;
-
-event bro_init() &priority=5
-	{
-	local r1: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM, Measurement::MIN, Measurement::MAX, Measurement::AVERAGE, Measurement::STD_DEV, Measurement::VARIANCE, Measurement::UNIQUE)];
-	Measurement::create([$epoch=5secs,
-	                     $reducers=set(r1),
-	                     $epoch_finished(rt: Measurement::ResultTable) =
-	                     	{
-	                     	for ( key in rt )
-	                     		{
-	                     		local r = rt[key]["test.metric"];
-	                     		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
-	                     		}
-
-	                     	terminate();
-	                     	}
-	                     ]);
-	}
-
-event remote_connection_closed(p: event_peer)
-	{
-	terminate();
-	}
-
-global ready_for_data: event();
-redef Cluster::manager2worker_events += /^ready_for_data$/;
-
-event ready_for_data()
-	{
-	if ( Cluster::node == "worker-1" )
-		{
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=34]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
-		Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=1]);
-		Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=54]);
-		}
-	if ( Cluster::node == "worker-2" )
-		{
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=75]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=30]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=57]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=52]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=61]);
-		Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=95]);
-		Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=5]);
-		Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=91]);
-		Measurement::add_data("test.metric", [$host=10.10.10.10], [$num=5]);
-		}
-	}
-
-@if ( Cluster::local_node_type() == Cluster::MANAGER )
-
-global peer_count = 0;
-event remote_connection_handshake_done(p: event_peer) &priority=-5
-	{
-	++peer_count;
-	if ( peer_count == 2 )
-		event ready_for_data();
-	}
-
-@endif
diff --git a/testing/btest/scripts/base/frameworks/measurement/basic.bro b/testing/btest/scripts/base/frameworks/measurement/basic.bro
deleted file mode 100644
index e9dd21e0ef..0000000000
--- a/testing/btest/scripts/base/frameworks/measurement/basic.bro
+++ /dev/null
@@ -1,34 +0,0 @@
-# @TEST-EXEC: bro %INPUT
-# @TEST-EXEC: btest-diff .stdout
-
-event bro_init() &priority=5
-	{
-	local r1: Measurement::Reducer = [$stream="test.metric", 
-	                                  $apply=set(Measurement::SUM, 
-	                                             Measurement::VARIANCE, 
-	                                             Measurement::AVERAGE, 
-	                                             Measurement::MAX, 
-	                                             Measurement::MIN, 
-	                                             Measurement::STD_DEV,
-	                                             Measurement::UNIQUE)];
-	Measurement::create([$epoch=3secs,
-	                     $reducers=set(r1),
-	                     $epoch_finished(data: Measurement::ResultTable) = 
-	                     	{
-	                     	for ( key in data )
-	                     		{
-	                     		local r = data[key]["test.metric"];
-	                     		print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
-	                     		}
-	                     	}
-		 ]);
-
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=5]);
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=22]);
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=94]);
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=50]);
-
-	Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
-	Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
-	}
diff --git a/testing/btest/scripts/base/frameworks/measurement/thresholding.bro b/testing/btest/scripts/base/frameworks/measurement/thresholding.bro
deleted file mode 100644
index d25350930e..0000000000
--- a/testing/btest/scripts/base/frameworks/measurement/thresholding.bro
+++ /dev/null
@@ -1,73 +0,0 @@
-# @TEST-EXEC: bro %INPUT
-# @TEST-EXEC: btest-diff .stdout
-
-redef enum Notice::Type += {
-	Test_Notice,
-};
-
-event bro_init() &priority=5
-	{
-	local r1: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
-	Measurement::create([$epoch=3secs,
-	                     $reducers=set(r1),
-	                     #$threshold_val = Measurement::sum_threshold("test.metric"),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return double_to_count(result["test.metric"]$sum);
-	                     	},
-	                     $threshold=5,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
-	                     	{
-	                     	local r = result["test.metric"];
-	                     	print fmt("THRESHOLD: hit a threshold value at %.0f for %s", r$sum, Measurement::key2str(key));
-	                     	}
-	                     ]);
-
-	local r2: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
-	Measurement::create([$epoch=3secs,
-	                     $reducers=set(r2),
-	                     #$threshold_val = Measurement::sum_threshold("test.metric"),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	return double_to_count(result["test.metric"]$sum); 
-	                     	},
-	                     $threshold_series=vector(3,6,800),
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) = 
-	                     	{
-	                     	local r = result["test.metric"];
-	                     	print fmt("THRESHOLD_SERIES: hit a threshold series value at %.0f for %s", r$sum, Measurement::key2str(key));
-	                     	}
-	                     ]);
-
-	local r3: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
-	local r4: Measurement::Reducer = [$stream="test.metric2", $apply=set(Measurement::SUM)];
-	Measurement::create([$epoch=3secs,
-	                     $reducers=set(r3, r4),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{ 
-	                     	# Calculate a ratio between sums of two reducers.
-	                     	if ( "test.metric2" in result && "test.metric" in result &&
-	                     	     result["test.metric"]$sum > 0 )
-	                     		return double_to_count(result["test.metric2"]$sum / result["test.metric"]$sum);
-	                     	else
-	                     		return 0;
-	                     	},
-	                     # Looking for metric2 sum to be 5 times the sum of metric
-	                     $threshold=5, 
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	local thold = result["test.metric2"]$sum / result["test.metric"]$sum;
-	                     	print fmt("THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at %.0fx for %s", thold, Measurement::key2str(key));
-	                     	}
-	                     ]);
-
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
-	Measurement::add_data("test.metric", [$host=6.5.4.3], [$num=2]);
-	Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=1]);
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=3]);
-	Measurement::add_data("test.metric", [$host=7.2.1.5], [$num=1000]);
-	Measurement::add_data("test.metric2", [$host=7.2.1.5], [$num=10]);
-	Measurement::add_data("test.metric2", [$host=7.2.1.5], [$num=1000]);
-	Measurement::add_data("test.metric2", [$host=7.2.1.5], [$num=54321]);
-
-	}
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
new file mode 100644
index 0000000000..9d4653d77e
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
@@ -0,0 +1,82 @@
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait 15
+
+# @TEST-EXEC: btest-diff manager-1/.stdout
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["worker-1"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
+	["worker-2"]  = [$node_type=Cluster::WORKER,  $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+redef Log::default_rotation_interval = 0secs;
+
+global n = 0;
+
+event bro_init() &priority=5
+	{
+	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
+	SumStats::create([$epoch=5secs,
+	                  $reducers=set(r1),
+	                  $epoch_finished(rt: SumStats::ResultTable) =
+	                  	{
+	                  	for ( key in rt )
+	                  		{
+	                  		local r = rt[key]["test.metric"];
+	                  		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
+	                  		}
+
+	                  	terminate();
+	                  	}]);
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready_for_data: event();
+redef Cluster::manager2worker_events += /^ready_for_data$/;
+
+event ready_for_data()
+	{
+	if ( Cluster::node == "worker-1" )
+		{
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=34]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test.metric", [$host=6.5.4.3], [$num=1]);
+		SumStats::observe("test.metric", [$host=7.2.1.5], [$num=54]);
+		}
+	if ( Cluster::node == "worker-2" )
+		{
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=75]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=3]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=57]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=52]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=61]);
+		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=95]);
+		SumStats::observe("test.metric", [$host=6.5.4.3], [$num=5]);
+		SumStats::observe("test.metric", [$host=7.2.1.5], [$num=91]);
+		SumStats::observe("test.metric", [$host=10.10.10.10], [$num=5]);
+		}
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+event remote_connection_handshake_done(p: event_peer) &priority=-5
+	{
+	++peer_count;
+	if ( peer_count == 2 )
+		event ready_for_data();
+	}
+
+@endif
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic.bro b/testing/btest/scripts/base/frameworks/sumstats/basic.bro
new file mode 100644
index 0000000000..0b2851bf10
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic.bro
@@ -0,0 +1,34 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+event bro_init() &priority=5
+	{
+	local r1: SumStats::Reducer = [$stream="test.metric", 
+	                               $apply=set(SumStats::SUM, 
+	                                          SumStats::VARIANCE, 
+	                                          SumStats::AVERAGE, 
+	                                          SumStats::MAX, 
+	                                          SumStats::MIN, 
+	                                          SumStats::STD_DEV,
+	                                          SumStats::UNIQUE)];
+	SumStats::create([$epoch=3secs,
+	                     $reducers=set(r1),
+	                     $epoch_finished(data: SumStats::ResultTable) = 
+	                     	{
+	                     	for ( key in data )
+	                     		{
+	                     		local r = data[key]["test.metric"];
+	                     		print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
+	                     		}
+	                     	}
+		 ]);
+
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=5]);
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=22]);
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=94]);
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=50]);
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=50]);
+
+	SumStats::observe("test.metric", [$host=6.5.4.3], [$num=2]);
+	SumStats::observe("test.metric", [$host=7.2.1.5], [$num=1]);
+	}
diff --git a/testing/btest/scripts/base/frameworks/measurement/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
similarity index 59%
rename from testing/btest/scripts/base/frameworks/measurement/cluster-intermediate-update.bro
rename to testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
index 56f44db2eb..303a0dc852 100644
--- a/testing/btest/scripts/base/frameworks/measurement/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
@@ -19,20 +19,19 @@ redef Log::default_rotation_interval = 0secs;
 
 event bro_init() &priority=5
 	{
-	local r1: Measurement::Reducer = [$stream="test.metric", $apply=set(Measurement::SUM)];
-	Measurement::create([$epoch=1hr,
-	                     $reducers=set(r1),
-	                     $threshold_val(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	return double_to_count(result["test.metric"]$sum);
-	                     	},
-	                     $threshold=100,
-	                     $threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
-	                     	{
-	                     	print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);
-	                     	terminate();
-	                     	}
-	                    ]);
+	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=1hr,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return double_to_count(result["test.metric"]$sum);
+	                  	},
+	                  $threshold=100,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);
+	                  	terminate();
+	                  	}]);
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -40,12 +39,12 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
-event do_metrics(i: count)
+event do_stats(i: count)
 	{
 	# Worker-1 will trigger an intermediate update and then if everything
 	# works correctly, the data from worker-2 will hit the threshold and
 	# should trigger the notice.
-	Measurement::add_data("test.metric", [$host=1.2.3.4], [$num=i]);
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=i]);
 	}
 
 event remote_connection_handshake_done(p: event_peer)
@@ -53,8 +52,8 @@ event remote_connection_handshake_done(p: event_peer)
 	if ( p$descr == "manager-1" )
 		{
 		if ( Cluster::node == "worker-1" )
-			schedule 0.1sec { do_metrics(1) };
+			schedule 0.1sec { do_stats(1) };
 		if ( Cluster::node == "worker-2" )
-			schedule 0.5sec { do_metrics(99) };
+			schedule 0.5sec { do_stats(99) };
 		}
 	}
diff --git a/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro b/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro
new file mode 100644
index 0000000000..ddc053bd23
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/sumstats/thresholding.bro
@@ -0,0 +1,73 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+event bro_init() &priority=5
+	{
+	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=3secs,
+	                  $reducers=set(r1),
+	                  #$threshold_val = SumStats::sum_threshold("test.metric"),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return double_to_count(result["test.metric"]$sum);
+	                  	},
+	                  $threshold=5,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	local r = result["test.metric"];
+	                  	print fmt("THRESHOLD: hit a threshold value at %.0f for %s", r$sum, SumStats::key2str(key));
+	                  	}
+	                  ]);
+
+	local r2: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=3secs,
+	                  $reducers=set(r2),
+	                  #$threshold_val = SumStats::sum_threshold("test.metric"),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	return double_to_count(result["test.metric"]$sum); 
+	                  	},
+	                  $threshold_series=vector(3,6,800),
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = 
+	                  	{
+	                  	local r = result["test.metric"];
+	                  	print fmt("THRESHOLD_SERIES: hit a threshold series value at %.0f for %s", r$sum, SumStats::key2str(key));
+	                  	}
+	                  ]);
+
+	local r3: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
+	local r4: SumStats::Reducer = [$stream="test.metric2", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=3secs,
+	                  $reducers=set(r3, r4),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{ 
+	                  	# Calculate a ratio between sums of two reducers.
+	                  	if ( "test.metric2" in result && "test.metric" in result &&
+	                  	     result["test.metric"]$sum > 0 )
+	                  		return double_to_count(result["test.metric2"]$sum / result["test.metric"]$sum);
+	                  	else
+	                  		return 0;
+	                  	},
+	                  # Looking for metric2 sum to be 5 times the sum of metric
+	                  $threshold=5, 
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local thold = result["test.metric2"]$sum / result["test.metric"]$sum;
+	                  	print fmt("THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at %.0fx for %s", thold, SumStats::key2str(key));
+	                  	}
+	                  ]);
+
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=3]);
+	SumStats::observe("test.metric", [$host=6.5.4.3], [$num=2]);
+	SumStats::observe("test.metric", [$host=7.2.1.5], [$num=1]);
+	SumStats::observe("test.metric", [$host=1.2.3.4], [$num=3]);
+	SumStats::observe("test.metric", [$host=7.2.1.5], [$num=1000]);
+	SumStats::observe("test.metric2", [$host=7.2.1.5], [$num=10]);
+	SumStats::observe("test.metric2", [$host=7.2.1.5], [$num=1000]);
+	SumStats::observe("test.metric2", [$host=7.2.1.5], [$num=54321]);
+
+	}

From 437815454d275392c91bcffd9e981e7baced0df6 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 15 Apr 2013 15:28:11 -0400
Subject: [PATCH 62/70] SumStats tests pass.

---
 scripts/base/frameworks/sumstats/cluster.bro  | 56 +++++++++----------
 scripts/base/frameworks/sumstats/main.bro     | 10 ++--
 .../base/frameworks/sumstats/non-cluster.bro  | 20 +++----
 scripts/base/init-default.bro                 |  2 +-
 .../.stdout                                   | 12 ++--
 .../frameworks/sumstats/basic-cluster.bro     | 32 +++++------
 6 files changed, 66 insertions(+), 66 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index 098c047961..ee763c1d9d 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -35,34 +35,34 @@ export {
 
 	## Event sent by the manager in a cluster to initiate the 
 	## collection of metrics values for a measurement.
-	global cluster_measurement_request: event(uid: string, mid: string);
+	global cluster_ss_request: event(uid: string, ssid: string);
 
 	## Event sent by nodes that are collecting metrics after receiving
 	## a request for the metric measurement from the manager.
-	global cluster_measurement_response: event(uid: string, mid: string, data: ResultTable, done: bool);
+	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
 	## This event is sent by the manager in a cluster to initiate the
 	## collection of a single key value from a measurement.  It's typically
 	## used to get intermediate updates before the break interval triggers
 	## to speed detection of a value crossing a threshold.
-	global cluster_key_request: event(uid: string, mid: string, key: Key);
+	global cluster_key_request: event(uid: string, ssid: string, key: Key);
 
 	## This event is sent by nodes in response to a 
 	## :bro:id:`SumStats::cluster_key_request` event.
-	global cluster_key_response: event(uid: string, mid: string, key: Key, result: Result);
+	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
 
 	## This is sent by workers to indicate that they crossed the percent of the 
 	## current threshold by the percentage defined globally in 
 	## :bro:id:`SumStats::cluster_request_global_view_percent`
-	global cluster_key_intermediate_response: event(mid: string, key: SumStats::Key);
+	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
 
 	## This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, mid: string, data: ResultTable);
+	global send_data: event(uid: string, ssid: string, data: ResultTable);
 }
 
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /SumStats::cluster_(measurement_request|key_request)/;
-redef Cluster::worker2manager_events += /SumStats::cluster_(measurement_response|key_response|key_intermediate_response)/;
+redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request)/;
+redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # This variable is maintained to know what keys have recently sent as 
@@ -99,7 +99,7 @@ function data_added(ss: SumStat, key: Key, result: Result)
 		}
 	}
 
-event SumStats::send_data(uid: string, mid: string, data: ResultTable)
+event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
 	{
 	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
 
@@ -121,39 +121,39 @@ event SumStats::send_data(uid: string, mid: string, data: ResultTable)
 	if ( |data| == 0 )
 		done = T;
 	
-	event SumStats::cluster_measurement_response(uid, mid, local_data, done);
+	event SumStats::cluster_ss_response(uid, ssid, local_data, done);
 	if ( ! done )
-		schedule 0.01 sec { SumStats::send_data(uid, mid, data) };
+		schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
 	}
 
-event SumStats::cluster_measurement_request(uid: string, mid: string)
+event SumStats::cluster_ss_request(uid: string, ssid: string)
 	{
-	#print fmt("WORKER %s: received the cluster_measurement_request event for %s.", Cluster::node, id);
+	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
 	
 	# Initiate sending all of the data for the requested measurement.
-	if ( mid in result_store )
-		event SumStats::send_data(uid, mid, result_store[mid]);
+	if ( ssid in result_store )
+		event SumStats::send_data(uid, ssid, result_store[ssid]);
 	else
-		event SumStats::send_data(uid, mid, table());
+		event SumStats::send_data(uid, ssid, table());
 
 	# Lookup the actual measurement and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
-	if ( mid in stats_store )
-		reset(stats_store[mid]);
+	if ( ssid in stats_store )
+		reset(stats_store[ssid]);
 	}
 	
-event SumStats::cluster_key_request(uid: string, mid: string, key: Key)
+event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
 	{
-	if ( mid in result_store && key in result_store[mid] )
+	if ( ssid in result_store && key in result_store[ssid] )
 		{
 		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
-		event SumStats::cluster_key_response(uid, mid, key, result_store[mid][key]);
+		event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]);
 		}
 	else
 		{
 		# We need to send an empty response if we don't have the data so that the manager
 		# can know that it heard back from all of the workers.
-		event SumStats::cluster_key_response(uid, mid, key, table());
+		event SumStats::cluster_key_response(uid, ssid, key, table());
 		}
 	}
 
@@ -195,16 +195,16 @@ event SumStats::finish_epoch(ss: SumStat)
 		#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
 		local uid = unique_id("");
 		
-		if ( uid in measurement_results )
-			delete measurement_results[uid];
+		if ( uid in stats_results )
+			delete stats_results[uid];
 		stats_results[uid] = table();
 		
 		# Request data from peers.
-		event SumStats::cluster_measurement_request(uid, ss$id);
+		event SumStats::cluster_ss_request(uid, ss$id);
 		}
 
 	# Schedule the next finish_epoch event.
-	schedule m$epoch { SumStats::finish_epoch(m) };
+	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 
 # This is unlikely to be called often, but it's here in case there are measurements
@@ -252,7 +252,7 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
 	#print fmt("MANAGER: requesting key data for %s", key2str(key));
 
 	if ( ssid in outstanding_global_views &&
-	     |outstanding_global_views[mid]| > max_outstanding_global_views )
+	     |outstanding_global_views[ssid]| > max_outstanding_global_views )
 		{
 		# Don't do this intermediate update.  Perhaps at some point in the future 
 		# we will queue and randomly select from these ignored intermediate
@@ -266,7 +266,7 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
 	event SumStats::cluster_key_request(uid, ssid, key);
 	}
 
-event SumStats::cluster_measurement_response(uid: string, ssid: string, data: ResultTable, done: bool)
+event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
 	{
 	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
 
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index a5c41ba1f3..643502efb4 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -120,7 +120,7 @@ export {
 	};
 	
 	## Create a summary statistic.
-	global create: function(m: SumStats::SumStat);
+	global create: function(ss: SumStats::SumStat);
 
 	## Add data into an observation stream. This should be 
 	## called when a script has measured some point value.
@@ -158,13 +158,13 @@ type Thresholding: record {
 # Internal use only.  For tracking thresholds per sumstat and key.
 global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
 
-redef record SumStats += {
+redef record SumStat += {
 	# Internal use only (mostly for cluster coherency).
 	id: string &optional;
 };
 
 # Store of sumstats indexed on the sumstat id.
-global stats_store: table[string] of SumStats = table();
+global stats_store: table[string] of SumStat = table();
 
 # Store of reducers indexed on the data point stream id.
 global reducer_store: table[string] of set[Reducer] = table();
@@ -179,7 +179,7 @@ global thresholds_store: table[string, Key] of bool = table();
 # key values are updated and the new val is given as the `val` argument.
 # It's only prototyped here because cluster and non-cluster have separate 
 # implementations.
-global data_added: function(m: SumStats, key: Key, result: Result);
+global data_added: function(ss: SumStat, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
 global add_to_reducer_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
@@ -190,7 +190,7 @@ global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: Res
 
 # Event that is used to "finish" measurements and adapt the measurement
 # framework for clustered or non-clustered usage.
-global finish_epoch: event(m: SumStats);
+global finish_epoch: event(ss: SumStat);
 
 function key2str(key: Key): string
 	{
diff --git a/scripts/base/frameworks/sumstats/non-cluster.bro b/scripts/base/frameworks/sumstats/non-cluster.bro
index 6163548ec6..21386a246e 100644
--- a/scripts/base/frameworks/sumstats/non-cluster.bro
+++ b/scripts/base/frameworks/sumstats/non-cluster.bro
@@ -2,23 +2,23 @@
 
 module SumStats;
 
-event SumStats::finish_epoch(m: SumStats)
+event SumStats::finish_epoch(ss: SumStat)
 	{
-	if ( m$id in result_store )
+	if ( ss$id in result_store )
 		{
-		local data = result_store[m$id];
-		if ( m?$epoch_finished )
-			m$epoch_finished(data);
+		local data = result_store[ss$id];
+		if ( ss?$epoch_finished )
+			ss$epoch_finished(data);
 
-		reset(m);
+		reset(ss);
 		}
 
-	schedule m$epoch { SumStats::finish_epoch(m) };
+	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 	
 	
-function data_added(m: SumStats, key: Key, result: Result)
+function data_added(ss: SumStat, key: Key, result: Result)
 	{
-	if ( check_thresholds(m, key, result, 1.0) )
-		threshold_crossed(m, key, result);
+	if ( check_thresholds(ss, key, result, 1.0) )
+		threshold_crossed(ss, key, result);
 	}
diff --git a/scripts/base/init-default.bro b/scripts/base/init-default.bro
index bb64accaea..829a1b9982 100644
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@@ -29,9 +29,9 @@
 @load base/frameworks/communication
 @load base/frameworks/control
 @load base/frameworks/cluster
-@load base/frameworks/measurement
 @load base/frameworks/intel
 @load base/frameworks/reporter
+@load base/frameworks/sumstats
 @load base/frameworks/tunnels
 
 @load base/protocols/conn
diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout
index ac8785d182..132a1114fc 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout
+++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout
@@ -1,6 +1,6 @@
-THRESHOLD_SERIES: hit a threshold series value at 3 for measurement_key(host=1.2.3.4)
-THRESHOLD_SERIES: hit a threshold series value at 6 for measurement_key(host=1.2.3.4)
-THRESHOLD: hit a threshold value at 6 for measurement_key(host=1.2.3.4)
-THRESHOLD_SERIES: hit a threshold series value at 1001 for measurement_key(host=7.2.1.5)
-THRESHOLD: hit a threshold value at 1001 for measurement_key(host=7.2.1.5)
-THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at 55x for measurement_key(host=7.2.1.5)
+THRESHOLD_SERIES: hit a threshold series value at 3 for sumstats_key(host=1.2.3.4)
+THRESHOLD_SERIES: hit a threshold series value at 6 for sumstats_key(host=1.2.3.4)
+THRESHOLD: hit a threshold value at 6 for sumstats_key(host=1.2.3.4)
+THRESHOLD_SERIES: hit a threshold series value at 1001 for sumstats_key(host=7.2.1.5)
+THRESHOLD: hit a threshold value at 1001 for sumstats_key(host=7.2.1.5)
+THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at 55x for sumstats_key(host=7.2.1.5)
diff --git a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
index 9d4653d77e..1b7903ca1a 100644
--- a/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
@@ -22,14 +22,14 @@ global n = 0;
 
 event bro_init() &priority=5
 	{
-	local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
+	local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
 	SumStats::create([$epoch=5secs,
 	                  $reducers=set(r1),
 	                  $epoch_finished(rt: SumStats::ResultTable) =
 	                  	{
 	                  	for ( key in rt )
 	                  		{
-	                  		local r = rt[key]["test.metric"];
+	                  		local r = rt[key]["test"];
 	                  		print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
 	                  		}
 
@@ -49,23 +49,23 @@ event ready_for_data()
 	{
 	if ( Cluster::node == "worker-1" )
 		{
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=34]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=30]);
-		SumStats::observe("test.metric", [$host=6.5.4.3], [$num=1]);
-		SumStats::observe("test.metric", [$host=7.2.1.5], [$num=54]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=34]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test", [$host=6.5.4.3], [$num=1]);
+		SumStats::observe("test", [$host=7.2.1.5], [$num=54]);
 		}
 	if ( Cluster::node == "worker-2" )
 		{
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=75]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=30]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=3]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=57]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=52]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=61]);
-		SumStats::observe("test.metric", [$host=1.2.3.4], [$num=95]);
-		SumStats::observe("test.metric", [$host=6.5.4.3], [$num=5]);
-		SumStats::observe("test.metric", [$host=7.2.1.5], [$num=91]);
-		SumStats::observe("test.metric", [$host=10.10.10.10], [$num=5]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=75]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=3]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=57]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=52]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=61]);
+		SumStats::observe("test", [$host=1.2.3.4], [$num=95]);
+		SumStats::observe("test", [$host=6.5.4.3], [$num=5]);
+		SumStats::observe("test", [$host=7.2.1.5], [$num=91]);
+		SumStats::observe("test", [$host=10.10.10.10], [$num=5]);
 		}
 	}
 

From 1cac89e4f8bd0efe9e5ed9c4dbb4070448a74d47 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 16 Apr 2013 00:54:41 -0400
Subject: [PATCH 63/70] SumStats test checkpoint.

---
 DocSourcesList.cmake                          | 24 ++++++------
 scripts/base/frameworks/sumstats/cluster.bro  | 24 ++++++------
 .../frameworks/sumstats/plugins/average.bro   |  2 +-
 .../base/frameworks/sumstats/plugins/max.bro  |  2 +-
 .../base/frameworks/sumstats/plugins/min.bro  |  2 +-
 .../frameworks/sumstats/plugins/sample.bro    |  4 +-
 .../frameworks/sumstats/plugins/std-dev.bro   |  7 +---
 .../base/frameworks/sumstats/plugins/sum.bro  |  2 +-
 .../frameworks/sumstats/plugins/unique.bro    |  4 +-
 .../frameworks/sumstats/plugins/variance.bro  |  2 +-
 scripts/policy/misc/app-metrics.bro           |  2 +-
 scripts/policy/protocols/conn/metrics.bro     | 24 ------------
 scripts/policy/protocols/smtp/metrics.bro     | 37 -------------------
 scripts/test-all-policy.bro                   |  5 +++
 .../canonified_loaded_scripts.log             | 28 +++++++-------
 .../coverage.init-default/missing_loads       |  2 +-
 16 files changed, 55 insertions(+), 116 deletions(-)
 delete mode 100644 scripts/policy/protocols/conn/metrics.bro
 delete mode 100644 scripts/policy/protocols/smtp/metrics.bro

diff --git a/DocSourcesList.cmake b/DocSourcesList.cmake
index 5ac87a6305..a9ba4838db 100644
--- a/DocSourcesList.cmake
+++ b/DocSourcesList.cmake
@@ -46,17 +46,6 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/measurement/cluster.bro)
-rest_target(${psd} base/frameworks/measurement/main.bro)
-rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/average.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/max.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/min.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/sample.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/sum.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/unique.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/variance.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -72,6 +61,17 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/sumstats/cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/main.bro)
+rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
@@ -147,7 +147,6 @@ rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/metrics.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
@@ -166,7 +165,6 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
 rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/metrics.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
 rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)
diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index ee763c1d9d..4f9743547b 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -34,15 +34,15 @@ export {
 	const enable_intermediate_updates = T &redef;
 
 	## Event sent by the manager in a cluster to initiate the 
-	## collection of metrics values for a measurement.
+	## collection of metrics values for a sumstat.
 	global cluster_ss_request: event(uid: string, ssid: string);
 
 	## Event sent by nodes that are collecting metrics after receiving
-	## a request for the metric measurement from the manager.
+	## a request for the metric sumstat from the manager.
 	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
 	## This event is sent by the manager in a cluster to initiate the
-	## collection of a single key value from a measurement.  It's typically
+	## collection of a single key value from a sumstat.  It's typically
 	## used to get intermediate updates before the break interval triggers
 	## to speed detection of a value crossing a threshold.
 	global cluster_key_request: event(uid: string, ssid: string, key: Key);
@@ -130,13 +130,13 @@ event SumStats::cluster_ss_request(uid: string, ssid: string)
 	{
 	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
 	
-	# Initiate sending all of the data for the requested measurement.
+	# Initiate sending all of the data for the requested stats.
 	if ( ssid in result_store )
 		event SumStats::send_data(uid, ssid, result_store[ssid]);
 	else
 		event SumStats::send_data(uid, ssid, table());
 
-	# Lookup the actual measurement and reset it, the reference to the data
+	# Lookup the actual sumstats and reset it, the reference to the data
 	# currently stored will be maintained internally by the send_data event.
 	if ( ssid in stats_store )
 		reset(stats_store[ssid]);
@@ -181,9 +181,9 @@ global done_with: table[string] of count &read_expire=1min &default=0;
 global key_requests: table[string] of Result &read_expire=1min;
 
 # This variable is maintained by managers to prevent overwhelming communication due
-# to too many intermediate updates.  Each measurement is tracked separately so that 
-# one won't overwhelm and degrade other quieter measurements. 
-# Indexed on a measurement id.
+# to too many intermediate updates.  Each sumstat is tracked separately so that 
+# one won't overwhelm and degrade other quieter sumstats. 
+# Indexed on a sumstat id.
 global outstanding_global_views: table[string] of count &default=0;
 
 const zero_time = double_to_time(0.0);
@@ -192,7 +192,7 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( network_time() > zero_time )
 		{
-		#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
+		#print fmt("%.6f MANAGER: breaking %s sumstat for %s metric", network_time(), ss$name, ss$id);
 		local uid = unique_id("");
 		
 		if ( uid in stats_results )
@@ -207,8 +207,8 @@ event SumStats::finish_epoch(ss: SumStat)
 	schedule ss$epoch { SumStats::finish_epoch(ss) };
 	}
 
-# This is unlikely to be called often, but it's here in case there are measurements
-# being collected by managers.
+# This is unlikely to be called often, but it's here in 
+# case there are sumstats being collected by managers.
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	if ( check_thresholds(ss, key, result, 1.0) )
@@ -305,7 +305,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
 		# Clean up
 		delete stats_results[uid];
 		delete done_with[uid];
-		# Not sure I need to reset the measurement on the manager.
+		# Not sure I need to reset the sumstat on the manager.
 		reset(ss);
 		}
 	}
diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro
index 002a0147ff..35228a28f5 100644
--- a/scripts/base/frameworks/sumstats/plugins/average.bro
+++ b/scripts/base/frameworks/sumstats/plugins/average.bro
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( AVERAGE in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro
index 0e377ff320..0a959f2d09 100644
--- a/scripts/base/frameworks/sumstats/plugins/max.bro
+++ b/scripts/base/frameworks/sumstats/plugins/max.bro
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( MAX in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro
index 5e1e3fbbb7..16c2dfc3d7 100644
--- a/scripts/base/frameworks/sumstats/plugins/min.bro
+++ b/scripts/base/frameworks/sumstats/plugins/min.bro
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( MIN in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro
index a694296727..622b160cbe 100644
--- a/scripts/base/frameworks/sumstats/plugins/sample.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sample.bro
@@ -29,13 +29,13 @@ function get_samples(rv: ResultVal): vector of Observation
 	return s;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( r$samples > 0 )
 		{
 		if ( ! rv?$samples )
 			rv$samples = Queue::init([$max_len=r$samples]);
-		Queue::put(rv$samples, data);
+		Queue::put(rv$samples, obs);
 		}
 	}
 
diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
index af6eea8cdc..7c2754570a 100644
--- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro
+++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
@@ -22,13 +22,10 @@ function calc_std_dev(rv: ResultVal)
 	}
 
 # This depends on the variance plugin which uses priority -5
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-10
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
 	{
 	if ( STD_DEV in r$apply )
-		{
-		if ( rv?$variance )
-			calc_std_dev(rv);
-		}
+		calc_std_dev(rv);
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro
index 572402d6c5..8c8c65cd61 100644
--- a/scripts/base/frameworks/sumstats/plugins/sum.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sum.bro
@@ -34,7 +34,7 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( SUM in r$apply )
 		rv$sum += val;
diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro
index f260148af4..d3a4464d0d 100644
--- a/scripts/base/frameworks/sumstats/plugins/unique.bro
+++ b/scripts/base/frameworks/sumstats/plugins/unique.bro
@@ -23,13 +23,13 @@ redef record ResultVal += {
 	unique_vals: set[Observation] &optional;
 };
 
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( UNIQUE in r$apply )
 		{
 		if ( ! rv?$unique_vals ) 
 			rv$unique_vals=set();
-		add rv$unique_vals[data];
+		add rv$unique_vals[obs];
 		rv$unique = |rv$unique_vals|;
 		}
 	}
diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro
index a26a2d4095..29118b284b 100644
--- a/scripts/base/frameworks/sumstats/plugins/variance.bro
+++ b/scripts/base/frameworks/sumstats/plugins/variance.bro
@@ -29,7 +29,7 @@ function calc_variance(rv: ResultVal)
 	}
 
 # Reduced priority since this depends on the average
-hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-5
+hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
 	{
 	if ( VARIANCE in r$apply )
 		{
diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro
index 53f210b46a..ec2e8f8d48 100644
--- a/scripts/policy/misc/app-metrics.bro
+++ b/scripts/policy/misc/app-metrics.bro
@@ -32,7 +32,7 @@ redef record connection += {
 
 event bro_init() &priority=3
 	{
-	Log::create_stream(AppSumStats::LOG, [$columns=Info]);
+	Log::create_stream(AppStats::LOG, [$columns=Info]);
 
 	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
 	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro
deleted file mode 100644
index 62ca96ea0a..0000000000
--- a/scripts/policy/protocols/conn/metrics.bro
+++ /dev/null
@@ -1,24 +0,0 @@
-@load base/frameworks/measurement
-@load base/utils/site
-
-event bro_init() &priority=3
-	{
-	Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM),
-	                                      $period_finished=Metrics::write_log]);
-	Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM),
-	                                     $period_finished=Metrics::write_log]);
-	}
-
-event connection_established(c: connection) &priority=3
-	{
-	if ( Site::is_local_addr(c$id$orig_h) )
-		{
-		local loc = lookup_location(c$id$resp_h);
-		if ( loc?$country_code )
-			Metrics::add_data("conns.country", [$str=loc$country_code], [$num=1]);
-		}
-		
-	local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h;
-	# There is no index for this.
-	Metrics::add_data("hosts.active", [], [$str=cat(the_host)]);
-	}
diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro
deleted file mode 100644
index 04e1185e25..0000000000
--- a/scripts/policy/protocols/smtp/metrics.bro
+++ /dev/null
@@ -1,37 +0,0 @@
-##! This script is meant to answer the following questions...
-##!   "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?"
-##!   "How much mail is being sent from each local mail server per hour?"
-
-@load base/protocols/smtp
-@load base/frameworks/measurement
-@load base/utils/site
-@load base/utils/directions-and-hosts
-
-module SMTPMetrics;
-
-export {
-	## Define the break intervals for all of the metrics collected and logged by this script.
-	const breaks=1hr &redef;
-}
-
-event bro_init() &priority=5
-	{
-	Metrics::add_filter("smtp.mailfrom", [$every=breaks,
-	                                      $measure=set(Metrics::SUM),
-	                                      $pred(index: Metrics::Index, data: Metrics::Observation) = { 
-	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
-	                                      }]);
-	Metrics::add_filter("smtp.messages", [$every=breaks,
-	                                      $measure=set(Metrics::SUM),
-	                                      $pred(index: Metrics::Index, data: Metrics::Observation) = { 
-	                                      	return addr_matches_host(index$host, LOCAL_HOSTS); 
-	                                      }]);
-	}
-
-event SMTP::log_smtp(rec: SMTP::Info)
-	{
-	Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], [$num=1]);
-	
-	if ( rec?$mailfrom )
-		Metrics::add_data("smtp.mailfrom", [$host=rec$id$orig_h], [$str=rec$mailfrom]);
-	}
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index 2fe32a4788..35d9f89de9 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -32,9 +32,13 @@
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
 @load misc/analysis-groups.bro
+@load misc/app-metrics.bro
 @load misc/capture-loss.bro
+@load misc/detect-traceroute/__load__.bro
+@load misc/detect-traceroute/main.bro
 @load misc/loaded-scripts.bro
 @load misc/profiling.bro
+@load misc/scan.bro
 @load misc/stats.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
@@ -42,6 +46,7 @@
 @load protocols/conn/weirds.bro
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
+@load protocols/ftp/detect-bruteforcing.bro
 @load protocols/ftp/detect.bro
 @load protocols/ftp/software.bro
 @load protocols/http/detect-MHR.bro
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index e691a906c2..a34f4dd3fc 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-04-02-04-22-32
+#open	2013-04-16-03-43-22
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -68,23 +68,23 @@ scripts/base/init-default.bro
     scripts/base/frameworks/software/main.bro
   scripts/base/frameworks/communication/__load__.bro
     scripts/base/frameworks/communication/main.bro
-  scripts/base/frameworks/measurement/__load__.bro
-    scripts/base/frameworks/measurement/main.bro
-    scripts/base/frameworks/measurement/plugins/__load__.bro
-      scripts/base/frameworks/measurement/plugins/average.bro
-      scripts/base/frameworks/measurement/plugins/max.bro
-      scripts/base/frameworks/measurement/plugins/min.bro
-      scripts/base/frameworks/measurement/plugins/sample.bro
-      scripts/base/frameworks/measurement/plugins/std-dev.bro
-        scripts/base/frameworks/measurement/plugins/variance.bro
-      scripts/base/frameworks/measurement/plugins/sum.bro
-      scripts/base/frameworks/measurement/plugins/unique.bro
-    scripts/base/frameworks/measurement/non-cluster.bro
   scripts/base/frameworks/intel/__load__.bro
     scripts/base/frameworks/intel/main.bro
     scripts/base/frameworks/intel/input.bro
   scripts/base/frameworks/reporter/__load__.bro
     scripts/base/frameworks/reporter/main.bro
+  scripts/base/frameworks/sumstats/__load__.bro
+    scripts/base/frameworks/sumstats/main.bro
+    scripts/base/frameworks/sumstats/plugins/__load__.bro
+      scripts/base/frameworks/sumstats/plugins/average.bro
+      scripts/base/frameworks/sumstats/plugins/max.bro
+      scripts/base/frameworks/sumstats/plugins/min.bro
+      scripts/base/frameworks/sumstats/plugins/sample.bro
+      scripts/base/frameworks/sumstats/plugins/variance.bro
+      scripts/base/frameworks/sumstats/plugins/std-dev.bro
+      scripts/base/frameworks/sumstats/plugins/sum.bro
+      scripts/base/frameworks/sumstats/plugins/unique.bro
+    scripts/base/frameworks/sumstats/non-cluster.bro
   scripts/base/frameworks/tunnels/__load__.bro
     scripts/base/frameworks/tunnels/main.bro
   scripts/base/protocols/conn/__load__.bro
@@ -130,4 +130,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-04-02-04-22-32
+#close	2013-04-16-03-43-22
diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads
index 554fcf012e..23cae7b694 100644
--- a/testing/btest/Baseline/coverage.init-default/missing_loads
+++ b/testing/btest/Baseline/coverage.init-default/missing_loads
@@ -3,5 +3,5 @@
 -./frameworks/cluster/nodes/worker.bro
 -./frameworks/cluster/setup-connections.bro
 -./frameworks/intel/cluster.bro
--./frameworks/measurement/cluster.bro
 -./frameworks/notice/cluster.bro
+-./frameworks/sumstats/cluster.bro

From 60605412abe961f0844ca74db90fdb629ce20869 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 22 Apr 2013 14:14:50 -0400
Subject: [PATCH 64/70] Fix a few tests.

---
 DocSourcesList.cmake                          | 187 ------------------
 doc/scripts/DocSourcesList.cmake              |  25 ++-
 .../canonified_loaded_scripts.log             |   6 +-
 3 files changed, 14 insertions(+), 204 deletions(-)
 delete mode 100644 DocSourcesList.cmake

diff --git a/DocSourcesList.cmake b/DocSourcesList.cmake
deleted file mode 100644
index a9ba4838db..0000000000
--- a/DocSourcesList.cmake
+++ /dev/null
@@ -1,187 +0,0 @@
-# DO NOT EDIT
-# This file is auto-generated from the genDocSourcesList.sh script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd ${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/init-bare.bro internal)
-
-rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
-rest_target(${psd} base/frameworks/cluster/main.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
-rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
-rest_target(${psd} base/frameworks/communication/main.bro)
-rest_target(${psd} base/frameworks/control/main.bro)
-rest_target(${psd} base/frameworks/dpd/main.bro)
-rest_target(${psd} base/frameworks/input/main.bro)
-rest_target(${psd} base/frameworks/input/readers/ascii.bro)
-rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
-rest_target(${psd} base/frameworks/input/readers/raw.bro)
-rest_target(${psd} base/frameworks/intel/cluster.bro)
-rest_target(${psd} base/frameworks/intel/input.bro)
-rest_target(${psd} base/frameworks/intel/main.bro)
-rest_target(${psd} base/frameworks/logging/main.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
-rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
-rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
-rest_target(${psd} base/frameworks/notice/actions/drop.bro)
-rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
-rest_target(${psd} base/frameworks/notice/actions/page.bro)
-rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
-rest_target(${psd} base/frameworks/notice/cluster.bro)
-rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
-rest_target(${psd} base/frameworks/notice/main.bro)
-rest_target(${psd} base/frameworks/notice/non-cluster.bro)
-rest_target(${psd} base/frameworks/notice/weird.bro)
-rest_target(${psd} base/frameworks/packet-filter/main.bro)
-rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
-rest_target(${psd} base/frameworks/reporter/main.bro)
-rest_target(${psd} base/frameworks/signatures/main.bro)
-rest_target(${psd} base/frameworks/software/main.bro)
-rest_target(${psd} base/frameworks/sumstats/cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/main.bro)
-rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
-rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
-rest_target(${psd} base/frameworks/tunnels/main.bro)
-rest_target(${psd} base/misc/find-checksum-offloading.bro)
-rest_target(${psd} base/protocols/conn/contents.bro)
-rest_target(${psd} base/protocols/conn/inactivity.bro)
-rest_target(${psd} base/protocols/conn/main.bro)
-rest_target(${psd} base/protocols/conn/polling.bro)
-rest_target(${psd} base/protocols/dns/consts.bro)
-rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/file-extract.bro)
-rest_target(${psd} base/protocols/ftp/gridftp.bro)
-rest_target(${psd} base/protocols/ftp/main.bro)
-rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/http/file-extract.bro)
-rest_target(${psd} base/protocols/http/file-hash.bro)
-rest_target(${psd} base/protocols/http/file-ident.bro)
-rest_target(${psd} base/protocols/http/main.bro)
-rest_target(${psd} base/protocols/http/utils.bro)
-rest_target(${psd} base/protocols/irc/dcc-send.bro)
-rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/modbus/consts.bro)
-rest_target(${psd} base/protocols/modbus/main.bro)
-rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
-rest_target(${psd} base/protocols/smtp/entities.bro)
-rest_target(${psd} base/protocols/smtp/main.bro)
-rest_target(${psd} base/protocols/socks/consts.bro)
-rest_target(${psd} base/protocols/socks/main.bro)
-rest_target(${psd} base/protocols/ssh/main.bro)
-rest_target(${psd} base/protocols/ssl/consts.bro)
-rest_target(${psd} base/protocols/ssl/main.bro)
-rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
-rest_target(${psd} base/protocols/syslog/consts.bro)
-rest_target(${psd} base/protocols/syslog/main.bro)
-rest_target(${psd} base/utils/addrs.bro)
-rest_target(${psd} base/utils/conn-ids.bro)
-rest_target(${psd} base/utils/directions-and-hosts.bro)
-rest_target(${psd} base/utils/files.bro)
-rest_target(${psd} base/utils/numbers.bro)
-rest_target(${psd} base/utils/paths.bro)
-rest_target(${psd} base/utils/patterns.bro)
-rest_target(${psd} base/utils/queue.bro)
-rest_target(${psd} base/utils/site.bro)
-rest_target(${psd} base/utils/strings.bro)
-rest_target(${psd} base/utils/thresholds.bro)
-rest_target(${psd} base/utils/time.bro)
-rest_target(${psd} base/utils/urls.bro)
-rest_target(${psd} policy/frameworks/communication/listen.bro)
-rest_target(${psd} policy/frameworks/control/controllee.bro)
-rest_target(${psd} policy/frameworks/control/controller.bro)
-rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
-rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/intel/conn-established.bro)
-rest_target(${psd} policy/frameworks/intel/dns.bro)
-rest_target(${psd} policy/frameworks/intel/http-host-header.bro)
-rest_target(${psd} policy/frameworks/intel/http-url.bro)
-rest_target(${psd} policy/frameworks/intel/http-user-agents.bro)
-rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
-rest_target(${psd} policy/frameworks/intel/smtp.bro)
-rest_target(${psd} policy/frameworks/intel/ssl.bro)
-rest_target(${psd} policy/frameworks/intel/where-locations.bro)
-rest_target(${psd} policy/frameworks/software/version-changes.bro)
-rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/integration/collective-intel/main.bro)
-rest_target(${psd} policy/misc/analysis-groups.bro)
-rest_target(${psd} policy/misc/app-metrics.bro)
-rest_target(${psd} policy/misc/capture-loss.bro)
-rest_target(${psd} policy/misc/detect-traceroute/main.bro)
-rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/profiling.bro)
-rest_target(${psd} policy/misc/scan.bro)
-rest_target(${psd} policy/misc/stats.bro)
-rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/known-hosts.bro)
-rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/weirds.bro)
-rest_target(${psd} policy/protocols/dns/auth-addl.bro)
-rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
-rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-MHR.bro)
-rest_target(${psd} policy/protocols/http/detect-sqli.bro)
-rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/header-names.bro)
-rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
-rest_target(${psd} policy/protocols/http/software.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
-rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
-rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
-rest_target(${psd} policy/protocols/smtp/blocklists.bro)
-rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/software.bro)
-rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ssh/geo-data.bro)
-rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
-rest_target(${psd} policy/protocols/ssh/software.bro)
-rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
-rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
-rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
-rest_target(${psd} policy/protocols/ssl/known-certs.bro)
-rest_target(${psd} policy/protocols/ssl/notary.bro)
-rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
-rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
-rest_target(${psd} policy/tuning/defaults/warnings.bro)
-rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
-rest_target(${psd} policy/tuning/track-all-assets.bro)
-rest_target(${psd} site/local-manager.bro)
-rest_target(${psd} site/local-proxy.bro)
-rest_target(${psd} site/local-worker.bro)
-rest_target(${psd} site/local.bro)
-rest_target(${psd} test-all-policy.bro)
diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index d4498b2fe3..a9ba4838db 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -46,17 +46,6 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/measurement/cluster.bro)
-rest_target(${psd} base/frameworks/measurement/main.bro)
-rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/average.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/max.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/min.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/sample.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/sum.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/unique.bro)
-rest_target(${psd} base/frameworks/measurement/plugins/variance.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -72,6 +61,17 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/sumstats/cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/main.bro)
+rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
@@ -145,10 +145,8 @@ rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/scan.bro)
 rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/conn-stats-per-host.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/metrics.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
@@ -167,7 +165,6 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
 rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/metrics.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
 rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index a34f4dd3fc..221e9b7a4a 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-04-16-03-43-22
+#open	2013-04-22-18-02-50
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -80,8 +80,8 @@ scripts/base/init-default.bro
       scripts/base/frameworks/sumstats/plugins/max.bro
       scripts/base/frameworks/sumstats/plugins/min.bro
       scripts/base/frameworks/sumstats/plugins/sample.bro
-      scripts/base/frameworks/sumstats/plugins/variance.bro
       scripts/base/frameworks/sumstats/plugins/std-dev.bro
+        scripts/base/frameworks/sumstats/plugins/variance.bro
       scripts/base/frameworks/sumstats/plugins/sum.bro
       scripts/base/frameworks/sumstats/plugins/unique.bro
     scripts/base/frameworks/sumstats/non-cluster.bro
@@ -130,4 +130,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/main.bro
   scripts/base/misc/find-checksum-offloading.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-04-16-03-43-22
+#close	2013-04-22-18-02-50

From 8f987e5066c5214f809ce3e7a0d897bca97f7955 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 22 Apr 2013 14:15:20 -0400
Subject: [PATCH 65/70] Fix a bug with path building in FTP.  Came up when
 changing the path utils.

---
 scripts/base/protocols/ftp/main.bro | 16 +++++++---------
 1 file changed, 7 insertions(+), 9 deletions(-)

diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index 69e7c331ae..b9709b6176 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -174,8 +174,9 @@ function ftp_message(s: Info)
 		if ( s$cmdarg$cmd in file_cmds )
 			{
 			local comp_path = build_path_compressed(s$cwd, arg);
-			if ( s$cwd[0] != "/" )
+			if ( comp_path[0] != "/" )
 				comp_path = cat("/", comp_path);
+
 			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path);
 			}
 		
@@ -245,16 +246,13 @@ event ftp_request(c: connection, command: string, arg: string) &priority=5
 
 event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &priority=5
 	{
-	# TODO: figure out what to do with continued FTP response (not used much)
-	#if ( cont_resp ) return;
-
-	local id = c$id;
 	set_ftp_session(c);
-	
 	c$ftp$cmdarg = get_pending_cmd(c$ftp$pending_commands, code, msg);
-	
 	c$ftp$reply_code = code;
 	c$ftp$reply_msg = msg;
+
+	# TODO: figure out what to do with continued FTP response (not used much)
+	if ( cont_resp ) return;
 	
 	# TODO: do some sort of generic clear text login processing here.
 	local response_xyz = parse_ftp_reply_code(code);
@@ -283,10 +281,10 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
 			c$ftp$passive=T;
 			
 			if ( code == 229 && data$h == [::] )
-				data$h = id$resp_h;
+				data$h = c$id$resp_h;
 			
 			ftp_data_expected[data$h, data$p] = c$ftp;
-			expect_connection(id$orig_h, data$h, data$p, ANALYZER_FILE, 5mins);
+			expect_connection(c$id$orig_h, data$h, data$p, ANALYZER_FILE, 5mins);
 			}
 		else
 			{

From 95744993825f69d84d7e6fb64799246f33625bf7 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 22 Apr 2013 14:15:37 -0400
Subject: [PATCH 66/70] Move loading variance back to where it should be
 alphabetically.

---
 scripts/base/frameworks/sumstats/plugins/__load__.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/base/frameworks/sumstats/plugins/__load__.bro b/scripts/base/frameworks/sumstats/plugins/__load__.bro
index d739af29a7..0d4c2ed302 100644
--- a/scripts/base/frameworks/sumstats/plugins/__load__.bro
+++ b/scripts/base/frameworks/sumstats/plugins/__load__.bro
@@ -2,7 +2,7 @@
 @load ./max
 @load ./min
 @load ./sample
-@load ./variance
 @load ./std-dev
 @load ./sum
 @load ./unique
+@load ./variance
\ No newline at end of file

From 91362717da2822e2af33d315c03fe4905036087c Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 22 Apr 2013 15:27:03 -0400
Subject: [PATCH 67/70] Renamed a plugin hook in sumstats framework.

---
 scripts/base/frameworks/sumstats/main.bro             | 4 ++--
 scripts/base/frameworks/sumstats/plugins/average.bro  | 2 +-
 scripts/base/frameworks/sumstats/plugins/max.bro      | 2 +-
 scripts/base/frameworks/sumstats/plugins/min.bro      | 2 +-
 scripts/base/frameworks/sumstats/plugins/sample.bro   | 2 +-
 scripts/base/frameworks/sumstats/plugins/std-dev.bro  | 2 +-
 scripts/base/frameworks/sumstats/plugins/sum.bro      | 2 +-
 scripts/base/frameworks/sumstats/plugins/unique.bro   | 2 +-
 scripts/base/frameworks/sumstats/plugins/variance.bro | 2 +-
 9 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index 643502efb4..f6bd9ebfac 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -182,7 +182,7 @@ global thresholds_store: table[string, Key] of bool = table();
 global data_added: function(ss: SumStat, key: Key, result: Result);
 
 # Prototype the hook point for plugins to do calculations.
-global add_to_reducer_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
+global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
 # Prototype the hook point for plugins to initialize any result values.
 global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
 # Prototype the hook point for plugins to merge Results.
@@ -323,7 +323,7 @@ function observe(id: string, key: Key, obs: Observation)
 		if ( obs?$num || obs?$dbl )
 			val = obs?$dbl ? obs$dbl : obs$num;
 
-		hook add_to_reducer_hook(r, val, obs, result_val);
+		hook observe_hook(r, val, obs, result_val);
 		data_added(ss, key, result);
 		}
 	}
diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro
index 35228a28f5..baabb8ca5e 100644
--- a/scripts/base/frameworks/sumstats/plugins/average.bro
+++ b/scripts/base/frameworks/sumstats/plugins/average.bro
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( AVERAGE in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro
index 0a959f2d09..532883d46e 100644
--- a/scripts/base/frameworks/sumstats/plugins/max.bro
+++ b/scripts/base/frameworks/sumstats/plugins/max.bro
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( MAX in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro
index 16c2dfc3d7..2940b34a9b 100644
--- a/scripts/base/frameworks/sumstats/plugins/min.bro
+++ b/scripts/base/frameworks/sumstats/plugins/min.bro
@@ -14,7 +14,7 @@ export {
 	};
 }
 
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( MIN in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro
index 622b160cbe..91a295775d 100644
--- a/scripts/base/frameworks/sumstats/plugins/sample.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sample.bro
@@ -29,7 +29,7 @@ function get_samples(rv: ResultVal): vector of Observation
 	return s;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( r$samples > 0 )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
index 7c2754570a..cbe9197581 100644
--- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro
+++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro
@@ -22,7 +22,7 @@ function calc_std_dev(rv: ResultVal)
 	}
 
 # This depends on the variance plugin which uses priority -5
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
 	{
 	if ( STD_DEV in r$apply )
 		calc_std_dev(rv);
diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro
index 8c8c65cd61..18056d14fb 100644
--- a/scripts/base/frameworks/sumstats/plugins/sum.bro
+++ b/scripts/base/frameworks/sumstats/plugins/sum.bro
@@ -34,7 +34,7 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( SUM in r$apply )
 		rv$sum += val;
diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro
index d3a4464d0d..f44da07e07 100644
--- a/scripts/base/frameworks/sumstats/plugins/unique.bro
+++ b/scripts/base/frameworks/sumstats/plugins/unique.bro
@@ -23,7 +23,7 @@ redef record ResultVal += {
 	unique_vals: set[Observation] &optional;
 };
 
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
 	if ( UNIQUE in r$apply )
 		{
diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro
index 29118b284b..9aadd58bdd 100644
--- a/scripts/base/frameworks/sumstats/plugins/variance.bro
+++ b/scripts/base/frameworks/sumstats/plugins/variance.bro
@@ -29,7 +29,7 @@ function calc_variance(rv: ResultVal)
 	}
 
 # Reduced priority since this depends on the average
-hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
 	{
 	if ( VARIANCE in r$apply )
 		{

From 2c689b7f4041670d687953ad56d38c84d4ce3da1 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 22 Apr 2013 15:27:14 -0400
Subject: [PATCH 68/70] Removed some dead code in scan.bro

---
 scripts/policy/misc/scan.bro | 39 +-----------------------------------
 1 file changed, 1 insertion(+), 38 deletions(-)

diff --git a/scripts/policy/misc/scan.bro b/scripts/policy/misc/scan.bro
index 9a95cf9917..508e9316a8 100644
--- a/scripts/policy/misc/scan.bro
+++ b/scripts/policy/misc/scan.bro
@@ -1,4 +1,4 @@
-##! Scan detection
+##! TCP Scan detection
 ##!
 ##! ..Authors: Sheharbano Khattak
 ##!            Seth Hall
@@ -47,22 +47,9 @@ export {
 	const addr_scan_custom_thresholds: table[port] of count &redef;
 
 	global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
-	
 	global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
 }
 
-
-#function check_addr_scan_threshold(key: SumStats::Key, val: SumStats::Result): bool
-#	{
-#	# We don't need to do this if no custom thresholds are defined.
-#	if ( |addr_scan_custom_thresholds| == 0 )
-#		return F;
-#
-#	local service = to_port(key$str);
-#	return ( service in addr_scan_custom_thresholds &&
-#	         val$sum > addr_scan_custom_thresholds[service] );
-#	}
-
 event bro_init() &priority=5
 	{
 	local r1: SumStats::Reducer = [$stream="scan.addr.fail", $apply=set(SumStats::UNIQUE)];
@@ -124,30 +111,6 @@ function add_sumstats(id: conn_id, reverse: bool)
 		victim       = id$orig_h;
 		scanned_port = id$orig_p;
 		}
-
-	# Defaults to be implemented with a hook...
-	#local transport_layer_proto = get_port_transport_proto(service);
-	#if ( suppress_UDP_scan_checks && (transport_layer_proto == udp) )
-	#	return F;
-	#else if ( suppress_TCP_scan_checks && (transport_layer_proto == tcp) )
-	#	return F;
-	#else if ( suppress_ICMP_scan_checks && (transport_layer_proto == icmp) )
-	#	return F;
-
-	# TODO: all of this whitelist/blacklist will be done 
-	#       through the upcoming hook mechanism
-	# Blacklisting/whitelisting services
-	#if ( |analyze_services| > 0 )
-	#	{
-	#	if ( service !in analyze_services )
-	#		return F;
-	#	}
-	#else if ( service in skip_services )
-	#	return F;
-	#
-	## Blacklisting/whitelisting subnets
-	#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
-	#	return F;
 	
 	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
 		SumStats::observe("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);

From 48cbb31747f39a8823260f6d86eab07b306125f9 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 25 Apr 2013 12:51:55 -0400
Subject: [PATCH 69/70] Added an automatic state limiter for threshold based
 SumStats.

---
 scripts/base/frameworks/sumstats/cluster.bro | 44 ++++++++++----
 scripts/base/frameworks/sumstats/main.bro    | 62 ++++++++++++++------
 2 files changed, 79 insertions(+), 27 deletions(-)

diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro
index 4f9743547b..405395a687 100644
--- a/scripts/base/frameworks/sumstats/cluster.bro
+++ b/scripts/base/frameworks/sumstats/cluster.bro
@@ -1,7 +1,7 @@
-##! This implements transparent cluster support for the metrics framework.
+##! This implements transparent cluster support for the SumStats framework.
 ##! Do not load this file directly.  It's only meant to be loaded automatically
 ##! and will be depending on if the cluster framework has been enabled.
-##! The goal of this script is to make metric calculation completely and
+##! The goal of this script is to make sumstats calculation completely and
 ##! transparently automated when running on a cluster.
 
 @load base/frameworks/cluster
@@ -11,7 +11,7 @@ module SumStats;
 
 export {
 	## Allows a user to decide how large of result groups the 
-	## workers should transmit values for cluster metric aggregation.
+	## workers should transmit values for cluster stats aggregation.
 	const cluster_send_in_groups_of = 50 &redef;
 	
 	## The percent of the full threshold value that needs to be met 
@@ -34,11 +34,11 @@ export {
 	const enable_intermediate_updates = T &redef;
 
 	## Event sent by the manager in a cluster to initiate the 
-	## collection of metrics values for a sumstat.
+	## collection of values for a sumstat.
 	global cluster_ss_request: event(uid: string, ssid: string);
 
-	## Event sent by nodes that are collecting metrics after receiving
-	## a request for the metric sumstat from the manager.
+	## Event sent by nodes that are collecting sumstats after receiving
+	## a request for the sumstat from the manager.
 	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
 
 	## This event is sent by the manager in a cluster to initiate the
@@ -58,10 +58,14 @@ export {
 
 	## This event is scheduled internally on workers to send result chunks.
 	global send_data: event(uid: string, ssid: string, data: ResultTable);
+
+	## This event is generated when a threshold is crossed.
+	global cluster_threshold_crossed: event(ssid: string, key: SumStats::Key, thold: Thresholding);
 }
 
 # Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request)/;
+redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request|threshold_crossed)/;
+redef Cluster::manager2worker_events += /SumStats::thresholds_reset/;
 redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
 
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
@@ -78,7 +82,7 @@ event bro_init() &priority=-100
 	reducer_store = table();
 	}
 
-# This is done on all non-manager node types in the event that a metric is 
+# This is done on all non-manager node types in the event that a sumstat is 
 # being collected somewhere other than a worker.
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
@@ -117,7 +121,7 @@ event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
 		}
 	
 	local done = F;
-	# If data is empty, this metric is done.
+	# If data is empty, this sumstat is done.
 	if ( |data| == 0 )
 		done = T;
 	
@@ -157,6 +161,19 @@ event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
 		}
 	}
 
+event SumStats::cluster_threshold_crossed(ssid: string, key: SumStats::Key, thold: Thresholding)
+	{
+	if ( ssid !in threshold_tracker )
+		threshold_tracker[ssid] = table();
+
+	threshold_tracker[ssid][key] = thold;
+	}
+
+event SumStats::thresholds_reset(ssid: string)
+	{
+	threshold_tracker[ssid] = table();
+	}
+
 @endif
 
 
@@ -192,7 +209,7 @@ event SumStats::finish_epoch(ss: SumStat)
 	{
 	if ( network_time() > zero_time )
 		{
-		#print fmt("%.6f MANAGER: breaking %s sumstat for %s metric", network_time(), ss$name, ss$id);
+		#print fmt("%.6f MANAGER: breaking %s sumstat for %s sumstat", network_time(), ss$name, ss$id);
 		local uid = unique_id("");
 		
 		if ( uid in stats_results )
@@ -212,7 +229,10 @@ event SumStats::finish_epoch(ss: SumStat)
 function data_added(ss: SumStat, key: Key, result: Result)
 	{
 	if ( check_thresholds(ss, key, result, 1.0) )
+		{
 		threshold_crossed(ss, key, result);
+		event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+		}
 	}
 	
 event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
@@ -235,7 +255,10 @@ event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result
 		local ss = stats_store[ssid];
 		local ir = key_requests[uid];
 		if ( check_thresholds(ss, key, ir, 1.0) )
+			{
 			threshold_crossed(ss, key, ir);
+			event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+			}
 
 		delete done_with[uid];
 		delete key_requests[uid];
@@ -292,6 +315,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
 			if ( check_thresholds(ss, key, local_data[key], 1.0) )
 				{
 				threshold_crossed(ss, key, local_data[key]);
+				event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
 				}
 			}
 		}
diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro
index f6bd9ebfac..ef7a34a4a4 100644
--- a/scripts/base/frameworks/sumstats/main.bro
+++ b/scripts/base/frameworks/sumstats/main.bro
@@ -133,6 +133,20 @@ export {
 	## obs: The data point to send into the stream.
 	global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
 
+	## This record is primarily used for internal threshold tracking.
+	type Thresholding: record {
+		# Internal use only.  Indicates if a simple threshold was already crossed.
+		is_threshold_crossed: bool &default=F;
+
+		# Internal use only.  Current key for threshold series.
+		threshold_series_index: count &default=0;
+	};
+
+	## This event is generated when thresholds are reset for a SumStat.
+	## 
+	## ssid: SumStats ID that thresholds were reset for.
+	global thresholds_reset: event(ssid: string);
+
 	## Helper function to represent a :bro:type:`SumStats::Key` value as 
 	## a simple string.
 	## 
@@ -144,15 +158,7 @@ export {
 
 redef record Reducer += {
 	# Internal use only.  Provides a reference back to the related SumStats by it's ID.
-	mid: string &optional;
-};
-
-type Thresholding: record {
-	# Internal use only.  Indicates if a simple threshold was already crossed.
-	is_threshold_crossed: bool &default=F;
-
-	# Internal use only.  Current key for threshold series.
-	threshold_series_index: count &default=0;
+	sid: string &optional;
 };
 
 # Internal use only.  For tracking thresholds per sumstat and key.
@@ -257,7 +263,12 @@ function reset(ss: SumStat)
 		delete result_store[ss$id];
 
 	result_store[ss$id] = table();
-	threshold_tracker[ss$id] = table();
+
+	if ( ss?$threshold || ss?$threshold_series )
+		{
+		threshold_tracker[ss$id] = table();
+		event SumStats::thresholds_reset(ss$id);
+		}
 	}
 
 function create(ss: SumStat)
@@ -274,7 +285,7 @@ function create(ss: SumStat)
 
 	for ( reducer in ss$reducers )
 		{
-		reducer$mid = ss$id;
+		reducer$sid = ss$id;
 		if ( reducer$stream !in reducer_store )
 			reducer_store[reducer$stream] = set();
 		add reducer_store[reducer$stream][reducer];
@@ -292,19 +303,36 @@ function observe(id: string, key: Key, obs: Observation)
 	# Try to add the data to all of the defined reducers.
 	for ( r in reducer_store[id] )
 		{
+		if ( r?$normalize_key )
+			key = r$normalize_key(copy(key));
+
 		# If this reducer has a predicate, run the predicate 
 		# and skip this key if the predicate return false.
 		if ( r?$pred && ! r$pred(key, obs) )
 			next;
 		
-		if ( r?$normalize_key )
-			key = r$normalize_key(copy(key));
+		local ss = stats_store[r$sid];
 		
-		local ss = stats_store[r$mid];
-		
-		if ( r$mid !in result_store )
+		# If there is a threshold and no epoch_finished callback
+		# we don't need to continue counting since the data will
+		# never be accessed.  This was leading
+		# to some state management issues when measuring 
+		# uniqueness.
+		# NOTE: this optimization could need removed in the 
+		#       future if on demand access is provided to the
+		#       SumStats results.
+		if ( ! ss?$epoch_finished &&
+		     r$sid in threshold_tracker &&
+		     key in threshold_tracker[r$sid] &&
+		     ( ss?$threshold && 
+		       threshold_tracker[r$sid][key]$is_threshold_crossed ) ||
+		     ( ss?$threshold_series &&
+		       threshold_tracker[r$sid][key]$threshold_series_index+1 == |ss$threshold_series| ) )
+			next;
+
+		if ( r$sid !in result_store )
 			result_store[ss$id] = table();
-		local results = result_store[r$mid];
+		local results = result_store[r$sid];
 
 		if ( key !in results )
 			results[key] = table();

From 04410237c2b694c204e7713fee518ad9c1e96e98 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 26 Apr 2013 12:24:46 -0400
Subject: [PATCH 70/70] Added protocol to the traceroute detection script.

---
 .../policy/misc/detect-traceroute/main.bro    | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/scripts/policy/misc/detect-traceroute/main.bro b/scripts/policy/misc/detect-traceroute/main.bro
index 9ac0f5c2f9..fd4190f8a6 100644
--- a/scripts/policy/misc/detect-traceroute/main.bro
+++ b/scripts/policy/misc/detect-traceroute/main.bro
@@ -39,11 +39,13 @@ export {
 	## The log record for the traceroute log.
 	type Info: record {
 		## Timestamp
-		ts:   time &log;
+		ts:    time &log;
 		## Address initiaing the traceroute.
-		src:  addr &log;
+		src:   addr &log;
 		## Destination address of the traceroute.
-		dst:  addr &log;
+		dst:   addr &log;
+		## Protocol used for the traceroute.
+		proto: string &log;
 	};
 
 	global log_traceroute: event(rec: Traceroute::Info);
@@ -69,14 +71,15 @@ event bro_init() &priority=5
 	                  $threshold=icmp_time_exceeded_threshold,
 	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
 	                  	{
-	                  	local parts = split1(key$str, /-/);
+	                  	local parts = split_n(key$str, /-/, F, 2);
 	                  	local src = to_addr(parts[1]);
 	                  	local dst = to_addr(parts[2]);
-	                  	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
+	                  	local proto = parts[3];
+	                  	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst, $proto=proto]);
 	                  	NOTICE([$note=Traceroute::Detected,
-	                  	        $msg=fmt("%s seems to be running traceroute", src),
-	                  	        $src=src, $dst=dst,
-	                  	        $identifier=cat(src)]);
+	                  	        $msg=fmt("%s seems to be running traceroute using %s", src, proto),
+	                  	        $src=src,
+	                  	        $identifier=cat(src,proto)]);
 	                  	}]);
 	}
 
@@ -84,10 +87,12 @@ event bro_init() &priority=5
 event signature_match(state: signature_state, msg: string, data: string)
 	{
 	if ( state$sig_id == /traceroute-detector.*/ )
-		SumStats::observe("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
+		{
+		SumStats::observe("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h,"-",get_port_transport_proto(state$conn$id$resp_p))], [$num=1]);
+		}
 	}
 
 event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
 	{
-	SumStats::observe("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
+	SumStats::observe("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h,"-",get_port_transport_proto(context$id$resp_p))], [$str=cat(c$id$orig_h)]);
 	}