Large update for the SumStats framework.

- On-demand access to sumstats results through "return from"
   functions named SumStats::request and Sumstats::request_key.
   Both functions are tested in standalone and clustered modes.

 - $name field has returned to SumStats which simplifies cluster
   code and makes the on-demand access stuff possible.

 - Clustered results can only be collected for 1 minute from their
   time of creation now instead of time of last read.

 - Thresholds use doubles instead of counts everywhere now.

 - Calculation dependency resolution occurs at start up time now
   instead of doing it at observation time which provide a minor
   cpu performance improvement.  A new plugin registration mechanism
   was created to support this change.

 - AppStats now has a minimal doc string and is broken into hook-based
   plugins.

 - AppStats and traceroute detection added to local.bro

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,17 +14,18 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( AVERAGE in r$apply )
+	register_observe_plugin(AVERAGE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$average )
 			rv$average = val;
 		else
 			rv$average += (val - rv$average) / rv$num;
-		}
+		});
 	}
 
+
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$average && rv2?$average )

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,15 +14,15 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( MAX in r$apply )
+	register_observe_plugin(MAX, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$max )
 			rv$max = val;
 		else if ( val > rv$max )
 			rv$max = val;
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,17 +14,18 @@ export {
 	};
 }
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( MIN in r$apply )
+	register_observe_plugin(MIN, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$min )
 			rv$min = val;
 		else if ( val < rv$min )
 			rv$min = val;
-		}
+		});
 	}
 
+
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
 	{
 	if ( rv1?$min && rv2?$min )

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -1,9 +1,14 @@
-@load base/frameworks/sumstats
 @load base/utils/queue
+@load ../main
 
 module SumStats;
 
 export {
+	redef enum Calculation += {
+		## Collect a sample of the last few observations.
+		SAMPLE
+	};
+
 	redef record Reducer += {
 		## A number of sample Observations to collect.
 		samples: count &default=0;
@@ -27,14 +32,14 @@ function get_samples(rv: ResultVal): vector of Observation
 	return s;
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( r$samples > 0 )
+	register_observe_plugin(SAMPLE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$samples )
 			rv$samples = Queue::init([$max_len=r$samples]);
 		Queue::put(rv$samples, obs);
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -1,5 +1,5 @@
 @load ./variance
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -21,11 +21,18 @@ function calc_std_dev(rv: ResultVal)
 		rv$std_dev = sqrt(rv$variance);
 	}
 
-# This depends on the variance plugin which uses priority -5
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
+hook std_dev_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 	{
-	if ( STD_DEV in r$apply )
+	calc_std_dev(rv);
+	}
+
+hook register_observe_plugins() &priority=-10
+	{
+	register_observe_plugin(STD_DEV, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
 		calc_std_dev(rv);
+		});
+	add_observe_plugin_dependency(STD_DEV, VARIANCE);
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -14,19 +14,19 @@ export {
 		sum: double &default=0.0;
 	};
 
-	type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
-	global sum_threshold: function(data_id: string): threshold_function;
+	#type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
+	#global sum_threshold: function(data_id: string): threshold_function;
 }
 
-function sum_threshold(data_id: string): threshold_function
-	{
-	return function(key: SumStats::Key, result: SumStats::Result): count
-		{
-		print fmt("data_id: %s", data_id);
-		print result;
-		return double_to_count(result[data_id]$sum);
-		};
-	}
+#function sum_threshold(data_id: string): threshold_function
+#	{
+#	return function(key: SumStats::Key, result: SumStats::Result): count
+#		{
+#		print fmt("data_id: %s", data_id);
+#		print result;
+#		return double_to_count(result[data_id]$sum);
+#		};
+#	}
 
 hook init_resultval_hook(r: Reducer, rv: ResultVal)
 	{
@@ -34,10 +34,12 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
 		rv$sum = 0;
 	}
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( SUM in r$apply )
+	register_observe_plugin(SUM, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+		{
 		rv$sum += val;
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -23,15 +23,15 @@ redef record ResultVal += {
 	unique_vals: set[Observation] &optional;
 };
 
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+hook register_observe_plugins()
 	{
-	if ( UNIQUE in r$apply )
+	register_observe_plugin(UNIQUE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( ! rv?$unique_vals )
 			rv$unique_vals=set();
 		add rv$unique_vals[obs];
 		rv$unique = |rv$unique_vals|;
-		}
+		});
 	}
 
 hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -1,5 +1,5 @@
 @load ./average
-@load base/frameworks/sumstats
+@load ../main
 
 module SumStats;
 
@@ -28,17 +28,17 @@ function calc_variance(rv: ResultVal)
 	rv$variance = (rv$num > 1) ? rv$var_s/(rv$num-1) : 0.0;
 	}
 
-# Reduced priority since this depends on the average
-hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
+hook register_observe_plugins() &priority=-5
 	{
-	if ( VARIANCE in r$apply )
+	register_observe_plugin(VARIANCE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
 		{
 		if ( rv$num > 1 )
 			rv$var_s += ((val - rv$prev_avg) * (val - rv$average));
 
 		calc_variance(rv);
 		rv$prev_avg = rv$average;
-		}
+		});
+	add_observe_plugin_dependency(VARIANCE, AVERAGE);
 	}
 
 # Reduced priority since this depends on the average