Checkpoint

scripts/base/frameworks/measurement/plugins/__load__.bro

@@ -0,0 +1,7 @@
+@load ./average
+@load ./max
+@load ./min
+@load ./std-dev
+@load ./sum
+@load ./unique
+@load ./variance

scripts/base/frameworks/measurement/plugins/average.bro

@@ -0,0 +1,35 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Calculate the average of the values.
+		AVERAGE
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the average of all values.
+		average: double &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( AVERAGE in filter$measure )
+		{
+		if ( ! result?$average ) 
+			result$average = val;
+		else
+			result$average += (val - result$average) / result$num;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$average && rv2?$average )
+		result$average = ((rv1$average*rv1$num) + (rv2$average*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$average )
+		result$average = rv1$average;
+	else if ( rv2?$average )
+		result$average = rv2$average;
+	}

scripts/base/frameworks/measurement/plugins/max.bro

@@ -0,0 +1,37 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the maximum value.
+		MAX
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the maximum value given.
+		max:      double        &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( MAX in filter$measure )
+		{
+		if ( ! result?$max ) 
+			result$max = val;
+		else if ( val > result$max )
+			result$max = val;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$max && rv2?$max )
+		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
+	else if ( rv1?$max )
+		result$max = rv1$max;
+	else if ( rv2?$max )
+		result$max = rv2$max;
+	}
+
+

scripts/base/frameworks/measurement/plugins/min.bro

@@ -0,0 +1,35 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the minimum value.
+		MIN
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the minimum value given.
+		min:      double        &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( MIN in filter$measure )
+		{
+		if ( ! result?$min ) 
+			result$min = val;
+		else if ( val < result$min )
+			result$min = val;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$min && rv2?$min )
+		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
+	else if ( rv1?$min )
+		result$min = rv1$min;
+	else if ( rv2?$min )
+		result$min = rv2$min;
+	}

scripts/base/frameworks/measurement/plugins/std-dev.bro

@@ -0,0 +1,36 @@
+@load ./sum
+@load ./variance
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the standard deviation of the values.
+		STD_DEV
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the standard deviation.
+		std_dev:  double &log &optional;
+	};
+}
+
+# This depends on the variance plugin which uses priority -5
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=-10
+	{
+	if ( STD_DEV in filter$measure )
+		{
+		if ( result?$variance )
+			result$std_dev = sqrt(result$variance);
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
+	{
+	if ( rv1?$sum || rv2?$sum )
+		{
+		result$sum = rv1?$sum ? rv1$sum : 0;
+		if ( rv2?$sum )
+			result$sum += rv2$sum;
+		}
+	}

scripts/base/frameworks/measurement/plugins/sum.bro

@@ -0,0 +1,35 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Sums the values given.  For string values,
+		## this will be the number of strings given.
+		SUM
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the sum of all values.
+		sum:      double        &log &optional;
+	};
+}
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( SUM in filter$measure )
+		{
+		if ( ! result?$sum ) 
+			result$sum = 0;
+		result$sum += val;
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$sum || rv2?$sum )
+		{
+		result$sum = rv1?$sum ? rv1$sum : 0;
+		if ( rv2?$sum )
+			result$sum += rv2$sum;
+		}
+	}

scripts/base/frameworks/measurement/plugins/unique.bro

@@ -0,0 +1,51 @@
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Calculate the number of unique values.
+		UNIQUE
+	};
+
+	redef record ResultVal += {
+		## If cardinality is being tracked, the number of unique
+		## items is tracked here.
+		unique: count &log &optional;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  This is not meant to be publically available 
+	# because we don't want to trust that we can inspect the values 
+	# since we will like move to a probalistic data structure in the future.
+	# TODO: in the future this will optionally be a hyperloglog structure
+	unique_vals: set[DataPoint] &optional;
+};
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal)
+	{
+	if ( UNIQUE in filter$measure )
+		{
+		if ( ! result?$unique_vals ) 
+			result$unique_vals=set();
+		add result$unique_vals[data];
+		}
+	}
+
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$unique_vals || rv2?$unique_vals )
+		{
+		if ( rv1?$unique_vals )
+			result$unique_vals = rv1$unique_vals;
+		
+		if ( rv2?$unique_vals )
+			if ( ! result?$unique_vals )
+				result$unique_vals = rv2$unique_vals;
+			else
+				for ( val2 in rv2$unique_vals )
+					add result$unique_vals[val2];
+
+		result$unique = |result$unique_vals|;
+		}
+	}

scripts/base/frameworks/measurement/plugins/variance.bro

@@ -0,0 +1,65 @@
+@load ./average
+
+module Metrics;
+
+export {
+	redef enum Calculation += { 
+		## Find the variance of the values.
+		VARIANCE
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the variance.
+		variance: double &log &optional;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  Used for incrementally calculating variance.
+	prev_avg: double &optional;
+
+	# Internal use only.  For calculating incremental variance.
+	var_s: double &optional;
+};
+
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=5
+	{
+	if ( VARIANCE in filter$measure )
+		result$prev_avg = result$average;
+	}
+
+# Reduced priority since this depends on the average
+hook add_to_calculation(filter: Filter, val: double, data: DataPoint, result: ResultVal) &priority=-5
+	{
+	if ( VARIANCE in filter$measure )
+		{
+		if ( ! result?$var_s )
+			result$var_s = 0.0;
+		result$var_s += (val - result$prev_avg) * (val - result$average);
+		result$variance = (val > 0) ? result$var_s/val : 0.0;
+		}
+	}
+
+# Reduced priority since this depends on the average
+hook plugin_merge_measurements(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-5
+	{
+	if ( rv1?$var_s && rv2?$var_s )
+		{
+		local rv1_avg_sq = (rv1$average - result$average);
+		rv1_avg_sq = rv1_avg_sq*rv1_avg_sq;
+		local rv2_avg_sq = (rv2$average - result$average);
+		rv2_avg_sq = rv2_avg_sq*rv2_avg_sq;
+		result$var_s = rv1$num*(rv1$var_s/rv1$num + rv1_avg_sq) + rv2$num*(rv2$var_s/rv2$num + rv2_avg_sq);
+		}
+	else if ( rv1?$var_s )
+		result$var_s = rv1$var_s;
+	else if ( rv2?$var_s )
+		result$var_s = rv2$var_s;
+
+	if ( rv1?$prev_avg && rv2?$prev_avg )
+		result$prev_avg = ((rv1$prev_avg*rv1$num) + (rv2$prev_avg*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$prev_avg )
+		result$prev_avg = rv1$prev_avg;
+	else if ( rv2?$prev_avg )
+		result$prev_avg = rv2$prev_avg;
+	}