Add more documentation to sumstats framework scripts

scripts/base/frameworks/sumstats/main.bro

@@ -5,7 +5,8 @@
 module SumStats;
 
 export {
-	## The various calculations are all defined as plugins.
+	## Type to represent the calculations that are available.  The calculations
+	## are all defined as plugins.
 	type Calculation: enum {
 		PLACEHOLDER
 	};
@@ -39,6 +40,7 @@ export {
 		str:  string &optional;
 	};
 
+	## Represents a reducer.
 	type Reducer: record {
 		## Observation stream identifier for the reducer
 		## to attach to.
@@ -56,7 +58,7 @@ export {
 		normalize_key:  function(key: SumStats::Key): Key &optional;
 	};
 
-	## Value calculated for an observation stream fed into a reducer.
+	## Result calculated for an observation stream fed into a reducer.
 	## Most of the fields are added by plugins.
 	type ResultVal: record {
 		## The time when the first observation was added to
@@ -71,14 +73,15 @@ export {
 		num:    count &default=0;
 	};
 
-	## Type to store results for multiple reducers.
+	## Type to store a table of results for multiple reducers indexed by
+	## observation stream identifier.
 	type Result: table[string] of ResultVal;
 
 	## Type to store a table of sumstats results indexed by keys.
 	type ResultTable: table[Key] of Result;
 
-	## SumStats represent an aggregation of reducers along with
-	## mechanisms to handle various situations like the epoch ending
+	## Represents a SumStat, which consists of an aggregation of reducers along
+	## with mechanisms to handle various situations like the epoch ending
 	## or thresholds being crossed.
 	##
 	## It's best to not access any global state outside
@@ -101,21 +104,28 @@ export {
 		## The reducers for the SumStat.
 		reducers:           set[Reducer];
 
-		## Provide a function to calculate a value from the
-		## :bro:see:`SumStats::Result` structure which will be used
-		## for thresholding.
-		## This is required if a *threshold* value is given.
+		## A function that will be called once for each observation in order
+		## to calculate a value from the :bro:see:`SumStats::Result` structure
+		## which will be used for thresholding.
+		## This function is required if a *threshold* value or
+		## a *threshold_series* is given.
 		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): double &optional;
 
-		## The threshold value for calling the
-		## *threshold_crossed* callback.
+		## The threshold value for calling the *threshold_crossed* callback.
+		## If you need more than one threshold value, then use
+		## *threshold_series* instead.
 		threshold:          double            &optional;
 
-		## A series of thresholds for calling the
-		## *threshold_crossed* callback.
+		## A series of thresholds for calling the *threshold_crossed*
+		## callback.  These thresholds must be listed in ascending order,
+		## because a threshold is not checked until the preceding one has
+		## been crossed.
 		threshold_series:   vector of double  &optional;
 
 		## A callback that is called when a threshold is crossed.
+		## A threshold is crossed when the value returned from *threshold_val*
+		## is greater than or equal to the threshold value, but only the first
+		## time this happens within an epoch.
 		threshold_crossed:  function(key: SumStats::Key, result: SumStats::Result) &optional;
 
 		## A callback that receives each of the results at the
@@ -130,6 +140,8 @@ export {
 	};
 
 	## Create a summary statistic.
+	##
+	## ss: The SumStat to create.
 	global create: function(ss: SumStats::SumStat);
 
 	## Add data into an observation stream. This should be

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -1,3 +1,5 @@
+##! Calculate the average.
+
 @load ../main
 
 module SumStats;
@@ -9,7 +11,7 @@ export {
 	};
 
 	redef record ResultVal += {
-		## For numeric data, this calculates the average of all values.
+		## For numeric data, this is the average of all values.
 		average: double &optional;
 	};
 }

scripts/base/frameworks/sumstats/plugins/hll_unique.bro

@@ -1,3 +1,5 @@
+##! Calculate the number of unique values (using the HyperLogLog algorithm).
+
 @load base/frameworks/sumstats
 
 module SumStats;

scripts/base/frameworks/sumstats/plugins/last.bro

@@ -1,3 +1,5 @@
+##! Keep the last X observations.
+
 @load base/frameworks/sumstats
 @load base/utils/queue
 

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -1,3 +1,5 @@
+##! Find the maximum value.
+
 @load ../main
 
 module SumStats;
@@ -9,7 +11,7 @@ export {
 	};
 
 	redef record ResultVal += {
-		## For numeric data, this tracks the maximum value given.
+		## For numeric data, this tracks the maximum value.
 		max: double &optional;
 	};
 }

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -1,3 +1,5 @@
+##! Find the minimum value.
+
 @load ../main
 
 module SumStats;
@@ -9,7 +11,7 @@ export {
 	};
 
 	redef record ResultVal += {
-		## For numeric data, this tracks the minimum value given.
+		## For numeric data, this tracks the minimum value.
 		min: double &optional;
 	};
 }

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -1,3 +1,5 @@
+##! Keep a random sample of values.
+
 @load base/frameworks/sumstats/main
 
 module SumStats;
@@ -10,7 +12,7 @@ export {
 	};
 
 	redef record Reducer += {
-		## A number of sample Observations to collect.
+		## The number of sample Observations to collect.
 		num_samples: count &default=0;
 	};
 

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -1,3 +1,5 @@
+##! Calculate the standard deviation.
+
 @load ./variance
 @load ../main
 
@@ -5,7 +7,7 @@ module SumStats;
 
 export {
 	redef enum Calculation += {
-		## Find the standard deviation of the values.
+		## Calculate the standard deviation of the values.
 		STD_DEV
 	};
 

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -1,11 +1,13 @@
+##! Calculate the sum.
+
 @load ../main
 
 module SumStats;
 
 export {
 	redef enum Calculation += {
-		## Sums the values given.  For string values,
-		## this will be the number of strings given.
+		## Calculate the sum of the values.  For string values,
+		## this will be the number of strings.
 		SUM
 	};
 

scripts/base/frameworks/sumstats/plugins/topk.bro

@@ -1,3 +1,5 @@
+##! Keep the top-k (i.e., most frequently occurring) observations.
+
 @load base/frameworks/sumstats
 
 module SumStats;
@@ -9,10 +11,13 @@ export {
 	};
 
 	redef enum Calculation += {
+		## Keep a top-k list of values.
 		TOPK
 	};
 
 	redef record ResultVal += {
+		## A handle which can be passed to some built-in functions to get
+		## the top-k results.
 		topk: opaque of topk &optional;
 	};
 

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -1,10 +1,12 @@
+##! Calculate the number of unique values.
+
 @load ../main
 
 module SumStats;
 
 export {
 	redef record Reducer += {
-		## Maximum number of unique elements to store.
+		## Maximum number of unique values to store.
 		unique_max: count &optional;
 	};
 
@@ -15,7 +17,7 @@ export {
 
 	redef record ResultVal += {
 		## If cardinality is being tracked, the number of unique
-		## items is tracked here.
+		## values is tracked here.
 		unique: count &default=0;
 	};
 }

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -1,3 +1,5 @@
+##! Calculate the variance.
+
 @load ./average
 @load ../main
 
@@ -5,12 +7,12 @@ module SumStats;
 
 export {
 	redef enum Calculation += {
-		## Find the variance of the values.
+		## Calculate the variance of the values.
 		VARIANCE
 	};
 
 	redef record ResultVal += {
-		## For numeric data, this calculates the variance.
+		## For numeric data, this is the variance.
 		variance: double &optional;
 	};
 }