diff --git a/scripts/base/frameworks/sumstats/main.bro b/scripts/base/frameworks/sumstats/main.bro index 8dbdb61edd..edd80ede0f 100644 --- a/scripts/base/frameworks/sumstats/main.bro +++ b/scripts/base/frameworks/sumstats/main.bro @@ -5,7 +5,8 @@ module SumStats; export { - ## The various calculations are all defined as plugins. + ## Type to represent the calculations that are available. The calculations + ## are all defined as plugins. type Calculation: enum { PLACEHOLDER }; @@ -39,6 +40,7 @@ export { str: string &optional; }; + ## Represents a reducer. type Reducer: record { ## Observation stream identifier for the reducer ## to attach to. @@ -56,7 +58,7 @@ export { normalize_key: function(key: SumStats::Key): Key &optional; }; - ## Value calculated for an observation stream fed into a reducer. + ## Result calculated for an observation stream fed into a reducer. ## Most of the fields are added by plugins. type ResultVal: record { ## The time when the first observation was added to @@ -71,14 +73,15 @@ export { num: count &default=0; }; - ## Type to store results for multiple reducers. + ## Type to store a table of results for multiple reducers indexed by + ## observation stream identifier. type Result: table[string] of ResultVal; ## Type to store a table of sumstats results indexed by keys. type ResultTable: table[Key] of Result; - ## SumStats represent an aggregation of reducers along with - ## mechanisms to handle various situations like the epoch ending + ## Represents a SumStat, which consists of an aggregation of reducers along + ## with mechanisms to handle various situations like the epoch ending ## or thresholds being crossed. ## ## It's best to not access any global state outside @@ -101,21 +104,28 @@ export { ## The reducers for the SumStat. reducers: set[Reducer]; - ## Provide a function to calculate a value from the - ## :bro:see:`SumStats::Result` structure which will be used - ## for thresholding. - ## This is required if a *threshold* value is given. + ## A function that will be called once for each observation in order + ## to calculate a value from the :bro:see:`SumStats::Result` structure + ## which will be used for thresholding. + ## This function is required if a *threshold* value or + ## a *threshold_series* is given. threshold_val: function(key: SumStats::Key, result: SumStats::Result): double &optional; - ## The threshold value for calling the - ## *threshold_crossed* callback. + ## The threshold value for calling the *threshold_crossed* callback. + ## If you need more than one threshold value, then use + ## *threshold_series* instead. threshold: double &optional; - ## A series of thresholds for calling the - ## *threshold_crossed* callback. + ## A series of thresholds for calling the *threshold_crossed* + ## callback. These thresholds must be listed in ascending order, + ## because a threshold is not checked until the preceding one has + ## been crossed. threshold_series: vector of double &optional; ## A callback that is called when a threshold is crossed. + ## A threshold is crossed when the value returned from *threshold_val* + ## is greater than or equal to the threshold value, but only the first + ## time this happens within an epoch. threshold_crossed: function(key: SumStats::Key, result: SumStats::Result) &optional; ## A callback that receives each of the results at the @@ -130,6 +140,8 @@ export { }; ## Create a summary statistic. + ## + ## ss: The SumStat to create. global create: function(ss: SumStats::SumStat); ## Add data into an observation stream. This should be diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro index 8f7f7b568f..160ca64d78 100644 --- a/scripts/base/frameworks/sumstats/plugins/average.bro +++ b/scripts/base/frameworks/sumstats/plugins/average.bro @@ -1,3 +1,5 @@ +##! Calculate the average. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this calculates the average of all values. + ## For numeric data, this is the average of all values. average: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/hll_unique.bro b/scripts/base/frameworks/sumstats/plugins/hll_unique.bro index 494cbf4667..43cafcff7f 100644 --- a/scripts/base/frameworks/sumstats/plugins/hll_unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/hll_unique.bro @@ -1,3 +1,5 @@ +##! Calculate the number of unique values (using the HyperLogLog algorithm). + @load base/frameworks/sumstats module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index 430c2e375b..ca04114f61 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -1,3 +1,5 @@ +##! Keep the last X observations. + @load base/frameworks/sumstats @load base/utils/queue diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro index d43ad9dc38..adcc6ae113 100644 --- a/scripts/base/frameworks/sumstats/plugins/max.bro +++ b/scripts/base/frameworks/sumstats/plugins/max.bro @@ -1,3 +1,5 @@ +##! Find the maximum value. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this tracks the maximum value given. + ## For numeric data, this tracks the maximum value. max: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro index 014755cf32..22cab1009c 100644 --- a/scripts/base/frameworks/sumstats/plugins/min.bro +++ b/scripts/base/frameworks/sumstats/plugins/min.bro @@ -1,3 +1,5 @@ +##! Find the minimum value. + @load ../main module SumStats; @@ -9,7 +11,7 @@ export { }; redef record ResultVal += { - ## For numeric data, this tracks the minimum value given. + ## For numeric data, this tracks the minimum value. min: double &optional; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index 809d696896..0200e85949 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -1,3 +1,5 @@ +##! Keep a random sample of values. + @load base/frameworks/sumstats/main module SumStats; @@ -10,7 +12,7 @@ export { }; redef record Reducer += { - ## A number of sample Observations to collect. + ## The number of sample Observations to collect. num_samples: count &default=0; }; diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro index 2e5b95b212..bfb02c82cc 100644 --- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro +++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro @@ -1,3 +1,5 @@ +##! Calculate the standard deviation. + @load ./variance @load ../main @@ -5,7 +7,7 @@ module SumStats; export { redef enum Calculation += { - ## Find the standard deviation of the values. + ## Calculate the standard deviation of the values. STD_DEV }; diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro index 074b4b72f3..fb1d96bcd4 100644 --- a/scripts/base/frameworks/sumstats/plugins/sum.bro +++ b/scripts/base/frameworks/sumstats/plugins/sum.bro @@ -1,11 +1,13 @@ +##! Calculate the sum. + @load ../main module SumStats; export { redef enum Calculation += { - ## Sums the values given. For string values, - ## this will be the number of strings given. + ## Calculate the sum of the values. For string values, + ## this will be the number of strings. SUM }; diff --git a/scripts/base/frameworks/sumstats/plugins/topk.bro b/scripts/base/frameworks/sumstats/plugins/topk.bro index 0ef0f01393..e7107cb4fb 100644 --- a/scripts/base/frameworks/sumstats/plugins/topk.bro +++ b/scripts/base/frameworks/sumstats/plugins/topk.bro @@ -1,3 +1,5 @@ +##! Keep the top-k (i.e., most frequently occurring) observations. + @load base/frameworks/sumstats module SumStats; @@ -9,10 +11,13 @@ export { }; redef enum Calculation += { + ## Keep a top-k list of values. TOPK }; redef record ResultVal += { + ## A handle which can be passed to some built-in functions to get + ## the top-k results. topk: opaque of topk &optional; }; diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro index abfbe3669d..5fcaa1dc3c 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/unique.bro @@ -1,10 +1,12 @@ +##! Calculate the number of unique values. + @load ../main module SumStats; export { redef record Reducer += { - ## Maximum number of unique elements to store. + ## Maximum number of unique values to store. unique_max: count &optional; }; @@ -15,7 +17,7 @@ export { redef record ResultVal += { ## If cardinality is being tracked, the number of unique - ## items is tracked here. + ## values is tracked here. unique: count &default=0; }; } diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro index 12d30cc4fe..989bf07eaf 100644 --- a/scripts/base/frameworks/sumstats/plugins/variance.bro +++ b/scripts/base/frameworks/sumstats/plugins/variance.bro @@ -1,3 +1,5 @@ +##! Calculate the variance. + @load ./average @load ../main @@ -5,12 +7,12 @@ module SumStats; export { redef enum Calculation += { - ## Find the variance of the values. + ## Calculate the variance of the values. VARIANCE }; redef record ResultVal += { - ## For numeric data, this calculates the variance. + ## For numeric data, this is the variance. variance: double &optional; }; }