mirror of
https://github.com/zeek/zeek.git
synced 2025-10-05 08:08:19 +00:00
Large update for the SumStats framework.
- On-demand access to sumstats results through "return from" functions named SumStats::request and Sumstats::request_key. Both functions are tested in standalone and clustered modes. - $name field has returned to SumStats which simplifies cluster code and makes the on-demand access stuff possible. - Clustered results can only be collected for 1 minute from their time of creation now instead of time of last read. - Thresholds use doubles instead of counts everywhere now. - Calculation dependency resolution occurs at start up time now instead of doing it at observation time which provide a minor cpu performance improvement. A new plugin registration mechanism was created to support this change. - AppStats now has a minimal doc string and is broken into hook-based plugins. - AppStats and traceroute detection added to local.bro
This commit is contained in:
Seth Hall
parent
7d7d30e1f7
commit
bec965b66f
34 changed files with 687 additions and 277 deletions
|
|
@ -27,39 +27,34 @@ export {
|
|||
## performed. In practice this should hopefully have a minimal effect.
|
||||
const max_outstanding_global_views = 10 &redef;
|
||||
|
||||
## Intermediate updates can cause overload situations on very large clusters. This
|
||||
## option may help reduce load and correct intermittent problems. The goal for this
|
||||
## option is also meant to be temporary.
|
||||
const enable_intermediate_updates = T &redef;
|
||||
|
||||
## Event sent by the manager in a cluster to initiate the collection of values for
|
||||
## a sumstat.
|
||||
global cluster_ss_request: event(uid: string, ssid: string);
|
||||
global cluster_ss_request: event(uid: string, ss_name: string, cleanup: bool);
|
||||
|
||||
## Event sent by nodes that are collecting sumstats after receiving a request for
|
||||
## the sumstat from the manager.
|
||||
global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
|
||||
global cluster_ss_response: event(uid: string, ss_name: string, data: ResultTable, done: bool);
|
||||
|
||||
## This event is sent by the manager in a cluster to initiate the collection of
|
||||
## a single key value from a sumstat. It's typically used to get intermediate
|
||||
## updates before the break interval triggers to speed detection of a value
|
||||
## crossing a threshold.
|
||||
global cluster_key_request: event(uid: string, ssid: string, key: Key);
|
||||
global cluster_key_request: event(uid: string, ss_name: string, key: Key, cleanup: bool);
|
||||
|
||||
## This event is sent by nodes in response to a
|
||||
## :bro:id:`SumStats::cluster_key_request` event.
|
||||
global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
|
||||
global cluster_key_response: event(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool);
|
||||
|
||||
## This is sent by workers to indicate that they crossed the percent
|
||||
## of the current threshold by the percentage defined globally in
|
||||
## :bro:id:`SumStats::cluster_request_global_view_percent`
|
||||
global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
|
||||
global cluster_key_intermediate_response: event(ss_name: string, key: SumStats::Key);
|
||||
|
||||
## This event is scheduled internally on workers to send result chunks.
|
||||
global send_data: event(uid: string, ssid: string, data: ResultTable);
|
||||
global send_data: event(uid: string, ss_name: string, data: ResultTable, cleanup: bool);
|
||||
|
||||
## This event is generated when a threshold is crossed.
|
||||
global cluster_threshold_crossed: event(ssid: string, key: SumStats::Key, thold: Thresholding);
|
||||
global cluster_threshold_crossed: event(ss_name: string, key: SumStats::Key, thold: Thresholding);
|
||||
}
|
||||
|
||||
# Add events to the cluster framework to make this work.
|
||||
|
|
@ -74,44 +69,38 @@ redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_resp
|
|||
# an intermediate result has been received.
|
||||
global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
|
||||
|
||||
event bro_init() &priority=-100
|
||||
{
|
||||
# The manager is the only host allowed to track these.
|
||||
stats_store = table();
|
||||
reducer_store = table();
|
||||
}
|
||||
|
||||
# This is done on all non-manager node types in the event that a sumstat is
|
||||
# being collected somewhere other than a worker.
|
||||
function data_added(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
# If an intermediate update for this value was sent recently, don't send
|
||||
# it again.
|
||||
if ( [ss$id, key] in recent_global_view_keys )
|
||||
if ( [ss$name, key] in recent_global_view_keys )
|
||||
return;
|
||||
|
||||
# If val is 5 and global view % is 0.1 (10%), pct_val will be 50. If that
|
||||
# crosses the full threshold then it's a candidate to send as an
|
||||
# intermediate update.
|
||||
if ( enable_intermediate_updates &&
|
||||
check_thresholds(ss, key, result, cluster_request_global_view_percent) )
|
||||
if ( check_thresholds(ss, key, result, cluster_request_global_view_percent) )
|
||||
{
|
||||
# kick off intermediate update
|
||||
event SumStats::cluster_key_intermediate_response(ss$id, key);
|
||||
++recent_global_view_keys[ss$id, key];
|
||||
event SumStats::cluster_key_intermediate_response(ss$name, key);
|
||||
++recent_global_view_keys[ss$name, key];
|
||||
}
|
||||
}
|
||||
|
||||
event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
|
||||
event SumStats::send_data(uid: string, ss_name: string, data: ResultTable, cleanup: bool)
|
||||
{
|
||||
#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
|
||||
|
||||
local local_data: ResultTable = table();
|
||||
local incoming_data: ResultTable = cleanup ? data : copy(data);
|
||||
|
||||
local num_added = 0;
|
||||
for ( key in data )
|
||||
for ( key in incoming_data )
|
||||
{
|
||||
local_data[key] = data[key];
|
||||
delete data[key];
|
||||
local_data[key] = incoming_data[key];
|
||||
delete incoming_data[key];
|
||||
|
||||
# Only send cluster_send_in_groups_of at a time. Queue another
|
||||
# event to send the next group.
|
||||
|
|
@ -121,56 +110,56 @@ event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
|
|||
|
||||
local done = F;
|
||||
# If data is empty, this sumstat is done.
|
||||
if ( |data| == 0 )
|
||||
if ( |incoming_data| == 0 )
|
||||
done = T;
|
||||
|
||||
event SumStats::cluster_ss_response(uid, ssid, local_data, done);
|
||||
event SumStats::cluster_ss_response(uid, ss_name, local_data, done);
|
||||
if ( ! done )
|
||||
schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
|
||||
schedule 0.01 sec { SumStats::send_data(uid, ss_name, incoming_data, T) };
|
||||
}
|
||||
|
||||
event SumStats::cluster_ss_request(uid: string, ssid: string)
|
||||
event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool)
|
||||
{
|
||||
#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
|
||||
|
||||
# Initiate sending all of the data for the requested stats.
|
||||
if ( ssid in result_store )
|
||||
event SumStats::send_data(uid, ssid, result_store[ssid]);
|
||||
if ( ss_name in result_store )
|
||||
event SumStats::send_data(uid, ss_name, result_store[ss_name], cleanup);
|
||||
else
|
||||
event SumStats::send_data(uid, ssid, table());
|
||||
event SumStats::send_data(uid, ss_name, table(), cleanup);
|
||||
|
||||
# Lookup the actual sumstats and reset it, the reference to the data
|
||||
# currently stored will be maintained internally by the send_data event.
|
||||
if ( ssid in stats_store )
|
||||
reset(stats_store[ssid]);
|
||||
if ( ss_name in stats_store && cleanup )
|
||||
reset(stats_store[ss_name]);
|
||||
}
|
||||
|
||||
event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
|
||||
event SumStats::cluster_key_request(uid: string, ss_name: string, key: Key, cleanup: bool)
|
||||
{
|
||||
if ( ssid in result_store && key in result_store[ssid] )
|
||||
if ( ss_name in result_store && key in result_store[ss_name] )
|
||||
{
|
||||
#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
|
||||
event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]);
|
||||
event SumStats::cluster_key_response(uid, ss_name, key, result_store[ss_name][key], cleanup);
|
||||
}
|
||||
else
|
||||
{
|
||||
# We need to send an empty response if we don't have the data so that the manager
|
||||
# can know that it heard back from all of the workers.
|
||||
event SumStats::cluster_key_response(uid, ssid, key, table());
|
||||
event SumStats::cluster_key_response(uid, ss_name, key, table(), cleanup);
|
||||
}
|
||||
}
|
||||
|
||||
event SumStats::cluster_threshold_crossed(ssid: string, key: SumStats::Key, thold: Thresholding)
|
||||
event SumStats::cluster_threshold_crossed(ss_name: string, key: SumStats::Key, thold: Thresholding)
|
||||
{
|
||||
if ( ssid !in threshold_tracker )
|
||||
threshold_tracker[ssid] = table();
|
||||
if ( ss_name !in threshold_tracker )
|
||||
threshold_tracker[ss_name] = table();
|
||||
|
||||
threshold_tracker[ssid][key] = thold;
|
||||
threshold_tracker[ss_name][key] = thold;
|
||||
}
|
||||
|
||||
event SumStats::thresholds_reset(ssid: string)
|
||||
event SumStats::thresholds_reset(ss_name: string)
|
||||
{
|
||||
threshold_tracker[ssid] = table();
|
||||
threshold_tracker[ss_name] = table();
|
||||
}
|
||||
|
||||
@endif
|
||||
|
|
@ -181,7 +170,7 @@ event SumStats::thresholds_reset(ssid: string)
|
|||
# This variable is maintained by manager nodes as they collect and aggregate
|
||||
# results.
|
||||
# Index on a uid.
|
||||
global stats_results: table[string] of ResultTable &read_expire=1min;
|
||||
global stats_results: table[string] of ResultTable &create_expire=1min &default=table();
|
||||
|
||||
# This variable is maintained by manager nodes to track how many "dones" they
|
||||
# collected per collection unique id. Once the number of results for a uid
|
||||
|
|
@ -189,18 +178,18 @@ global stats_results: table[string] of ResultTable &read_expire=1min;
|
|||
# result is written out and deleted from here.
|
||||
# Indexed on a uid.
|
||||
# TODO: add an &expire_func in case not all results are received.
|
||||
global done_with: table[string] of count &read_expire=1min &default=0;
|
||||
global done_with: table[string] of count &create_expire=1min &default=0;
|
||||
|
||||
# This variable is maintained by managers to track intermediate responses as
|
||||
# they are getting a global view for a certain key.
|
||||
# Indexed on a uid.
|
||||
global key_requests: table[string] of Result &read_expire=1min;
|
||||
global key_requests: table[string] of Result &create_expire=1min;
|
||||
|
||||
# This variable is maintained by managers to prevent overwhelming communication due
|
||||
# to too many intermediate updates. Each sumstat is tracked separately so that
|
||||
# one won't overwhelm and degrade other quieter sumstats.
|
||||
# Indexed on a sumstat id.
|
||||
global outstanding_global_views: table[string] of count &default=0;
|
||||
global outstanding_global_views: table[string] of count &create_expire=1min &default=0;
|
||||
|
||||
const zero_time = double_to_time(0.0);
|
||||
# Managers handle logging.
|
||||
|
|
@ -216,7 +205,7 @@ event SumStats::finish_epoch(ss: SumStat)
|
|||
stats_results[uid] = table();
|
||||
|
||||
# Request data from peers.
|
||||
event SumStats::cluster_ss_request(uid, ss$id);
|
||||
event SumStats::cluster_ss_request(uid, ss$name, T);
|
||||
}
|
||||
|
||||
# Schedule the next finish_epoch event.
|
||||
|
|
@ -230,20 +219,20 @@ function data_added(ss: SumStat, key: Key, result: Result)
|
|||
if ( check_thresholds(ss, key, result, 1.0) )
|
||||
{
|
||||
threshold_crossed(ss, key, result);
|
||||
event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
|
||||
event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
|
||||
}
|
||||
}
|
||||
|
||||
event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
|
||||
event SumStats::cluster_key_response(uid: string, ss_name: string, key: Key, result: Result, cleanup: bool)
|
||||
{
|
||||
#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
|
||||
|
||||
# We only want to try and do a value merge if there are actually measured datapoints
|
||||
# in the Result.
|
||||
if ( uid in key_requests )
|
||||
key_requests[uid] = compose_results(key_requests[uid], result);
|
||||
else
|
||||
if ( uid !in key_requests || |key_requests[uid]| == 0 )
|
||||
key_requests[uid] = result;
|
||||
else
|
||||
key_requests[uid] = compose_results(key_requests[uid], result);
|
||||
|
||||
# Mark that a worker is done.
|
||||
++done_with[uid];
|
||||
|
|
@ -251,30 +240,39 @@ event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result
|
|||
#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
local ss = stats_store[ssid];
|
||||
local ss = stats_store[ss_name];
|
||||
local ir = key_requests[uid];
|
||||
if ( check_thresholds(ss, key, ir, 1.0) )
|
||||
{
|
||||
threshold_crossed(ss, key, ir);
|
||||
event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
|
||||
event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
|
||||
}
|
||||
|
||||
delete done_with[uid];
|
||||
delete key_requests[uid];
|
||||
# Check that there is an outstanding view before subtracting.
|
||||
if ( outstanding_global_views[ssid] > 0 )
|
||||
--outstanding_global_views[ssid];
|
||||
if ( cleanup )
|
||||
{
|
||||
# We only want to delete the data if this is a non dynamic
|
||||
# request because the dynamic requests use when statements
|
||||
# and the data needs to remain available.
|
||||
delete key_requests[uid];
|
||||
delete done_with[uid];
|
||||
|
||||
# Check that there is an outstanding view before subtracting.
|
||||
# Global views only apply to non-dynamic requests. Dynamic
|
||||
# requests must be serviced.
|
||||
if ( outstanding_global_views[ss_name] > 0 )
|
||||
--outstanding_global_views[ss_name];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Managers handle intermediate updates here.
|
||||
event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
|
||||
event SumStats::cluster_key_intermediate_response(ss_name: string, key: Key)
|
||||
{
|
||||
#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
|
||||
#print fmt("MANAGER: requesting key data for %s", key2str(key));
|
||||
|
||||
if ( ssid in outstanding_global_views &&
|
||||
|outstanding_global_views[ssid]| > max_outstanding_global_views )
|
||||
if ( ss_name in outstanding_global_views &&
|
||||
|outstanding_global_views[ss_name]| > max_outstanding_global_views )
|
||||
{
|
||||
# Don't do this intermediate update. Perhaps at some point in the future
|
||||
# we will queue and randomly select from these ignored intermediate
|
||||
|
|
@ -282,13 +280,14 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
|
|||
return;
|
||||
}
|
||||
|
||||
++outstanding_global_views[ssid];
|
||||
++outstanding_global_views[ss_name];
|
||||
|
||||
local uid = unique_id("");
|
||||
event SumStats::cluster_key_request(uid, ssid, key);
|
||||
done_with[uid] = 0;
|
||||
event SumStats::cluster_key_request(uid, ss_name, key, T);
|
||||
}
|
||||
|
||||
event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
|
||||
event SumStats::cluster_ss_response(uid: string, ss_name: string, data: ResultTable, done: bool)
|
||||
{
|
||||
#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
|
||||
|
||||
|
|
@ -297,7 +296,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
|
|||
++done_with[uid];
|
||||
|
||||
local local_data = stats_results[uid];
|
||||
local ss = stats_store[ssid];
|
||||
local ss = stats_store[ss_name];
|
||||
|
||||
for ( key in data )
|
||||
{
|
||||
|
|
@ -314,13 +313,14 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
|
|||
if ( check_thresholds(ss, key, local_data[key], 1.0) )
|
||||
{
|
||||
threshold_crossed(ss, key, local_data[key]);
|
||||
event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
|
||||
event SumStats::cluster_threshold_crossed(ss$name, key, threshold_tracker[ss$name][key]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# If the data has been collected from all peers, we are done and ready to finish.
|
||||
if ( Cluster::worker_count == done_with[uid] )
|
||||
if ( Cluster::worker_count == done_with[uid] &&
|
||||
/^dyn-/ !in uid )
|
||||
{
|
||||
if ( ss?$epoch_finished )
|
||||
ss$epoch_finished(local_data);
|
||||
|
|
@ -328,14 +328,60 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
|
|||
# Clean up
|
||||
delete stats_results[uid];
|
||||
delete done_with[uid];
|
||||
# Not sure I need to reset the sumstat on the manager.
|
||||
reset(ss);
|
||||
}
|
||||
}
|
||||
|
||||
event remote_connection_handshake_done(p: event_peer) &priority=5
|
||||
function request(ss_name: string): ResultTable
|
||||
{
|
||||
send_id(p, "SumStats::stats_store");
|
||||
send_id(p, "SumStats::reducer_store");
|
||||
# This only needs to be implemented this way for cluster compatibility.
|
||||
local uid = unique_id("dyn-");
|
||||
stats_results[uid] = table();
|
||||
done_with[uid] = 0;
|
||||
event SumStats::cluster_ss_request(uid, ss_name, F);
|
||||
|
||||
return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
if ( uid in stats_results )
|
||||
{
|
||||
local ss_result = stats_results[uid];
|
||||
# Clean up
|
||||
delete stats_results[uid];
|
||||
delete done_with[uid];
|
||||
reset(stats_store[ss_name]);
|
||||
return ss_result;
|
||||
}
|
||||
else
|
||||
return table();
|
||||
}
|
||||
timeout 1.1min
|
||||
{
|
||||
Reporter::warning(fmt("Dynamic SumStat request for %s took longer than 1 minute and was automatically cancelled.", ss_name));
|
||||
return table();
|
||||
}
|
||||
}
|
||||
|
||||
function request_key(ss_name: string, key: Key): Result
|
||||
{
|
||||
local uid = unique_id("dyn-");
|
||||
done_with[uid] = 0;
|
||||
key_requests[uid] = table();
|
||||
|
||||
event SumStats::cluster_key_request(uid, ss_name, key, F);
|
||||
return when ( uid in done_with && Cluster::worker_count == done_with[uid] )
|
||||
{
|
||||
local result = key_requests[uid];
|
||||
# Clean up
|
||||
delete key_requests[uid];
|
||||
delete done_with[uid];
|
||||
|
||||
return result;
|
||||
}
|
||||
timeout 1.1min
|
||||
{
|
||||
Reporter::warning(fmt("Dynamic SumStat key request for %s (%s) took longer than 1 minute and was automatically cancelled.", ss_name, key));
|
||||
return table();
|
||||
}
|
||||
}
|
||||
|
||||
@endif
|
||||
|
|
|
|||
|
|
@ -87,6 +87,10 @@ export {
|
|||
## is no assurance provided as to where the callbacks
|
||||
## will be executed on clusters.
|
||||
type SumStat: record {
|
||||
## An arbitrary name for the sumstat so that it can
|
||||
## be referred to later.
|
||||
name: string;
|
||||
|
||||
## The interval at which this filter should be "broken"
|
||||
## and the '$epoch_finished' callback called. The
|
||||
## results are also reset at this time so any threshold
|
||||
|
|
@ -102,22 +106,22 @@ export {
|
|||
## :bro:see:`Result` structure which will be used
|
||||
## for thresholding.
|
||||
## This is required if a $threshold value is given.
|
||||
threshold_val: function(key: SumStats::Key, result: SumStats::Result): count &optional;
|
||||
threshold_val: function(key: SumStats::Key, result: SumStats::Result): double &optional;
|
||||
|
||||
## The threshold value for calling the
|
||||
## $threshold_crossed callback.
|
||||
threshold: count &optional;
|
||||
threshold: double &optional;
|
||||
|
||||
## A series of thresholds for calling the
|
||||
## $threshold_crossed callback.
|
||||
threshold_series: vector of count &optional;
|
||||
threshold_series: vector of double &optional;
|
||||
|
||||
## A callback that is called when a threshold is crossed.
|
||||
threshold_crossed: function(key: SumStats::Key, result: SumStats::Result) &optional;
|
||||
|
||||
## A callback with the full collection of Results for
|
||||
## this SumStat.
|
||||
epoch_finished: function(rt: SumStats::ResultTable) &optional;
|
||||
epoch_finished: function(rt: SumStats::ResultTable) &optional;
|
||||
};
|
||||
|
||||
## Create a summary statistic.
|
||||
|
|
@ -134,19 +138,37 @@ export {
|
|||
## obs: The data point to send into the stream.
|
||||
global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
|
||||
|
||||
## This record is primarily used for internal threshold tracking.
|
||||
type Thresholding: record {
|
||||
# Internal use only. Indicates if a simple threshold was already crossed.
|
||||
is_threshold_crossed: bool &default=F;
|
||||
## Dynamically request a sumstat. This function should be
|
||||
## used sparingly and not as a replacement for the callbacks
|
||||
## from the :bro:see:`SumStat` record. The function is only
|
||||
## available for use within "when" statements as an asynchronous
|
||||
## function.
|
||||
##
|
||||
## ss_name: SumState name.
|
||||
##
|
||||
## Returns: The result table for the requested sumstat.
|
||||
global request: function(ss_name: string): ResultTable;
|
||||
|
||||
# Internal use only. Current key for threshold series.
|
||||
threshold_series_index: count &default=0;
|
||||
};
|
||||
## Dynamically request a sumstat key. This function should be
|
||||
## used sparingly and not as a replacement for the callbacks
|
||||
## from the :bro:see:`SumStat` record. The function is only
|
||||
## available for use within "when" statements as an asynchronous
|
||||
## function.
|
||||
##
|
||||
## ss_name: SumStat name.
|
||||
##
|
||||
## key: The SumStat key being requested.
|
||||
##
|
||||
## Returns: The result for the requested sumstat key.
|
||||
global request_key: function(ss_name: string, key: Key): Result;
|
||||
|
||||
## This record is primarily used for internal threshold tracking.
|
||||
type Thresholding: record {};
|
||||
|
||||
## This event is generated when thresholds are reset for a SumStat.
|
||||
##
|
||||
## ssid: SumStats ID that thresholds were reset for.
|
||||
global thresholds_reset: event(ssid: string);
|
||||
## name: SumStats name that thresholds were reset for.
|
||||
global thresholds_reset: event(name: string);
|
||||
|
||||
## Helper function to represent a :bro:type:`SumStats::Key` value as
|
||||
## a simple string.
|
||||
|
|
@ -157,19 +179,43 @@ export {
|
|||
global key2str: function(key: SumStats::Key): string;
|
||||
}
|
||||
|
||||
# The function prototype for plugins to do calculations.
|
||||
type ObserveFunc: function(r: Reducer, val: double, data: Observation, rv: ResultVal);
|
||||
|
||||
redef record Reducer += {
|
||||
# Internal use only. Provides a reference back to the related SumStats by it's ID.
|
||||
sid: string &optional;
|
||||
# Internal use only. Provides a reference back to the related SumStats by its name.
|
||||
ssname: string &optional;
|
||||
|
||||
calc_funcs: vector of Calculation &optional;
|
||||
};
|
||||
|
||||
redef record Thresholding += {
|
||||
# Internal use only. Indicates if a simple threshold was already crossed.
|
||||
is_threshold_crossed: bool &default=F;
|
||||
|
||||
# Internal use only. Current key for threshold series.
|
||||
threshold_series_index: count &default=0;
|
||||
};
|
||||
|
||||
|
||||
# Internal use only. For tracking thresholds per sumstat and key.
|
||||
global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
|
||||
|
||||
redef record SumStat += {
|
||||
# Internal use only.
|
||||
ssname: string &optional;
|
||||
|
||||
# Internal use only (mostly for cluster coherency).
|
||||
id: string &optional;
|
||||
};
|
||||
|
||||
# Prototype the hook point for plugins to initialize any result values.
|
||||
global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
|
||||
|
||||
# Prototype the hook point for plugins to merge Results.
|
||||
global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
|
||||
|
||||
|
||||
# Store of sumstats indexed on the sumstat id.
|
||||
global stats_store: table[string] of SumStat = table();
|
||||
|
||||
|
|
@ -182,20 +228,20 @@ global result_store: table[string] of ResultTable = table();
|
|||
# Store of threshold information.
|
||||
global thresholds_store: table[string, Key] of bool = table();
|
||||
|
||||
# Store the calculations.
|
||||
global calc_store: table[Calculation] of ObserveFunc = table();
|
||||
|
||||
# Store the dependencies for Calculations.
|
||||
global calc_deps: table[Calculation] of vector of Calculation = table();
|
||||
|
||||
# Hook for registering observation calculation plugins.
|
||||
global register_observe_plugins: hook();
|
||||
|
||||
# This is called whenever key values are updated and the new val is given as the
|
||||
# `val` argument. It's only prototyped here because cluster and non-cluster have
|
||||
# separate implementations.
|
||||
global data_added: function(ss: SumStat, key: Key, result: Result);
|
||||
|
||||
# Prototype the hook point for plugins to do calculations.
|
||||
global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
|
||||
|
||||
# Prototype the hook point for plugins to initialize any result values.
|
||||
global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
|
||||
|
||||
# Prototype the hook point for plugins to merge Results.
|
||||
global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
|
||||
|
||||
# Event that is used to "finish" measurements and adapt the measurement
|
||||
# framework for clustered or non-clustered usage.
|
||||
global finish_epoch: event(ss: SumStat);
|
||||
|
|
@ -210,6 +256,24 @@ function key2str(key: Key): string
|
|||
return fmt("sumstats_key(%s)", out);
|
||||
}
|
||||
|
||||
function register_observe_plugin(calc: Calculation, func: ObserveFunc)
|
||||
{
|
||||
calc_store[calc] = func;
|
||||
}
|
||||
|
||||
function add_observe_plugin_dependency(calc: Calculation, depends_on: Calculation)
|
||||
{
|
||||
if ( calc !in calc_deps )
|
||||
calc_deps[calc] = vector();
|
||||
calc_deps[calc][|calc_deps[calc]|] = depends_on;
|
||||
}
|
||||
|
||||
event bro_init() &priority=100000
|
||||
{
|
||||
# Call all of the plugin registration hooks
|
||||
hook register_observe_plugins();
|
||||
}
|
||||
|
||||
function init_resultval(r: Reducer): ResultVal
|
||||
{
|
||||
local rv: ResultVal = [$begin=network_time(), $end=network_time()];
|
||||
|
|
@ -234,25 +298,17 @@ function compose_results(r1: Result, r2: Result): Result
|
|||
{
|
||||
local result: Result = table();
|
||||
|
||||
if ( |r1| > |r2| )
|
||||
for ( id in r1 )
|
||||
{
|
||||
for ( data_id in r1 )
|
||||
{
|
||||
if ( data_id in r2 )
|
||||
result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
|
||||
else
|
||||
result[data_id] = r1[data_id];
|
||||
}
|
||||
result[id] = r1[id];
|
||||
}
|
||||
else
|
||||
|
||||
for ( id in r2 )
|
||||
{
|
||||
for ( data_id in r2 )
|
||||
{
|
||||
if ( data_id in r1 )
|
||||
result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
|
||||
else
|
||||
result[data_id] = r2[data_id];
|
||||
}
|
||||
if ( id in r1 )
|
||||
result[id] = compose_resultvals(r1[id], r2[id]);
|
||||
else
|
||||
result[id] = r2[id];
|
||||
}
|
||||
|
||||
return result;
|
||||
|
|
@ -261,18 +317,42 @@ function compose_results(r1: Result, r2: Result): Result
|
|||
|
||||
function reset(ss: SumStat)
|
||||
{
|
||||
if ( ss$id in result_store )
|
||||
delete result_store[ss$id];
|
||||
if ( ss$name in result_store )
|
||||
delete result_store[ss$name];
|
||||
|
||||
result_store[ss$id] = table();
|
||||
result_store[ss$name] = table();
|
||||
|
||||
if ( ss?$threshold || ss?$threshold_series )
|
||||
{
|
||||
threshold_tracker[ss$id] = table();
|
||||
event SumStats::thresholds_reset(ss$id);
|
||||
threshold_tracker[ss$name] = table();
|
||||
event SumStats::thresholds_reset(ss$name);
|
||||
}
|
||||
}
|
||||
|
||||
# This could potentially recurse forever, but plugin authors
|
||||
# should be making sure they aren't causing reflexive dependencies.
|
||||
function add_calc_deps(calcs: vector of Calculation, c: Calculation)
|
||||
{
|
||||
#print fmt("Checking for deps for %s", c);
|
||||
for ( i in calc_deps[c] )
|
||||
{
|
||||
local skip_calc=F;
|
||||
for ( j in calcs )
|
||||
{
|
||||
if ( calcs[j] == calc_deps[c][i] )
|
||||
skip_calc=T;
|
||||
}
|
||||
if ( ! skip_calc )
|
||||
{
|
||||
if ( calc_deps[c][i] in calc_deps )
|
||||
add_calc_deps(calcs, calc_deps[c][i]);
|
||||
calcs[|c|] = calc_deps[c][i];
|
||||
#print fmt("add dep for %s [%s] ", c, calc_deps[c][i]);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
function create(ss: SumStat)
|
||||
{
|
||||
if ( (ss?$threshold || ss?$threshold_series) && ! ss?$threshold_val )
|
||||
|
|
@ -280,14 +360,32 @@ function create(ss: SumStat)
|
|||
Reporter::error("SumStats given a threshold with no $threshold_val function");
|
||||
}
|
||||
|
||||
if ( ! ss?$id )
|
||||
ss$id=unique_id("");
|
||||
threshold_tracker[ss$id] = table();
|
||||
stats_store[ss$id] = ss;
|
||||
threshold_tracker[ss$name] = table();
|
||||
stats_store[ss$name] = ss;
|
||||
|
||||
for ( reducer in ss$reducers )
|
||||
{
|
||||
reducer$sid = ss$id;
|
||||
reducer$ssname = ss$name;
|
||||
reducer$calc_funcs = vector();
|
||||
for ( calc in reducer$apply )
|
||||
{
|
||||
# Add in dependencies recursively.
|
||||
if ( calc in calc_deps )
|
||||
add_calc_deps(reducer$calc_funcs, calc);
|
||||
|
||||
# Don't add this calculation to the vector if
|
||||
# it was already added by something else as a
|
||||
# dependency.
|
||||
local skip_calc=F;
|
||||
for ( j in reducer$calc_funcs )
|
||||
{
|
||||
if ( calc == reducer$calc_funcs[j] )
|
||||
skip_calc=T;
|
||||
}
|
||||
if ( ! skip_calc )
|
||||
reducer$calc_funcs[|reducer$calc_funcs|] = calc;
|
||||
}
|
||||
|
||||
if ( reducer$stream !in reducer_store )
|
||||
reducer_store[reducer$stream] = set();
|
||||
add reducer_store[reducer$stream][reducer];
|
||||
|
|
@ -313,7 +411,7 @@ function observe(id: string, key: Key, obs: Observation)
|
|||
if ( r?$pred && ! r$pred(key, obs) )
|
||||
next;
|
||||
|
||||
local ss = stats_store[r$sid];
|
||||
local ss = stats_store[r$ssname];
|
||||
|
||||
# If there is a threshold and no epoch_finished callback
|
||||
# we don't need to continue counting since the data will
|
||||
|
|
@ -324,17 +422,21 @@ function observe(id: string, key: Key, obs: Observation)
|
|||
# future if on demand access is provided to the
|
||||
# SumStats results.
|
||||
if ( ! ss?$epoch_finished &&
|
||||
r$sid in threshold_tracker &&
|
||||
key in threshold_tracker[r$sid] &&
|
||||
( ss?$threshold &&
|
||||
threshold_tracker[r$sid][key]$is_threshold_crossed ) ||
|
||||
r$ssname in threshold_tracker &&
|
||||
key in threshold_tracker[r$ssname] &&
|
||||
threshold_tracker[r$ssname][key]$is_threshold_crossed ) ||
|
||||
( ss?$threshold_series &&
|
||||
threshold_tracker[r$sid][key]$threshold_series_index+1 == |ss$threshold_series| ) )
|
||||
r$ssname in threshold_tracker &&
|
||||
key in threshold_tracker[r$ssname] &&
|
||||
threshold_tracker[r$ssname][key]$threshold_series_index == |ss$threshold_series| ) )
|
||||
{
|
||||
next;
|
||||
}
|
||||
|
||||
if ( r$sid !in result_store )
|
||||
result_store[ss$id] = table();
|
||||
local results = result_store[r$sid];
|
||||
if ( r$ssname !in result_store )
|
||||
result_store[r$ssname] = table();
|
||||
local results = result_store[r$ssname];
|
||||
|
||||
if ( key !in results )
|
||||
results[key] = table();
|
||||
|
|
@ -350,10 +452,13 @@ function observe(id: string, key: Key, obs: Observation)
|
|||
|
||||
# If a string was given, fall back to 1.0 as the value.
|
||||
local val = 1.0;
|
||||
if ( obs?$num || obs?$dbl )
|
||||
val = obs?$dbl ? obs$dbl : obs$num;
|
||||
if ( obs?$num )
|
||||
val = obs$num;
|
||||
else if ( obs?$dbl )
|
||||
val = obs$dbl;
|
||||
|
||||
hook observe_hook(r, val, obs, result_val);
|
||||
for ( i in r$calc_funcs )
|
||||
calc_store[r$calc_funcs[i]](r, val, obs, result_val);
|
||||
data_added(ss, key, result);
|
||||
}
|
||||
}
|
||||
|
|
@ -366,6 +471,8 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
|
|||
return F;
|
||||
|
||||
# Add in the extra ResultVals to make threshold_vals easier to write.
|
||||
# This length comparison should work because we just need to make
|
||||
# sure that we have the same number of reducers and results.
|
||||
if ( |ss$reducers| != |result| )
|
||||
{
|
||||
for ( reducer in ss$reducers )
|
||||
|
|
@ -378,11 +485,11 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
|
|||
local watch = ss$threshold_val(key, result);
|
||||
|
||||
if ( modify_pct < 1.0 && modify_pct > 0.0 )
|
||||
watch = double_to_count(floor(watch/modify_pct));
|
||||
watch = watch/modify_pct;
|
||||
|
||||
if ( ss$id !in threshold_tracker )
|
||||
threshold_tracker[ss$id] = table();
|
||||
local t_tracker = threshold_tracker[ss$id];
|
||||
if ( ss$name !in threshold_tracker )
|
||||
threshold_tracker[ss$name] = table();
|
||||
local t_tracker = threshold_tracker[ss$name];
|
||||
|
||||
if ( key !in t_tracker )
|
||||
{
|
||||
|
|
@ -398,7 +505,7 @@ function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: dou
|
|||
}
|
||||
|
||||
if ( ss?$threshold_series &&
|
||||
|ss$threshold_series| >= tt$threshold_series_index &&
|
||||
|ss$threshold_series| > tt$threshold_series_index &&
|
||||
watch >= ss$threshold_series[tt$threshold_series_index] )
|
||||
{
|
||||
# A threshold series was given and the value crossed the next
|
||||
|
|
@ -426,7 +533,7 @@ function threshold_crossed(ss: SumStat, key: Key, result: Result)
|
|||
}
|
||||
|
||||
ss$threshold_crossed(key, result);
|
||||
local tt = threshold_tracker[ss$id][key];
|
||||
local tt = threshold_tracker[ss$name][key];
|
||||
tt$is_threshold_crossed = T;
|
||||
|
||||
# Bump up to the next threshold series index if a threshold series is being used.
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ module SumStats;
|
|||
|
||||
event SumStats::finish_epoch(ss: SumStat)
|
||||
{
|
||||
if ( ss$id in result_store )
|
||||
if ( ss$name in result_store )
|
||||
{
|
||||
local data = result_store[ss$id];
|
||||
local data = result_store[ss$name];
|
||||
if ( ss?$epoch_finished )
|
||||
ss$epoch_finished(data);
|
||||
|
||||
|
|
@ -16,9 +16,32 @@ event SumStats::finish_epoch(ss: SumStat)
|
|||
schedule ss$epoch { SumStats::finish_epoch(ss) };
|
||||
}
|
||||
|
||||
|
||||
function data_added(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
if ( check_thresholds(ss, key, result, 1.0) )
|
||||
threshold_crossed(ss, key, result);
|
||||
}
|
||||
|
||||
function request(ss_name: string): ResultTable
|
||||
{
|
||||
# This only needs to be implemented this way for cluster compatibility.
|
||||
return when ( T )
|
||||
{
|
||||
if ( ss_name in result_store )
|
||||
return result_store[ss_name];
|
||||
else
|
||||
return table();
|
||||
}
|
||||
}
|
||||
|
||||
function request_key(ss_name: string, key: Key): Result
|
||||
{
|
||||
# This only needs to be implemented this way for cluster compatibility.
|
||||
return when ( T )
|
||||
{
|
||||
if ( ss_name in result_store && key in result_store[ss_name] )
|
||||
return result_store[ss_name][key];
|
||||
else
|
||||
return table();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -14,17 +14,18 @@ export {
|
|||
};
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
hook register_observe_plugins()
|
||||
{
|
||||
if ( AVERAGE in r$apply )
|
||||
register_observe_plugin(AVERAGE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( ! rv?$average )
|
||||
rv$average = val;
|
||||
else
|
||||
rv$average += (val - rv$average) / rv$num;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
if ( rv1?$average && rv2?$average )
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -14,15 +14,15 @@ export {
|
|||
};
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
hook register_observe_plugins()
|
||||
{
|
||||
if ( MAX in r$apply )
|
||||
register_observe_plugin(MAX, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( ! rv?$max )
|
||||
rv$max = val;
|
||||
else if ( val > rv$max )
|
||||
rv$max = val;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -14,17 +14,18 @@ export {
|
|||
};
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
hook register_observe_plugins()
|
||||
{
|
||||
if ( MIN in r$apply )
|
||||
register_observe_plugin(MIN, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( ! rv?$min )
|
||||
rv$min = val;
|
||||
else if ( val < rv$min )
|
||||
rv$min = val;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
{
|
||||
if ( rv1?$min && rv2?$min )
|
||||
|
|
|
|||
|
|
@ -1,9 +1,14 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load base/utils/queue
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
export {
|
||||
redef enum Calculation += {
|
||||
## Collect a sample of the last few observations.
|
||||
SAMPLE
|
||||
};
|
||||
|
||||
redef record Reducer += {
|
||||
## A number of sample Observations to collect.
|
||||
samples: count &default=0;
|
||||
|
|
@ -27,14 +32,14 @@ function get_samples(rv: ResultVal): vector of Observation
|
|||
return s;
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
hook register_observe_plugins()
|
||||
{
|
||||
if ( r$samples > 0 )
|
||||
register_observe_plugin(SAMPLE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( ! rv?$samples )
|
||||
rv$samples = Queue::init([$max_len=r$samples]);
|
||||
Queue::put(rv$samples, obs);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
@load ./variance
|
||||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -21,11 +21,18 @@ function calc_std_dev(rv: ResultVal)
|
|||
rv$std_dev = sqrt(rv$variance);
|
||||
}
|
||||
|
||||
# This depends on the variance plugin which uses priority -5
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
|
||||
hook std_dev_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( STD_DEV in r$apply )
|
||||
calc_std_dev(rv);
|
||||
}
|
||||
|
||||
hook register_observe_plugins() &priority=-10
|
||||
{
|
||||
register_observe_plugin(STD_DEV, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
calc_std_dev(rv);
|
||||
});
|
||||
add_observe_plugin_dependency(STD_DEV, VARIANCE);
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -14,19 +14,19 @@ export {
|
|||
sum: double &default=0.0;
|
||||
};
|
||||
|
||||
type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
|
||||
global sum_threshold: function(data_id: string): threshold_function;
|
||||
#type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
|
||||
#global sum_threshold: function(data_id: string): threshold_function;
|
||||
}
|
||||
|
||||
function sum_threshold(data_id: string): threshold_function
|
||||
{
|
||||
return function(key: SumStats::Key, result: SumStats::Result): count
|
||||
{
|
||||
print fmt("data_id: %s", data_id);
|
||||
print result;
|
||||
return double_to_count(result[data_id]$sum);
|
||||
};
|
||||
}
|
||||
#function sum_threshold(data_id: string): threshold_function
|
||||
# {
|
||||
# return function(key: SumStats::Key, result: SumStats::Result): count
|
||||
# {
|
||||
# print fmt("data_id: %s", data_id);
|
||||
# print result;
|
||||
# return double_to_count(result[data_id]$sum);
|
||||
# };
|
||||
# }
|
||||
|
||||
hook init_resultval_hook(r: Reducer, rv: ResultVal)
|
||||
{
|
||||
|
|
@ -34,10 +34,12 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
|
|||
rv$sum = 0;
|
||||
}
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
hook register_observe_plugins()
|
||||
{
|
||||
if ( SUM in r$apply )
|
||||
register_observe_plugin(SUM, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
rv$sum += val;
|
||||
});
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -23,15 +23,15 @@ redef record ResultVal += {
|
|||
unique_vals: set[Observation] &optional;
|
||||
};
|
||||
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
hook register_observe_plugins()
|
||||
{
|
||||
if ( UNIQUE in r$apply )
|
||||
register_observe_plugin(UNIQUE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( ! rv?$unique_vals )
|
||||
rv$unique_vals=set();
|
||||
add rv$unique_vals[obs];
|
||||
rv$unique = |rv$unique_vals|;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
@load ./average
|
||||
@load base/frameworks/sumstats
|
||||
@load ../main
|
||||
|
||||
module SumStats;
|
||||
|
||||
|
|
@ -28,17 +28,17 @@ function calc_variance(rv: ResultVal)
|
|||
rv$variance = (rv$num > 1) ? rv$var_s/(rv$num-1) : 0.0;
|
||||
}
|
||||
|
||||
# Reduced priority since this depends on the average
|
||||
hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
|
||||
hook register_observe_plugins() &priority=-5
|
||||
{
|
||||
if ( VARIANCE in r$apply )
|
||||
register_observe_plugin(VARIANCE, function(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( rv$num > 1 )
|
||||
rv$var_s += ((val - rv$prev_avg) * (val - rv$average));
|
||||
|
||||
calc_variance(rv);
|
||||
rv$prev_avg = rv$average;
|
||||
}
|
||||
});
|
||||
add_observe_plugin_dependency(VARIANCE, AVERAGE);
|
||||
}
|
||||
|
||||
# Reduced priority since this depends on the average
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue