Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-16 05:28:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

SumStats tests pass.

Browse source
This commit is contained in:
Seth Hall 2013-04-15 15:28:11 -04:00
parent fbe967e16a
commit 437815454d
6 changed files with 66 additions and 66 deletions
Download patch file Download diff file Expand all files Collapse all files

56
scripts/base/frameworks/sumstats/cluster.bro
View file

@ -35,34 +35,34 @@ export {
## Event sent by the manager in a cluster to initiate the
## collection of metrics values for a measurement.
global cluster_measurement_request: event(uid: string, mid: string);
global cluster_ss_request: event(uid: string, ssid: string);
## Event sent by nodes that are collecting metrics after receiving
## a request for the metric measurement from the manager.
global cluster_measurement_response: event(uid: string, mid: string, data: ResultTable, done: bool);
global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
## This event is sent by the manager in a cluster to initiate the
## collection of a single key value from a measurement. It's typically
## used to get intermediate updates before the break interval triggers
## to speed detection of a value crossing a threshold.
global cluster_key_request: event(uid: string, mid: string, key: Key);
global cluster_key_request: event(uid: string, ssid: string, key: Key);
## This event is sent by nodes in response to a
## :bro:id:`SumStats::cluster_key_request` event.
global cluster_key_response: event(uid: string, mid: string, key: Key, result: Result);
global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
## This is sent by workers to indicate that they crossed the percent of the
## current threshold by the percentage defined globally in
## :bro:id:`SumStats::cluster_request_global_view_percent`
global cluster_key_intermediate_response: event(mid: string, key: SumStats::Key);
global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
## This event is scheduled internally on workers to send result chunks.
global send_data: event(uid: string, mid: string, data: ResultTable);
global send_data: event(uid: string, ssid: string, data: ResultTable);
}
# Add events to the cluster framework to make this work.
redef Cluster::manager2worker_events += /SumStats::cluster_(measurement_request|key_request)/;
redef Cluster::worker2manager_events += /SumStats::cluster_(measurement_response|key_response|key_intermediate_response)/;
redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request)/;
redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
@if ( Cluster::local_node_type() != Cluster::MANAGER )
# This variable is maintained to know what keys have recently sent as
@ -99,7 +99,7 @@ function data_added(ss: SumStat, key: Key, result: Result)
}
}
event SumStats::send_data(uid: string, mid: string, data: ResultTable)
event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
{
#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
@ -121,39 +121,39 @@ event SumStats::send_data(uid: string, mid: string, data: ResultTable)
if ( |data| == 0 )
done = T;
event SumStats::cluster_measurement_response(uid, mid, local_data, done);
event SumStats::cluster_ss_response(uid, ssid, local_data, done);
if ( ! done )
schedule 0.01 sec { SumStats::send_data(uid, mid, data) };
schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
}
event SumStats::cluster_measurement_request(uid: string, mid: string)
event SumStats::cluster_ss_request(uid: string, ssid: string)
{
#print fmt("WORKER %s: received the cluster_measurement_request event for %s.", Cluster::node, id);
#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
# Initiate sending all of the data for the requested measurement.
if ( mid in result_store )
event SumStats::send_data(uid, mid, result_store[mid]);
if ( ssid in result_store )
event SumStats::send_data(uid, ssid, result_store[ssid]);
else
event SumStats::send_data(uid, mid, table());
event SumStats::send_data(uid, ssid, table());
# Lookup the actual measurement and reset it, the reference to the data
# currently stored will be maintained internally by the send_data event.
if ( mid in stats_store )
reset(stats_store[mid]);
if ( ssid in stats_store )
reset(stats_store[ssid]);
}
event SumStats::cluster_key_request(uid: string, mid: string, key: Key)
event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
{
if ( mid in result_store && key in result_store[mid] )
if ( ssid in result_store && key in result_store[ssid] )
{
#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
event SumStats::cluster_key_response(uid, mid, key, result_store[mid][key]);
event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]);
}
else
{
# We need to send an empty response if we don't have the data so that the manager
# can know that it heard back from all of the workers.
event SumStats::cluster_key_response(uid, mid, key, table());
event SumStats::cluster_key_response(uid, ssid, key, table());
}
}
@ -195,16 +195,16 @@ event SumStats::finish_epoch(ss: SumStat)
#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
local uid = unique_id("");
if ( uid in measurement_results )
delete measurement_results[uid];
if ( uid in stats_results )
delete stats_results[uid];
stats_results[uid] = table();
# Request data from peers.
event SumStats::cluster_measurement_request(uid, ss$id);
event SumStats::cluster_ss_request(uid, ss$id);
}
# Schedule the next finish_epoch event.
schedule m$epoch { SumStats::finish_epoch(m) };
schedule ss$epoch { SumStats::finish_epoch(ss) };
}
# This is unlikely to be called often, but it's here in case there are measurements
@ -252,7 +252,7 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
#print fmt("MANAGER: requesting key data for %s", key2str(key));
if ( ssid in outstanding_global_views &&
|outstanding_global_views[mid]| > max_outstanding_global_views )
|outstanding_global_views[ssid]| > max_outstanding_global_views )
{
# Don't do this intermediate update. Perhaps at some point in the future
# we will queue and randomly select from these ignored intermediate
@ -266,7 +266,7 @@ event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
event SumStats::cluster_key_request(uid, ssid, key);
}
event SumStats::cluster_measurement_response(uid: string, ssid: string, data: ResultTable, done: bool)
event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
{
#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);

10
scripts/base/frameworks/sumstats/main.bro
View file

@ -120,7 +120,7 @@ export {
};
## Create a summary statistic.
global create: function(m: SumStats::SumStat);
global create: function(ss: SumStats::SumStat);
## Add data into an observation stream. This should be
## called when a script has measured some point value.
@ -158,13 +158,13 @@ type Thresholding: record {
# Internal use only. For tracking thresholds per sumstat and key.
global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
redef record SumStats += {
redef record SumStat += {
# Internal use only (mostly for cluster coherency).
id: string &optional;
};
# Store of sumstats indexed on the sumstat id.
global stats_store: table[string] of SumStats = table();
global stats_store: table[string] of SumStat = table();
# Store of reducers indexed on the data point stream id.
global reducer_store: table[string] of set[Reducer] = table();
@ -179,7 +179,7 @@ global thresholds_store: table[string, Key] of bool = table();
# key values are updated and the new val is given as the `val` argument.
# It's only prototyped here because cluster and non-cluster have separate
# implementations.
global data_added: function(m: SumStats, key: Key, result: Result);
global data_added: function(ss: SumStat, key: Key, result: Result);
# Prototype the hook point for plugins to do calculations.
global add_to_reducer_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
@ -190,7 +190,7 @@ global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: Res
# Event that is used to "finish" measurements and adapt the measurement
# framework for clustered or non-clustered usage.
global finish_epoch: event(m: SumStats);
global finish_epoch: event(ss: SumStat);
function key2str(key: Key): string
{

20
scripts/base/frameworks/sumstats/non-cluster.bro
View file

@ -2,23 +2,23 @@
module SumStats;
event SumStats::finish_epoch(m: SumStats)
event SumStats::finish_epoch(ss: SumStat)
{
if ( m$id in result_store )
if ( ss$id in result_store )
{
local data = result_store[m$id];
if ( m?$epoch_finished )
m$epoch_finished(data);
local data = result_store[ss$id];
if ( ss?$epoch_finished )
ss$epoch_finished(data);
reset(m);
reset(ss);
}
schedule m$epoch { SumStats::finish_epoch(m) };
schedule ss$epoch { SumStats::finish_epoch(ss) };
}
function data_added(m: SumStats, key: Key, result: Result)
function data_added(ss: SumStat, key: Key, result: Result)
{
if ( check_thresholds(m, key, result, 1.0) )
threshold_crossed(m, key, result);
if ( check_thresholds(ss, key, result, 1.0) )
threshold_crossed(ss, key, result);
}

2
scripts/base/init-default.bro
View file

@ -29,9 +29,9 @@
@load base/frameworks/communication
@load base/frameworks/control
@load base/frameworks/cluster
@load base/frameworks/measurement
@load base/frameworks/intel
@load base/frameworks/reporter
@load base/frameworks/sumstats
@load base/frameworks/tunnels
@load base/protocols/conn

12
testing/btest/Baseline/scripts.base.frameworks.sumstats.thresholding/.stdout
View file

@ -1,6 +1,6 @@
THRESHOLD_SERIES: hit a threshold series value at 3 for measurement_key(host=1.2.3.4)
THRESHOLD_SERIES: hit a threshold series value at 6 for measurement_key(host=1.2.3.4)
THRESHOLD: hit a threshold value at 6 for measurement_key(host=1.2.3.4)
THRESHOLD_SERIES: hit a threshold series value at 1001 for measurement_key(host=7.2.1.5)
THRESHOLD: hit a threshold value at 1001 for measurement_key(host=7.2.1.5)
THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at 55x for measurement_key(host=7.2.1.5)
THRESHOLD_SERIES: hit a threshold series value at 3 for sumstats_key(host=1.2.3.4)
THRESHOLD_SERIES: hit a threshold series value at 6 for sumstats_key(host=1.2.3.4)
THRESHOLD: hit a threshold value at 6 for sumstats_key(host=1.2.3.4)
THRESHOLD_SERIES: hit a threshold series value at 1001 for sumstats_key(host=7.2.1.5)
THRESHOLD: hit a threshold value at 1001 for sumstats_key(host=7.2.1.5)
THRESHOLD WITH RATIO BETWEEN REDUCERS: hit a threshold value at 55x for sumstats_key(host=7.2.1.5)

32
testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
View file

@ -22,14 +22,14 @@ global n = 0;
event bro_init() &priority=5
{
local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
SumStats::create([$epoch=5secs,
$reducers=set(r1),
$epoch_finished(rt: SumStats::ResultTable) =
{
for ( key in rt )
{
local r = rt[key]["test.metric"];
local r = rt[key]["test"];
print fmt("Host: %s - num:%d - sum:%.1f - avg:%.1f - max:%.1f - min:%.1f - var:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$average, r$max, r$min, r$variance, r$std_dev, r$unique);
}
@ -49,23 +49,23 @@ event ready_for_data()
{
if ( Cluster::node == "worker-1" )
{
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=34]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=30]);
SumStats::observe("test.metric", [$host=6.5.4.3], [$num=1]);
SumStats::observe("test.metric", [$host=7.2.1.5], [$num=54]);
SumStats::observe("test", [$host=1.2.3.4], [$num=34]);
SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
SumStats::observe("test", [$host=6.5.4.3], [$num=1]);
SumStats::observe("test", [$host=7.2.1.5], [$num=54]);
}
if ( Cluster::node == "worker-2" )
{
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=75]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=30]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=3]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=57]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=52]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=61]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=95]);
SumStats::observe("test.metric", [$host=6.5.4.3], [$num=5]);
SumStats::observe("test.metric", [$host=7.2.1.5], [$num=91]);
SumStats::observe("test.metric", [$host=10.10.10.10], [$num=5]);
SumStats::observe("test", [$host=1.2.3.4], [$num=75]);
SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
SumStats::observe("test", [$host=1.2.3.4], [$num=3]);
SumStats::observe("test", [$host=1.2.3.4], [$num=57]);
SumStats::observe("test", [$host=1.2.3.4], [$num=52]);
SumStats::observe("test", [$host=1.2.3.4], [$num=61]);
SumStats::observe("test", [$host=1.2.3.4], [$num=95]);
SumStats::observe("test", [$host=6.5.4.3], [$num=5]);
SumStats::observe("test", [$host=7.2.1.5], [$num=91]);
SumStats::observe("test", [$host=10.10.10.10], [$num=5]);
}
}