Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-06 08:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Large update for the SumStats framework.

Browse source 
- On-demand access to sumstats results through "return from"
   functions named SumStats::request and Sumstats::request_key.
   Both functions are tested in standalone and clustered modes.

 - $name field has returned to SumStats which simplifies cluster
   code and makes the on-demand access stuff possible.

 - Clustered results can only be collected for 1 minute from their
   time of creation now instead of time of last read.

 - Thresholds use doubles instead of counts everywhere now.

 - Calculation dependency resolution occurs at start up time now
   instead of doing it at observation time which provide a minor
   cpu performance improvement.  A new plugin registration mechanism
   was created to support this change.

 - AppStats now has a minimal doc string and is broken into hook-based
   plugins.

 - AppStats and traceroute detection added to local.bro
This commit is contained in:
Seth Hall 2013-05-21 15:52:59 -04:00
parent 7d7d30e1f7
commit bec965b66f
34 changed files with 687 additions and 277 deletions
Download patch file Download diff file Expand all files Collapse all files

3
testing/btest/scripts/base/frameworks/sumstats/basic-cluster.bro
View file

@ -23,7 +23,8 @@ global n = 0;
event bro_init() &priority=5
{
local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
SumStats::create([$epoch=5secs,
SumStats::create([$name="test",
$epoch=5secs,
$reducers=set(r1),
$epoch_finished(rt: SumStats::ResultTable) =
{

21
testing/btest/scripts/base/frameworks/sumstats/basic.bro
View file

@ -11,16 +11,17 @@ event bro_init() &priority=5
SumStats::MIN,
SumStats::STD_DEV,
SumStats::UNIQUE)];
SumStats::create([$epoch=3secs,
$reducers=set(r1),
$epoch_finished(data: SumStats::ResultTable) =
{
for ( key in data )
{
local r = data[key]["test.metric"];
print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
}
}
SumStats::create([$name="test",
$epoch=3secs,
$reducers=set(r1),
$epoch_finished(data: SumStats::ResultTable) =
{
for ( key in data )
{
local r = data[key]["test.metric"];
print fmt("Host: %s - num:%d - sum:%.1f - var:%.1f - avg:%.1f - max:%.1f - min:%.1f - std_dev:%.1f - unique:%d", key$host, r$num, r$sum, r$variance, r$average, r$max, r$min, r$std_dev, r$unique);
}
}
]);
SumStats::observe("test.metric", [$host=1.2.3.4], [$num=5]);

7
testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro
View file

@ -20,13 +20,14 @@ redef Log::default_rotation_interval = 0secs;
event bro_init() &priority=5
{
local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
SumStats::create([$epoch=1hr,
SumStats::create([$name="test",
$epoch=1hr,
$reducers=set(r1),
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
return double_to_count(result["test.metric"]$sum);
return result["test.metric"]$sum;
},
$threshold=100,
$threshold=100.0,
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum);

93
testing/btest/scripts/base/frameworks/sumstats/on-demand-cluster.bro Normal file
View file

@ -0,0 +1,93 @@
# @TEST-SERIALIZE: comm
#
# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
# @TEST-EXEC: sleep 1
# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
# @TEST-EXEC: btest-bg-wait 15
# @TEST-EXEC: btest-diff manager-1/.stdout
@TEST-START-FILE cluster-layout.bro
redef Cluster::nodes = {
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"],
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"],
};
@TEST-END-FILE
redef Log::default_rotation_interval = 0secs;
global n = 0;
event bro_init() &priority=5
{
local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SUM, SumStats::MIN, SumStats::MAX, SumStats::AVERAGE, SumStats::STD_DEV, SumStats::VARIANCE, SumStats::UNIQUE)];
SumStats::create([$name="test sumstat",
$epoch=1hr,
$reducers=set(r1)]);
}
event remote_connection_closed(p: event_peer)
{
terminate();
}
global ready_for_data: event();
redef Cluster::manager2worker_events += /^ready_for_data$/;
event ready_for_data()
{
if ( Cluster::node == "worker-1" )
{
SumStats::observe("test", [$host=1.2.3.4], [$num=34]);
SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
SumStats::observe("test", [$host=6.5.4.3], [$num=1]);
SumStats::observe("test", [$host=7.2.1.5], [$num=54]);
}
if ( Cluster::node == "worker-2" )
{
SumStats::observe("test", [$host=1.2.3.4], [$num=75]);
SumStats::observe("test", [$host=1.2.3.4], [$num=30]);
SumStats::observe("test", [$host=7.2.1.5], [$num=91]);
SumStats::observe("test", [$host=10.10.10.10], [$num=5]);
}
}
event on_demand2()
{
local host = 7.2.1.5;
when ( local result = SumStats::request_key("test sumstat", [$host=host]) )
{
print "SumStat key request";
print fmt(" Host: %s -> %.0f", host, result["test"]$sum);
terminate();
}
}
event on_demand()
{
when ( local results = SumStats::request("test sumstat") )
{
print "Complete SumStat request";
for ( key in results )
print fmt(" Host: %s -> %.0f", key$host, results[key]["test"]$sum);
event on_demand2();
}
}
global peer_count = 0;
event remote_connection_handshake_done(p: event_peer) &priority=-5
{
++peer_count;
if ( peer_count == 2 )
{
if ( Cluster::local_node_type() == Cluster::MANAGER )
event ready_for_data();
schedule 1sec { on_demand() };
}
}

45
testing/btest/scripts/base/frameworks/sumstats/on-demand.bro Normal file
View file

@ -0,0 +1,45 @@
# @TEST-EXEC: bro %INPUT
# @TEST-EXEC: btest-diff .stdout
redef exit_only_after_terminate=T;
event on_demand()
{
when ( local results = SumStats::request("test") )
{
print "Complete SumStat request";
for ( key in results )
{
print fmt(" Host: %s -> %.0f", key$host, results[key]["test.reducer"]$sum);
}
}
}
event on_demand_key()
{
local host = 1.2.3.4;
when ( local result = SumStats::request_key("test", [$host=host]) )
{
print fmt("Key request for %s", host);
print fmt(" Host: %s -> %.0f", host, result["test.reducer"]$sum);
terminate();
}
}
event bro_init() &priority=5
{
local r1: SumStats::Reducer = [$stream="test.reducer",
$apply=set(SumStats::SUM)];
SumStats::create([$name="test",
$epoch=1hr,
$reducers=set(r1)]);
# Seed some data but notice there are no callbacks defined in the sumstat!
SumStats::observe("test.reducer", [$host=1.2.3.4], [$num=42]);
SumStats::observe("test.reducer", [$host=4.3.2.1], [$num=7]);
schedule 0.1 secs { on_demand() };
schedule 1 secs { on_demand_key() };
}

23
testing/btest/scripts/base/frameworks/sumstats/thresholding.bro
View file

@ -8,14 +8,15 @@ redef enum Notice::Type += {
event bro_init() &priority=5
{
local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
SumStats::create([$epoch=3secs,
SumStats::create([$name="test1",
$epoch=3secs,
$reducers=set(r1),
#$threshold_val = SumStats::sum_threshold("test.metric"),
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
return double_to_count(result["test.metric"]$sum);
return result["test.metric"]$sum;
},
$threshold=5,
$threshold=5.0,
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
local r = result["test.metric"];
@ -24,14 +25,15 @@ event bro_init() &priority=5
]);
local r2: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
SumStats::create([$epoch=3secs,
SumStats::create([$name="test2",
$epoch=3secs,
$reducers=set(r2),
#$threshold_val = SumStats::sum_threshold("test.metric"),
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
return double_to_count(result["test.metric"]$sum);
return result["test.metric"]$sum;
},
$threshold_series=vector(3,6,800),
$threshold_series=vector(3.0,6.0,800.0),
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
local r = result["test.metric"];
@ -41,19 +43,20 @@ event bro_init() &priority=5
local r3: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)];
local r4: SumStats::Reducer = [$stream="test.metric2", $apply=set(SumStats::SUM)];
SumStats::create([$epoch=3secs,
SumStats::create([$name="test3",
$epoch=3secs,
$reducers=set(r3, r4),
$threshold_val(key: SumStats::Key, result: SumStats::Result) =
{
# Calculate a ratio between sums of two reducers.
if ( "test.metric2" in result && "test.metric" in result &&
result["test.metric"]$sum > 0 )
return double_to_count(result["test.metric2"]$sum / result["test.metric"]$sum);
return result["test.metric2"]$sum / result["test.metric"]$sum;
else
return 0;
return 0.0;
},
# Looking for metric2 sum to be 5 times the sum of metric
$threshold=5,
$threshold=5.0,
$threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
{
local thold = result["test.metric2"]$sum / result["test.metric"]$sum;