Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-04 15:48:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Cluster support for the metrics framework returns and all tests work again.

Browse source
This commit is contained in:
Seth Hall 2012-03-28 14:00:59 -04:00
parent 6600e62ea3
commit 47f58e6340
10 changed files with 76 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

31
scripts/base/frameworks/metrics/cluster.bro
View file

@ -41,7 +41,7 @@ export {
## This event is sent by nodes in response to a ## This event is sent by nodes in response to a
## :bro:id:`Metrics::cluster_index_request` event. ## :bro:id:`Metrics::cluster_index_request` event.
global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, val: count); global cluster_index_response: event(uid: string, id: string, filter_name: string, index: Index, data: DataPoint);
## This is sent by workers to indicate that they crossed the percent of the ## This is sent by workers to indicate that they crossed the percent of the
## current threshold by the percentage defined globally in ## current threshold by the percentage defined globally in
@ -76,7 +76,7 @@ global done_with: table[string] of count &create_expire=5mins &default=0;
# This variable is maintained by managers to track intermediate responses as # This variable is maintained by managers to track intermediate responses as
# they are getting a global view for a certain index. # they are getting a global view for a certain index.
global index_requests: table[string, string, string, Index] of count &create_expire=5mins &default=0; global index_requests: table[string, string, string, Index] of DataPoint &create_expire=5mins &default=[];
# This variable is maintained by all hosts for different purposes. Non-managers # This variable is maintained by all hosts for different purposes. Non-managers
# maintain it to know what indexes they have recently sent as intermediate # maintain it to know what indexes they have recently sent as intermediate
@ -157,12 +157,12 @@ event Metrics::cluster_filter_request(uid: string, id: string, filter_name: stri
event Metrics::cluster_index_request(uid: string, id: string, filter_name: string, index: Index) event Metrics::cluster_index_request(uid: string, id: string, filter_name: string, index: Index)
{ {
local val=0; local data: DataPoint;
if ( index in store[id, filter_name] ) if ( index in store[id, filter_name] )
val = store[id, filter_name][index]; data = store[id, filter_name][index];
# fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val); # fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val);
event Metrics::cluster_index_response(uid, id, filter_name, index, val); event Metrics::cluster_index_response(uid, id, filter_name, index, data);
} }
@endif @endif
@ -195,21 +195,19 @@ function data_added(filter: Filter, index: Index, val: count)
do_notice(filter, index, val); do_notice(filter, index, val);
} }
event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, val: count) event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint)
{ {
#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr); #print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
if ( [uid, id, filter_name, index] !in index_requests )
index_requests[uid, id, filter_name, index] = 0;
index_requests[uid, id, filter_name, index] += val; index_requests[uid, id, filter_name, index] = merge_data_points(index_requests[uid, id, filter_name, index], data);
local ir = index_requests[uid, id, filter_name, index]; local ir = index_requests[uid, id, filter_name, index];
++done_with[uid]; ++done_with[uid];
if ( Cluster::worker_count == done_with[uid] ) if ( Cluster::worker_count == done_with[uid] )
{ {
if ( check_notice(filter_store[id, filter_name], index, ir) ) local size = ir?$num ? ir$num : |ir$unique_vals|;
do_notice(filter_store[id, filter_name], index, ir); if ( check_notice(filter_store[id, filter_name], index, size) )
do_notice(filter_store[id, filter_name], index, size);
delete done_with[uid]; delete done_with[uid];
delete index_requests[uid, id, filter_name, index]; delete index_requests[uid, id, filter_name, index];
} }
@ -233,12 +231,13 @@ event Metrics::cluster_filter_response(uid: string, id: string, filter_name: str
local local_data = filter_results[uid, id, filter_name]; local local_data = filter_results[uid, id, filter_name];
for ( index in data ) for ( index in data )
{ {
if ( index !in local_data ) if ( index in local_data )
local_data[index] = 0; local_data[index] = merge_data_points(local_data[index], data[index]);
local_data[index] += data[index]; else
local_data[index] = data[index];
} }
# Mark another worker as being "done" for this uid. # Mark another worker as being "done" for this uid.
if ( done ) if ( done )
++done_with[uid]; ++done_with[uid];

26
scripts/base/frameworks/metrics/main.bro
View file

@ -192,6 +192,32 @@ function index2str(index: Index): string
return fmt("metric_index(%s)", out); return fmt("metric_index(%s)", out);
} }
function merge_data_points(dp1: DataPoint, dp2: DataPoint): DataPoint
{
local result: DataPoint;
if ( dp1?$num || dp2?$num )
{
result$num = 0;
if ( dp1?$num )
result$num += dp1$num;
if ( dp2?$num )
result$num += dp2$num;
}
if ( dp1?$unique_vals || dp2?$unique_vals )
{
result$unique_vals = set();
if ( dp1?$unique_vals )
for ( val1 in dp1$unique_vals )
add result$unique_vals[val1];
if ( dp2?$unique_vals )
for ( val2 in dp2$unique_vals )
add result$unique_vals[val2];
}
return result;
}
function write_log(ts: time, filter: Filter, data: MetricTable) function write_log(ts: time, filter: Filter, data: MetricTable)
{ {
for ( index in data ) for ( index in data )

10
testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
View file

@ -3,8 +3,8 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path metrics #path metrics
#fields ts metric_id filter_name index.host index.str index.network value #fields ts ts_delta filter_name metric_id index.str index.host index.network value
#types time enum string addr string subnet count #types time interval string string string addr subnet count
1328303679.867377 TEST_METRIC foo-bar 6.5.4.3 - - 4 1332957065.172883 3.000000 foo-bar test.metric - 6.5.4.3 - 4
1328303679.867377 TEST_METRIC foo-bar 7.2.1.5 - - 2 1332957065.172883 3.000000 foo-bar test.metric - 1.2.3.4 - 6
1328303679.867377 TEST_METRIC foo-bar 1.2.3.4 - - 6 1332957065.172883 3.000000 foo-bar test.metric - 7.2.1.5 - 2

10
testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
View file

@ -3,8 +3,8 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path metrics #path metrics
#fields ts metric_id filter_name index.host index.str index.network value #fields ts ts_delta filter_name metric_id index.str index.host index.network value
#types time enum string addr string subnet count #types time interval string string string addr subnet count
1328303763.333948 TEST_METRIC foo-bar 6.5.4.3 - - 2 1332956138.267655 3.000000 foo-bar test.metric - 6.5.4.3 - 2
1328303763.333948 TEST_METRIC foo-bar 7.2.1.5 - - 1 1332956138.267655 3.000000 foo-bar test.metric - 1.2.3.4 - 3
1328303763.333948 TEST_METRIC foo-bar 1.2.3.4 - - 3 1332956138.267655 3.000000 foo-bar test.metric - 7.2.1.5 - 1

6
testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
View file

@ -3,6 +3,6 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path notice #path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.str metric_index.host metric_index.network
#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet #types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double string addr subnet
1325633225.777902 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - 1332957572.934499 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - 1.2.3.4 -

8
testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
View file

@ -3,7 +3,7 @@
#empty_field (empty) #empty_field (empty)
#unset_field - #unset_field -
#path notice #path notice
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.str metric_index.host metric_index.network
#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet #types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double string addr subnet
1325633274.875473 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - 1332956197.821031 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 3/2 - 1.2.3.4 - - 3 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - - 1.2.3.4 -
1325633274.875473 - - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - 6.5.4.3 - - 1332956197.821031 - - - - - - Test_Notice Threshold crossed by metric_index(host=6.5.4.3) 2/2 - 6.5.4.3 - - 2 bro Notice::ACTION_LOG 6 3600.000000 F - - - - - - 6.5.4.3 -

16
testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
View file

@ -10,8 +10,8 @@
@TEST-START-FILE cluster-layout.bro @TEST-START-FILE cluster-layout.bro
redef Cluster::nodes = { redef Cluster::nodes = {
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
}; };
@ -19,20 +19,16 @@ redef Cluster::nodes = {
redef Log::default_rotation_interval = 0secs; redef Log::default_rotation_interval = 0secs;
redef enum Metrics::ID += {
TEST_METRIC,
};
event bro_init() &priority=5 event bro_init() &priority=5
{ {
Metrics::add_filter(TEST_METRIC, Metrics::add_filter("test.metric",
[$name="foo-bar", [$name="foo-bar",
$break_interval=3secs]); $break_interval=3secs]);
if ( Cluster::local_node_type() == Cluster::WORKER ) if ( Cluster::local_node_type() == Cluster::WORKER )
{ {
Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
} }
} }

12
testing/btest/scripts/base/frameworks/metrics/basic.bro
View file

@ -1,16 +1,12 @@
# @TEST-EXEC: bro %INPUT # @TEST-EXEC: bro %INPUT
# @TEST-EXEC: btest-diff metrics.log # @TEST-EXEC: btest-diff metrics.log
redef enum Metrics::ID += {
TEST_METRIC,
};
event bro_init() &priority=5 event bro_init() &priority=5
{ {
Metrics::add_filter(TEST_METRIC, Metrics::add_filter("test.metric",
[$name="foo-bar", [$name="foo-bar",
$break_interval=3secs]); $break_interval=3secs]);
Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
} }

12
testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
View file

@ -10,8 +10,8 @@
@TEST-START-FILE cluster-layout.bro @TEST-START-FILE cluster-layout.bro
redef Cluster::nodes = { redef Cluster::nodes = {
["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
}; };
@ -23,13 +23,9 @@ redef enum Notice::Type += {
Test_Notice, Test_Notice,
}; };
redef enum Metrics::ID += {
TEST_METRIC,
};
event bro_init() &priority=5 event bro_init() &priority=5
{ {
Metrics::add_filter(TEST_METRIC, Metrics::add_filter("test.metric",
[$name="foo-bar", [$name="foo-bar",
$break_interval=1hr, $break_interval=1hr,
$note=Test_Notice, $note=Test_Notice,
@ -44,7 +40,7 @@ event do_metrics(i: count)
# Worker-1 will trigger an intermediate update and then if everything # Worker-1 will trigger an intermediate update and then if everything
# works correctly, the data from worker-2 will hit the threshold and # works correctly, the data from worker-2 will hit the threshold and
# should trigger the notice. # should trigger the notice.
Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], i); Metrics::add_data("test.metric", [$host=1.2.3.4], i);
} }
event bro_init() event bro_init()

12
testing/btest/scripts/base/frameworks/metrics/notice.bro
View file

@ -6,19 +6,15 @@ redef enum Notice::Type += {
Test_Notice, Test_Notice,
}; };
redef enum Metrics::ID += {
TEST_METRIC,
};
event bro_init() &priority=5 event bro_init() &priority=5
{ {
Metrics::add_filter(TEST_METRIC, Metrics::add_filter("test.metric",
[$name="foo-bar", [$name="foo-bar",
$break_interval=3secs, $break_interval=3secs,
$note=Test_Notice, $note=Test_Notice,
$notice_threshold=2, $notice_threshold=2,
$log=F]); $log=F]);
Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
} }