Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-12 19:48:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Added the branch /testing

Browse source
This commit is contained in:
Sheharbano Khattak 2012-10-16 05:54:38 +05:00
parent 83910eeb08
commit d510702078
3 changed files with 116 additions and 58 deletions
Download patch file Download diff file Expand all files Collapse all files

15
scripts/base/frameworks/metrics/cluster.bro
View file

@ -107,7 +107,7 @@ function data_added(filter: Filter, index: Index, val: count)
# intermediate update. # intermediate update.
local pct_val = double_to_count(val / cluster_request_global_view_percent); local pct_val = double_to_count(val / cluster_request_global_view_percent);
if ( check_notice(filter, index, pct_val) ) if ( check_threshold(filter, index, pct_val) )
{ {
# kick off intermediate update # kick off intermediate update
event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val); event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val);
@ -137,6 +137,9 @@ event Metrics::send_data(uid: string, id: string, filter_name: string, data: Met
# If data is empty, this metric is done. # If data is empty, this metric is done.
if ( |data| == 0 ) if ( |data| == 0 )
done = T; done = T;
#print "Here is local_data";
#print local_data;
event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done); event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
if ( ! done ) if ( ! done )
@ -191,8 +194,9 @@ event Metrics::log_it(filter: Filter)
# being collected by managers. # being collected by managers.
function data_added(filter: Filter, index: Index, val: count) function data_added(filter: Filter, index: Index, val: count)
{ {
if ( check_notice(filter, index, val) ) if ( check_threshold(filter, index, val) )
do_notice(filter, index, val); threshold_crossed_alert( filter, index, val );
#do_notice(filter, index, val);
} }
event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint) event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint)
@ -206,8 +210,9 @@ event Metrics::cluster_index_response(uid: string, id: string, filter_name: stri
if ( Cluster::worker_count == done_with[uid] ) if ( Cluster::worker_count == done_with[uid] )
{ {
local size = ir?$num ? ir$num : |ir$unique_vals|; local size = ir?$num ? ir$num : |ir$unique_vals|;
if ( check_notice(filter_store[id, filter_name], index, size) ) if ( check_threshold(filter_store[id, filter_name], index, size) )
do_notice(filter_store[id, filter_name], index, size); threshold_crossed_alert( filter_store[id, filter_name], index, size );
#do_notice(filter_store[id, filter_name], index, size);
delete done_with[uid]; delete done_with[uid];
delete index_requests[uid, id, filter_name, index]; delete index_requests[uid, id, filter_name, index];
} }

154
scripts/base/frameworks/metrics/main.bro
View file

@ -14,7 +14,7 @@ export {
## This is the interval for how often threshold based notices will happen ## This is the interval for how often threshold based notices will happen
## after they have already fired. ## after they have already fired.
const renotice_interval = 1hr &redef; const renotice_interval = 12hr &redef;
## Represents a thing which is having metrics collected for it. An instance ## Represents a thing which is having metrics collected for it. An instance
## of this record type and an id together represent a single measurement. ## of this record type and an id together represent a single measurement.
@ -98,11 +98,7 @@ export {
log: bool &default=T; log: bool &default=T;
## If this and a $notice_threshold value are set, this notice type ## If this and a $notice_threshold value are set, this notice type
## will be generated by the metrics framework. ## will be generated by the metrics framework.
note: Notice::Type &optional; note: Notice::Type &optional;
## A straight threshold for generating a notice.
notice_threshold: count &optional;
## A series of thresholds at which to generate notices.
notice_thresholds: vector of count &optional;
## Sheharbano's additions ## Sheharbano's additions
##-------------------------------------------- ##--------------------------------------------
@ -115,8 +111,20 @@ export {
custom_thresholds: table[Index] of count &optional; custom_thresholds: table[Index] of count &optional;
## A predicate so that you can decide when to flexibly declare when ## A predicate so that you can decide when to flexibly declare when
## a threshold crossed, and do extra stuff ## a threshold crossed, and do extra stuff
check_threshold: function(index: Index, default_thresh: count, custom_check_threshold: function(index: Index, default_thresh: count,
custom_thresh: table[Index] of count, val: count ): bool &optional; custom_thresh: table[Index] of count, val: count ): bool &optional;
## Even if custom_check_threshold has been defined, we don't want
## to call it every time because of function call overhead.
## Metrics::Filter$trigger_custom_check_threshold describes how often
## custom_check_threshold will be called
## e.g. call custom_check_threshold for every 5 items seen by the metrics fw
trigger_custom_check_threshold: count &default=1;
## A predicate that is called whenever a threshold is crossed
## ToDo: Also have a parameter here that is a sample of the
## observed trackable items
threshold_crossed: function(index: Index, val: count ) &optional;
## A series of thresholds at which to generate notices.
threshold_series: vector of count &optional;
}; };
## Function to associate a metric filter with a metric ID. ## Function to associate a metric filter with a metric ID.
@ -136,6 +144,13 @@ export {
## ##
## increment: How much to increment the counter by. ## increment: How much to increment the counter by.
global add_data: function(id: string, index: Index, increment: count); global add_data: function(id: string, index: Index, increment: count);
# This function does the following:
# If index (src,) doesn't exist, it creates an entry for this index. It
# adds data (c$id$orig_h) to a set associated with this index. If the number
# of unique data values for an index exceeds threshold, a notice is generated.
# So the threshold applies to the number of unique data values associated with
# an index.
global add_unique: function(id: string, index: Index, data: string); global add_unique: function(id: string, index: Index, data: string);
@ -173,15 +188,13 @@ global store: table[string, string] of MetricTable = table() &default=table();
# This function checks if a threshold has been crossed and generates a # This function checks if a threshold has been crossed and generates a
# notice if it has. It is also used as a method to implement # notice if it has. It is also used as a method to implement
# mid-break-interval threshold crossing detection for cluster deployments. # mid-break-interval threshold crossing detection for cluster deployments.
global check_notice: function(filter: Filter, index: Index, val: count): bool; global check_threshold: function(filter: Filter, index: Index, val: count): bool;
# This is hook for watching thresholds being crossed. It is called whenever # This is hook for watching thresholds being crossed. It is called whenever
# index values are updated and the new val is given as the `val` argument. # index values are updated and the new val is given as the `val` argument.
global data_added: function(filter: Filter, index: Index, val: count); global data_added: function(filter: Filter, index: Index, val: count);
# This stores the current threshold index for filters using the # This stores the current threshold index for filters using $threshold_series.
# $notice_threshold and $notice_thresholds elements. global threshold_series_index: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0;
global thresholds: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0;
event bro_init() &priority=5 event bro_init() &priority=5
{ {
@ -265,16 +278,23 @@ function add_filter(id: string, filter: Filter)
print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name); print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
return; return;
} }
if ( filter?$notice_threshold && filter?$notice_thresholds ) if ( filter?$threshold_series && ( filter?$default_threshold || filter?$custom_thresholds ) )
{ {
print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds"; print "INVALID Metric filter: Cannot define $custom_thresholds and $default_threshold with $threshold_series";
return; return;
} }
if ( !filter?$default_threshold && !filter?$custom_thresholds ) if ( !filter?$default_threshold && !filter?$custom_thresholds && !filter?$threshold_series )
{ {
print "INVALID Metric filter: Must define one of $default_threshold and $custom_thresholds"; print "INVALID Metric filter: Must define one of $default_threshold, $custom_thresholds and threshold_series";
return; return;
} }
#Bro throws error anyway when a non-optional record field is missing
#if ( !filter?$threshold_crossed )
# {
# print "INVALID Metric filter: Must define the function $threshold_crossed";
# return;
# }
if ( ! filter?$id ) if ( ! filter?$id )
filter$id = id; filter$id = id;
@ -345,6 +365,8 @@ function add_it(id: string, index: Index, integer_value: bool, num: count, str:
metric_tbl[index] = [$unique_vals=empty_ss]; metric_tbl[index] = [$unique_vals=empty_ss];
} }
add metric_tbl[index]$unique_vals[str]; add metric_tbl[index]$unique_vals[str];
#print metric_tbl[index]$unique_vals;
#print "-------------------------------------";
data_added(filter, index, |metric_tbl[index]$unique_vals|); data_added(filter, index, |metric_tbl[index]$unique_vals|);
} }
} }
@ -360,7 +382,7 @@ function add_unique(id: string, index: Index, data: string)
add_it(id, index, F, 0, data); add_it(id, index, F, 0, data);
} }
function check_notice(filter: Filter, index: Index, val: count): bool function check_threshold(filter: Filter, index: Index, val: count): bool
{ {
## It's possible for a user to skip defining either default_threshold or custom_thresholds. ## It's possible for a user to skip defining either default_threshold or custom_thresholds.
## Therefore must check which one is missing, so we can craft and send a dummy value in the function ## Therefore must check which one is missing, so we can craft and send a dummy value in the function
@ -373,59 +395,89 @@ function check_notice(filter: Filter, index: Index, val: count): bool
if ( filter?$default_threshold ) if ( filter?$default_threshold )
def_thresh = filter$default_threshold; def_thresh = filter$default_threshold;
if ( filter?$custom_check_threshold && ( val%filter$trigger_custom_check_threshold == 0 ) )
return filter$custom_check_threshold( index, def_thresh, cust_thresh, val );
if ( filter?$check_threshold ) # No custom check threshold defined
return filter$check_threshold( index, def_thresh, cust_thresh, val ); else if ( !filter?$custom_check_threshold )
else
{ {
if ( index in cust_thresh ) if ( index in cust_thresh )
{ {
if ( val > cust_thresh[index] ) if ( val > cust_thresh[index] )
return T; return T;
} }
else if ( val > def_thresh) else if ( filter?$default_threshold )
return T; {
if ( val > def_thresh)
{
if ( index$str == "80/tcp")
print fmt("default threshold--val is %d for index %s",val,index);
return T;
}
}
else if ( filter?$threshold_series )
{
#print threshold_series_index[filter$id, filter$name, index];
if ( |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] &&
val >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] )
{
if ( index$str == "80/tcp")
print fmt("series threshold--val is %d for index %s",val,index);
return T;
}
}
return F; return F;
} }
#else if ( !filter?$custom_check_threshold )
# {
# if ( index in cust_thresh )
# {
# if ( val > cust_thresh[index] )
# return T;
# }
# else if ( val > def_thresh)
# return T;
# }
#if ( (filter?$notice_threshold && return F;
# [filter$id, filter$name, index] !in thresholds &&
# val >= filter$notice_threshold) ||
# (filter?$notice_thresholds &&
# |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
# val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
#return T;
#else
#return F;
} }
function do_notice(filter: Filter, index: Index, val: count) function threshold_crossed_alert(filter: Filter, index: Index, val: count)
{ {
# We include $peer_descr here because the a manager count have actually # We include $peer_descr here because the a manager count have actually
# generated the notice even though the current remote peer for the event # generated the notice even though the current remote peer for the event
# calling this could be a worker if this is running as a cluster. # calling this could be a worker if this is running as a cluster.
local n: Notice::Info = [$note=filter$note, #local n: Notice::Info = [$note=filter$note,
$n=val, # $n=val,
$metric_index=index, # $metric_index=index,
$peer_descr=peer_description]; # $peer_descr=peer_description];
n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold); #n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
if ( index?$str ) #if ( index?$str )
n$sub = index$str; # n$sub = index$str;
if ( index?$host ) #if ( index?$host )
n$src = index$host; # n$src = index$host;
# TODO: not sure where to put the network yet. # TODO: not sure where to put the network yet.
NOTICE(n); #NOTICE(n);
if ( filter?$threshold_crossed )
filter$threshold_crossed( index, val );
# Resetting unique values # Resetting unique values
local metric_tbl = store[filter$id, filter$name]; #local metric_tbl = store[filter$id, filter$name];
metric_tbl[index]$unique_vals = set(); #metric_tbl[index]$unique_vals = set();
# If I don't reset here, the value just keeps
# retriggering once the threshold has been exceeded
# This just needs set to some value so that it doesn't refire the if ( !filter?$threshold_series )
# notice until it expires from the table or it crosses the next reset(filter);
# threshold in the case of vectors of thresholds. else
++thresholds[filter$id, filter$name, index]; {
# This just needs set to some value so that it doesn't refire the
# notice until it expires from the table or it crosses the next
# threshold in the case of vectors of thresholds.
++threshold_series_index[filter$id, filter$name, index];
}
} }

5
scripts/base/frameworks/metrics/non-cluster.bro
View file

@ -16,6 +16,7 @@ event Metrics::log_it(filter: Filter)
function data_added(filter: Filter, index: Index, val: count) function data_added(filter: Filter, index: Index, val: count)
{ {
if ( check_notice(filter, index, val) ) if ( check_threshold(filter, index, val) )
do_notice(filter, index, val); threshold_crossed_alert( filter, index, val );
#do_notice(filter, index, val);
} }