From d510702078e89a1c6914794f902437614aca48d2 Mon Sep 17 00:00:00 2001 From: Sheharbano Khattak Date: Tue, 16 Oct 2012 05:54:38 +0500 Subject: [PATCH] Added the branch /testing --- scripts/base/frameworks/metrics/cluster.bro | 15 +- scripts/base/frameworks/metrics/main.bro | 154 ++++++++++++------ .../base/frameworks/metrics/non-cluster.bro | 5 +- 3 files changed, 116 insertions(+), 58 deletions(-) diff --git a/scripts/base/frameworks/metrics/cluster.bro b/scripts/base/frameworks/metrics/cluster.bro index 03bbcdf584..19f431460f 100644 --- a/scripts/base/frameworks/metrics/cluster.bro +++ b/scripts/base/frameworks/metrics/cluster.bro @@ -107,7 +107,7 @@ function data_added(filter: Filter, index: Index, val: count) # intermediate update. local pct_val = double_to_count(val / cluster_request_global_view_percent); - if ( check_notice(filter, index, pct_val) ) + if ( check_threshold(filter, index, pct_val) ) { # kick off intermediate update event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val); @@ -137,6 +137,9 @@ event Metrics::send_data(uid: string, id: string, filter_name: string, data: Met # If data is empty, this metric is done. if ( |data| == 0 ) done = T; + + #print "Here is local_data"; + #print local_data; event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done); if ( ! done ) @@ -191,8 +194,9 @@ event Metrics::log_it(filter: Filter) # being collected by managers. function data_added(filter: Filter, index: Index, val: count) { - if ( check_notice(filter, index, val) ) - do_notice(filter, index, val); + if ( check_threshold(filter, index, val) ) + threshold_crossed_alert( filter, index, val ); + #do_notice(filter, index, val); } event Metrics::cluster_index_response(uid: string, id: string, filter_name: string, index: Index, data: DataPoint) @@ -206,8 +210,9 @@ event Metrics::cluster_index_response(uid: string, id: string, filter_name: stri if ( Cluster::worker_count == done_with[uid] ) { local size = ir?$num ? ir$num : |ir$unique_vals|; - if ( check_notice(filter_store[id, filter_name], index, size) ) - do_notice(filter_store[id, filter_name], index, size); + if ( check_threshold(filter_store[id, filter_name], index, size) ) + threshold_crossed_alert( filter_store[id, filter_name], index, size ); + #do_notice(filter_store[id, filter_name], index, size); delete done_with[uid]; delete index_requests[uid, id, filter_name, index]; } diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro index 744eaf731d..d4bfd8bd5f 100644 --- a/scripts/base/frameworks/metrics/main.bro +++ b/scripts/base/frameworks/metrics/main.bro @@ -14,7 +14,7 @@ export { ## This is the interval for how often threshold based notices will happen ## after they have already fired. - const renotice_interval = 1hr &redef; + const renotice_interval = 12hr &redef; ## Represents a thing which is having metrics collected for it. An instance ## of this record type and an id together represent a single measurement. @@ -98,11 +98,7 @@ export { log: bool &default=T; ## If this and a $notice_threshold value are set, this notice type ## will be generated by the metrics framework. - note: Notice::Type &optional; - ## A straight threshold for generating a notice. - notice_threshold: count &optional; - ## A series of thresholds at which to generate notices. - notice_thresholds: vector of count &optional; + note: Notice::Type &optional; ## Sheharbano's additions ##-------------------------------------------- @@ -115,8 +111,20 @@ export { custom_thresholds: table[Index] of count &optional; ## A predicate so that you can decide when to flexibly declare when ## a threshold crossed, and do extra stuff - check_threshold: function(index: Index, default_thresh: count, + custom_check_threshold: function(index: Index, default_thresh: count, custom_thresh: table[Index] of count, val: count ): bool &optional; + ## Even if custom_check_threshold has been defined, we don't want + ## to call it every time because of function call overhead. + ## Metrics::Filter$trigger_custom_check_threshold describes how often + ## custom_check_threshold will be called + ## e.g. call custom_check_threshold for every 5 items seen by the metrics fw + trigger_custom_check_threshold: count &default=1; + ## A predicate that is called whenever a threshold is crossed + ## ToDo: Also have a parameter here that is a sample of the + ## observed trackable items + threshold_crossed: function(index: Index, val: count ) &optional; + ## A series of thresholds at which to generate notices. + threshold_series: vector of count &optional; }; ## Function to associate a metric filter with a metric ID. @@ -136,6 +144,13 @@ export { ## ## increment: How much to increment the counter by. global add_data: function(id: string, index: Index, increment: count); + + # This function does the following: + # If index (src,) doesn't exist, it creates an entry for this index. It + # adds data (c$id$orig_h) to a set associated with this index. If the number + # of unique data values for an index exceeds threshold, a notice is generated. + # So the threshold applies to the number of unique data values associated with + # an index. global add_unique: function(id: string, index: Index, data: string); @@ -173,15 +188,13 @@ global store: table[string, string] of MetricTable = table() &default=table(); # This function checks if a threshold has been crossed and generates a # notice if it has. It is also used as a method to implement # mid-break-interval threshold crossing detection for cluster deployments. -global check_notice: function(filter: Filter, index: Index, val: count): bool; - +global check_threshold: function(filter: Filter, index: Index, val: count): bool; # This is hook for watching thresholds being crossed. It is called whenever # index values are updated and the new val is given as the `val` argument. global data_added: function(filter: Filter, index: Index, val: count); -# This stores the current threshold index for filters using the -# $notice_threshold and $notice_thresholds elements. -global thresholds: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0; +# This stores the current threshold index for filters using $threshold_series. +global threshold_series_index: table[string, string, Index] of count = {} &create_expire=renotice_interval &default=0; event bro_init() &priority=5 { @@ -265,16 +278,23 @@ function add_filter(id: string, filter: Filter) print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name); return; } - if ( filter?$notice_threshold && filter?$notice_thresholds ) + if ( filter?$threshold_series && ( filter?$default_threshold || filter?$custom_thresholds ) ) { - print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds"; + print "INVALID Metric filter: Cannot define $custom_thresholds and $default_threshold with $threshold_series"; return; } - if ( !filter?$default_threshold && !filter?$custom_thresholds ) + if ( !filter?$default_threshold && !filter?$custom_thresholds && !filter?$threshold_series ) { - print "INVALID Metric filter: Must define one of $default_threshold and $custom_thresholds"; + print "INVALID Metric filter: Must define one of $default_threshold, $custom_thresholds and threshold_series"; return; } + + #Bro throws error anyway when a non-optional record field is missing + #if ( !filter?$threshold_crossed ) + # { + # print "INVALID Metric filter: Must define the function $threshold_crossed"; + # return; + # } if ( ! filter?$id ) filter$id = id; @@ -345,6 +365,8 @@ function add_it(id: string, index: Index, integer_value: bool, num: count, str: metric_tbl[index] = [$unique_vals=empty_ss]; } add metric_tbl[index]$unique_vals[str]; + #print metric_tbl[index]$unique_vals; + #print "-------------------------------------"; data_added(filter, index, |metric_tbl[index]$unique_vals|); } } @@ -360,7 +382,7 @@ function add_unique(id: string, index: Index, data: string) add_it(id, index, F, 0, data); } -function check_notice(filter: Filter, index: Index, val: count): bool +function check_threshold(filter: Filter, index: Index, val: count): bool { ## It's possible for a user to skip defining either default_threshold or custom_thresholds. ## Therefore must check which one is missing, so we can craft and send a dummy value in the function @@ -373,59 +395,89 @@ function check_notice(filter: Filter, index: Index, val: count): bool if ( filter?$default_threshold ) def_thresh = filter$default_threshold; + + if ( filter?$custom_check_threshold && ( val%filter$trigger_custom_check_threshold == 0 ) ) + return filter$custom_check_threshold( index, def_thresh, cust_thresh, val ); - if ( filter?$check_threshold ) - return filter$check_threshold( index, def_thresh, cust_thresh, val ); - - else + # No custom check threshold defined + else if ( !filter?$custom_check_threshold ) { if ( index in cust_thresh ) { if ( val > cust_thresh[index] ) return T; } - else if ( val > def_thresh) - return T; + else if ( filter?$default_threshold ) + { + if ( val > def_thresh) + { + if ( index$str == "80/tcp") + print fmt("default threshold--val is %d for index %s",val,index); + return T; + } + } + else if ( filter?$threshold_series ) + { + #print threshold_series_index[filter$id, filter$name, index]; + if ( |filter$threshold_series| >= threshold_series_index[filter$id, filter$name, index] && + val >= filter$threshold_series[threshold_series_index[filter$id, filter$name, index]] ) + { + if ( index$str == "80/tcp") + print fmt("series threshold--val is %d for index %s",val,index); + return T; + } + } + return F; } + #else if ( !filter?$custom_check_threshold ) + # { + # if ( index in cust_thresh ) + # { + # if ( val > cust_thresh[index] ) + # return T; + # } + # else if ( val > def_thresh) + # return T; + # } - #if ( (filter?$notice_threshold && - # [filter$id, filter$name, index] !in thresholds && - # val >= filter$notice_threshold) || - # (filter?$notice_thresholds && - # |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] && - # val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) ) - #return T; - #else - #return F; + return F; } -function do_notice(filter: Filter, index: Index, val: count) +function threshold_crossed_alert(filter: Filter, index: Index, val: count) { # We include $peer_descr here because the a manager count have actually # generated the notice even though the current remote peer for the event # calling this could be a worker if this is running as a cluster. - local n: Notice::Info = [$note=filter$note, - $n=val, - $metric_index=index, - $peer_descr=peer_description]; - n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold); - if ( index?$str ) - n$sub = index$str; - if ( index?$host ) - n$src = index$host; + #local n: Notice::Info = [$note=filter$note, + # $n=val, + # $metric_index=index, + # $peer_descr=peer_description]; + #n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold); + #if ( index?$str ) + # n$sub = index$str; + #if ( index?$host ) + # n$src = index$host; # TODO: not sure where to put the network yet. - NOTICE(n); + #NOTICE(n); + + if ( filter?$threshold_crossed ) + filter$threshold_crossed( index, val ); # Resetting unique values - local metric_tbl = store[filter$id, filter$name]; - metric_tbl[index]$unique_vals = set(); - - - # This just needs set to some value so that it doesn't refire the - # notice until it expires from the table or it crosses the next - # threshold in the case of vectors of thresholds. - ++thresholds[filter$id, filter$name, index]; + #local metric_tbl = store[filter$id, filter$name]; + #metric_tbl[index]$unique_vals = set(); + # If I don't reset here, the value just keeps + # retriggering once the threshold has been exceeded + if ( !filter?$threshold_series ) + reset(filter); + else + { + # This just needs set to some value so that it doesn't refire the + # notice until it expires from the table or it crosses the next + # threshold in the case of vectors of thresholds. + ++threshold_series_index[filter$id, filter$name, index]; + } } diff --git a/scripts/base/frameworks/metrics/non-cluster.bro b/scripts/base/frameworks/metrics/non-cluster.bro index 85c050fb25..dc0daea2be 100644 --- a/scripts/base/frameworks/metrics/non-cluster.bro +++ b/scripts/base/frameworks/metrics/non-cluster.bro @@ -16,6 +16,7 @@ event Metrics::log_it(filter: Filter) function data_added(filter: Filter, index: Index, val: count) { - if ( check_notice(filter, index, val) ) - do_notice(filter, index, val); + if ( check_threshold(filter, index, val) ) + threshold_crossed_alert( filter, index, val ); + #do_notice(filter, index, val); }