diff --git a/scripts/base/frameworks/metrics/conn-example.bro b/scripts/base/frameworks/metrics/conn-example.bro deleted file mode 100644 index 9e20798cc0..0000000000 --- a/scripts/base/frameworks/metrics/conn-example.bro +++ /dev/null @@ -1,19 +0,0 @@ -@load frameworks/metrics - -redef enum Metrics::ID += { - CONNS_ORIGINATED, - CONNS_RESPONDED -}; - -event bro_init() - { - Metrics::configure(CONNS_ORIGINATED, [$aggregation_mask=24, $break_interval=5mins]); - Metrics::configure(CONNS_RESPONDED, [$aggregation_mask=24, $break_interval=5mins]); - } - -event connection_established(c: connection) - { - Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h], 1); - Metrics::add_data(CONNS_RESPONDED, [$host=c$id$resp_h], 1); - } - \ No newline at end of file diff --git a/scripts/base/frameworks/metrics/http-example.bro b/scripts/base/frameworks/metrics/http-example.bro deleted file mode 100644 index a91fdbb47d..0000000000 --- a/scripts/base/frameworks/metrics/http-example.bro +++ /dev/null @@ -1,20 +0,0 @@ -@load frameworks/metrics - -redef enum Metrics::ID += { - HTTP_REQUESTS_BY_STATUS_CODE, - HTTP_REQUESTS_BY_HOST, -}; - -event bro_init() - { - Metrics::configure(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_mask=24, $break_interval=10secs]); - Metrics::configure(HTTP_REQUESTS_BY_HOST, [$break_interval=10secs]); - } - -event HTTP::log_http(rec: HTTP::Info) - { - if ( rec?$host ) - Metrics::add_data(HTTP_REQUESTS_BY_HOST, [$index=rec$host], 1); - if ( rec?$status_code ) - Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $index=fmt("%d", rec$status_code)], 1); - } \ No newline at end of file diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro index b603add6de..29f18ab824 100644 --- a/scripts/base/frameworks/metrics/main.bro +++ b/scripts/base/frameworks/metrics/main.bro @@ -1,30 +1,28 @@ -##! This is the implementation of the metrics framework +##! This is the implementation of the metrics framework. module Metrics; export { redef enum Log::ID += { METRICS }; - + type ID: enum { - ALL, + NOTHING, }; - const default_aggregation_mask = 24 &redef; - const default_break_interval = 5mins &redef; + ## The default interval used for "breaking" metrics and writing the + ## current value to the logging stream. + const default_break_interval = 15mins &redef; - # TODO: configure a metrics filter logging stream to log the current - # metrics configuration in case someone is looking through - # old logs and the configuration has changed since then. - type Filter: record { - name: ID &optional; - ## Global mask by which you'd like to aggregate traffic. - aggregation_mask: count &optional; - ## This is essentially applying names to various subnets. - aggregation_table: table[subnet] of string &optional; - break_interval: interval &default=default_break_interval; + type Info: record { + ts: time &log; + metric_id: ID &log; + filter_name: string &log; + agg_subnet: string &log &optional; + index: string &log &optional; + value: count &log; }; - type Index: record { + type Entry: record { ## Host is the value to which this metric applies. host: addr &optional; @@ -36,36 +34,71 @@ export { ## metric since multiple IP addresses could respond for the same Host ## header value. index: string &default=""; + + ## The value by which the counter should be increased in each filter + ## where this entry is accepted. + increment: count &default=1; }; - type Info: record { - ts: time &log; - name: ID &log; - index: string &log &optional; - agg_subnet: string &log &optional; - value: count &log; + # TODO: configure a metrics filter logging stream to log the current + # metrics configuration in case someone is looking through + # old logs and the configuration has changed since then. + type Filter: record { + ## The :bro:type:`Metrics::ID` that this filter applies to. + id: ID &optional; + ## The name for this filter so that multiple filters can be + ## applied to a single metrics to get a different view of the same + ## metric data being collected (different aggregation, break, etc). + name: string &default="default"; + ## A predicate so that you can decide per index if you would like + ## to accept the data being inserted. + pred: function(entry: Entry): bool &optional; + ## Global mask by which you'd like to aggregate traffic. + aggregation_mask: count &optional; + ## This is essentially applying names to various subnets. + aggregation_table: table[subnet] of string &optional; + ## The interval at which the metric should be "broken" and written + ## to the logging stream. + break_interval: interval &default=default_break_interval; + ## This determines if the result of this filter is sent to the metrics + ## logging stream. One use for the logging framework is as an internal + ## thresholding and statistics gathering utility that is meant to + ## never log but rather to generate notices and derive data. + log: bool &default=T; + ## A straight threshold for generating a notice. + notice_threshold: count &optional; + ## A series of thresholds at which to generate notices. + notice_thresholds: vector of count &optional; + ## If this and a $notice_threshold value are set, this notice type + ## will be generated by the metrics framework. + note: Notice::Type &optional; }; - global add_filter: function(name: ID, filter: Filter); - global add_data: function(name: ID, index: Index, increment: count); + global add_filter: function(id: ID, filter: Filter); + global add_data: function(id: ID, entry: Entry); global log_metrics: event(rec: Info); } -global metric_filters: table[ID] of Filter = table(); +global metric_filters: table[ID] of vector of Filter = table(); type MetricIndex: table[string] of count &default=0; type MetricTable: table[string] of MetricIndex; -global store: table[ID] of MetricTable = table(); +# This is indexed by metric ID and stream filter name. +global store: table[ID, string] of MetricTable = table(); -event bro_init() +# This stores the current threshold index for filters using the +# $notice_thresholds element. +global thresholds: table[string] of count = {} &default=0; + +event bro_init() &priority=5 { Log::create_stream(METRICS, [$columns=Info, $ev=log_metrics]); } -function reset(name: ID) +function reset(filter: Filter) { - store[name] = table(); + store[filter$id, filter$name] = table(); } event log_it(filter: Filter) @@ -73,70 +106,128 @@ event log_it(filter: Filter) # If this node is the manager in a cluster, this needs to request values # for this metric from all of the workers. + local id = filter$id; local name = filter$name; - for ( agg_subnet in store[name] ) + for ( agg_subnet in store[id, name] ) { - local metric_values = store[name][agg_subnet]; + local metric_values = store[id, name][agg_subnet]; for ( index in metric_values ) { local val = metric_values[index]; local m: Info = [$ts=network_time(), - $name=name, - $agg_subnet=fmt("%s", agg_subnet), - $index=index, + $metric_id=id, + $filter_name=name, + $agg_subnet=fmt("%s", agg_subnet), + $index=index, $value=val]; + + if ( filter?$notice_threshold && + m$value >= filter$notice_threshold ) + { + print m; + NOTICE([$note=filter$note, + $msg=fmt("Metrics threshold crossed by %s %d/%d", m$agg_subnet, m$value, filter$notice_threshold), + $n=m$value]); + } + + else if ( filter?$notice_thresholds && + m$value >= filter$notice_thresholds[thresholds[cat(id,name)]] ) + { + # TODO: implement this + } + + # If there wasn't an index, remove the field. if ( index == "" ) delete m$index; + + # If there wasn't an aggregation subnet, remove the field. if ( agg_subnet == "" ) delete m$agg_subnet; + Log::write(METRICS, m); } } - - - reset(name); + reset(filter); schedule filter$break_interval { log_it(filter) }; } -function add_filter(name: ID, filter: Filter) +function add_filter(id: ID, filter: Filter) { if ( filter?$aggregation_table && filter?$aggregation_mask ) { print "INVALID Metric filter: Defined $aggregation_table and $aggregation_mask."; return; } + if ( [id, filter$name] in store ) + { + print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name); + return; + } + if ( filter?$notice_threshold && filter?$notice_thresholds ) + { + print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds"; + return; + } - filter$name = name; - metric_filters[name] = filter; - store[name] = table(); + if ( ! filter?$id ) + filter$id = id; + + if ( id !in metric_filters ) + metric_filters[id] = vector(); + metric_filters[id][|metric_filters[id]|] = filter; + + store[id, filter$name] = table(); # Only do this on the manager if in a cluster. schedule filter$break_interval { log_it(filter) }; } -function add_data(name: ID, index: Index, increment: count) +function add_data(id: ID, entry: Entry) { - local conf = metric_filters[name]; - - local agg_subnet = ""; - if ( index?$host ) + if ( id !in metric_filters ) + return; + + local filters = metric_filters[id]; + + # Add the data to any of the defined filters. + for ( filter_id in filters ) { - if ( conf?$aggregation_mask ) + local filter = filters[filter_id]; + + # If this filter has a predicate, run the predicate and skip this + # entry if the predicate return false. + if ( filter?$pred && + ! filter$pred(entry) ) + next; + + local agg_subnet = ""; + local filt_store = store[id, filter$name]; + if ( entry?$host ) { - local agg_mask = conf$aggregation_mask; - agg_subnet = fmt("%s", mask_addr(index$host, agg_mask)); + if ( filter?$aggregation_mask ) + { + local agg_mask = filter$aggregation_mask; + agg_subnet = fmt("%s", mask_addr(entry$host, agg_mask)); + } + else if ( filter?$aggregation_table ) + { + agg_subnet = fmt("%s", filter$aggregation_table[entry$host]); + # if an aggregation table is being used and the value isn't + # in the table, that means we aren't interested in it. + if ( agg_subnet == "" ) + next; + } + else + agg_subnet = fmt("%s", entry$host); } - else if ( conf?$aggregation_table ) - agg_subnet = fmt("%s", conf$aggregation_table[index$host]); - else - agg_subnet = fmt("%s", index$host); + + if ( agg_subnet !in filt_store ) + filt_store[agg_subnet] = table(); + + local fs = filt_store[agg_subnet]; + if ( entry$index !in fs ) + fs[entry$index] = 0; + fs[entry$index] = fs[entry$index] + entry$increment; } - - if ( agg_subnet !in store[name] ) - store[name][agg_subnet] = table(); - - if ( index$index !in store[name][agg_subnet] ) - store[name][agg_subnet][index$index] = 0; - store[name][agg_subnet][index$index] = store[name][agg_subnet][index$index] + increment; } diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro new file mode 100644 index 0000000000..e67117a7e0 --- /dev/null +++ b/scripts/policy/frameworks/metrics/conn-example.bro @@ -0,0 +1,20 @@ + +redef enum Metrics::ID += { + CONNS_ORIGINATED, + CONNS_RESPONDED +}; + +event bro_init() + { + Metrics::add_filter(CONNS_ORIGINATED, [$aggregation_mask=24, $break_interval=1mins]); + + # Site::local_nets must be defined in order for this to actually do anything. + Metrics::add_filter(CONNS_RESPONDED, [$aggregation_table=Site::local_nets_table, $break_interval=1mins]); + } + +event connection_established(c: connection) + { + Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h]); + Metrics::add_data(CONNS_RESPONDED, [$host=c$id$resp_h]); + } + \ No newline at end of file diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro new file mode 100644 index 0000000000..904ec9a227 --- /dev/null +++ b/scripts/policy/frameworks/metrics/http-example.bro @@ -0,0 +1,22 @@ + + +redef enum Metrics::ID += { + HTTP_REQUESTS_BY_STATUS_CODE, + HTTP_REQUESTS_BY_HOST_HEADER, +}; + +event bro_init() + { + Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER, [$break_interval=5mins]); + + # Site::local_nets must be defined in order for this to actually do anything. + Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table, $break_interval=5mins]); + } + +event HTTP::log_http(rec: HTTP::Info) + { + if ( rec?$host ) + Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$index=rec$host]); + if ( rec?$status_code ) + Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $index=fmt("%d", rec$status_code)]); + } \ No newline at end of file diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro new file mode 100644 index 0000000000..e043690feb --- /dev/null +++ b/scripts/policy/frameworks/metrics/ssl-example.bro @@ -0,0 +1,22 @@ + + +redef enum Metrics::ID += { + SSL_SERVERNAME, +}; + +event bro_init() + { + Metrics::add_filter(SSL_SERVERNAME, + [$name="no-google-ssl-servers", + $pred(entry: Metrics::Entry) = { + return (/google\.com$/ !in entry$index); + }, + $break_interval=10secs + ]); + } + +event SSL::log_ssl(rec: SSL::Info) + { + if ( rec?$server_name ) + Metrics::add_data(SSL_SERVERNAME, [$index=rec$server_name]); + } \ No newline at end of file diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index 514c599e7e..45a2bdb205 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -9,17 +9,17 @@ export { }; redef enum Metrics::ID += { - SQL_ATTACKER, - SQL_ATTACK_AGAINST, + SQL_ATTACKS, + SQL_ATTACKS_AGAINST, }; redef enum Tags += { ## Indicator of a URI based SQL injection attack. URI_SQLI, ## Indicator of client body based SQL injection attack. This is - ## typically the body content of a POST request. Not implemented yet! + ## typically the body content of a POST request. Not implemented yet. POST_SQLI, - ## Indicator of a cookie based SQL injection attack. Not implemented yet! + ## Indicator of a cookie based SQL injection attack. Not implemented yet. COOKIE_SQLI, }; @@ -30,13 +30,18 @@ export { | /[\?&][^[:blank:]\x00-\x37]+?=[\-0-9%]*([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/)*(-|=|\+|\|\|)([[:blank:]\x00-\x37]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/ | /[\?&][^[:blank:]\x00-\x37\|]+?=([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/|;)*([xX]?[oO][rR]|[nN]?[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/|[\[(])+[a-zA-Z&]{2,}/ | /[\?&][^[:blank:]\x00-\x37]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/ - | /\/\*![[:digit:]]{5}.*?\*\//; + | /\/\*![[:digit:]]{5}.*?\*\// &redef; } event bro_init() { - Metrics::add_filter(SQL_ATTACKER, [$break_interval=5mins, $note=SQL_Injection_Attack]); - Metrics::add_filter(SQL_ATTACK_AGAINST, [$break_interval=5mins, $note=SQL_Injection_Attack]); + Metrics::add_filter(SQL_ATTACKS, [$log=T, + $break_interval=1mins, + $note=SQL_Injection_Attacker]); + Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=T, + $break_interval=1mins, + $note=SQL_Injection_Attack, + $notice_thresholds=vector(10,100)]); } event http_request(c: connection, method: string, original_URI: string, @@ -46,7 +51,7 @@ event http_request(c: connection, method: string, original_URI: string, { add c$http$tags[URI_SQLI]; - Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1); - Metrics::add_data(SQL_ATTACK_AGAINST, [$host=c$id$resp_h], 1); + Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h]); + Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h]); } } \ No newline at end of file