diff --git a/scripts/base/frameworks/metrics/main.bro b/scripts/base/frameworks/metrics/main.bro index 6b587a0939..48f11ef7f7 100644 --- a/scripts/base/frameworks/metrics/main.bro +++ b/scripts/base/frameworks/metrics/main.bro @@ -135,7 +135,7 @@ export { measure: set[Calculation] &optional; ## A predicate so that you can decide per index if you would like ## to accept the data being inserted. - pred: function(index: Metrics::Index, data: DataPoint): bool &optional; + pred: function(index: Metrics::Index, data: Metrics::DataPoint): bool &optional; ## A function to normalize the index. This can be used to aggregate or ## normalize the entire index. normalize_func: function(index: Metrics::Index): Index &optional; diff --git a/scripts/policy/frameworks/metrics/conn-example.bro b/scripts/policy/frameworks/metrics/conn-example.bro index 00c82f840d..e5c604a5b2 100644 --- a/scripts/policy/frameworks/metrics/conn-example.bro +++ b/scripts/policy/frameworks/metrics/conn-example.bro @@ -16,6 +16,6 @@ event bro_init() event connection_established(c: connection) { - Metrics::add_data("conns.originated", [$host=c$id$orig_h], 1); - Metrics::add_data("conns.responded", [$host=c$id$resp_h], 1); + Metrics::add_data("conns.originated", [$host=c$id$orig_h], [$num=1]); + Metrics::add_data("conns.responded", [$host=c$id$resp_h], [$num=1]); } diff --git a/scripts/policy/frameworks/metrics/http-example.bro b/scripts/policy/frameworks/metrics/http-example.bro index 58ca4e6614..3c60f3c931 100644 --- a/scripts/policy/frameworks/metrics/http-example.bro +++ b/scripts/policy/frameworks/metrics/http-example.bro @@ -6,15 +6,6 @@ @load base/protocols/http @load base/utils/site -redef enum Metrics::ID += { - ## Measures HTTP requests indexed on both the request host and the response - ## code from the server. - HTTP_REQUESTS_BY_STATUS_CODE, - - ## Currently unfinished and not working. - HTTP_REQUESTS_BY_HOST_HEADER, -}; - event bro_init() { # TODO: these are waiting on a fix with table vals + records before they will work. @@ -24,14 +15,14 @@ event bro_init() # $break_interval=1min]); # Site::local_nets must be defined in order for this to actually do anything. - Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table, - $break_interval=1min]); + Metrics::add_filter("http.request.by_status_code", [$aggregation_table=Site::local_nets_table, + $break_interval=1min]); } event HTTP::log_http(rec: HTTP::Info) { if ( rec?$host ) - Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host], 1); + Metrics::add_data("http.request.by_host_header", [$str=rec$host], [$num=1]); if ( rec?$status_code ) - Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], 1); + Metrics::add_data("http.request.by_status_code", [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], [$num=1]); } diff --git a/scripts/policy/frameworks/metrics/ssl-example.bro b/scripts/policy/frameworks/metrics/ssl-example.bro index 5ec675779a..64e63bc215 100644 --- a/scripts/policy/frameworks/metrics/ssl-example.bro +++ b/scripts/policy/frameworks/metrics/ssl-example.bro @@ -6,15 +6,11 @@ @load base/frameworks/metrics @load base/protocols/ssl -redef enum Metrics::ID += { - SSL_SERVERNAME, -}; - event bro_init() { - Metrics::add_filter(SSL_SERVERNAME, + Metrics::add_filter("ssl.by_servername", [$name="no-google-ssl-servers", - $pred(index: Metrics::Index) = { + $pred(index: Metrics::Index, data: Metrics::DataPoint) = { return (/google\.com$/ !in index$str); }, $break_interval=10secs @@ -24,5 +20,5 @@ event bro_init() event SSL::log_ssl(rec: SSL::Info) { if ( rec?$server_name ) - Metrics::add_data(SSL_SERVERNAME, [$str=rec$server_name], 1); + Metrics::add_data("ssl.by_servername", [$str=rec$server_name], [$num=1]); } diff --git a/scripts/policy/protocols/conn/conn-stats-per-host.bro b/scripts/policy/protocols/conn/conn-stats-per-host.bro index 9e532b8590..df58081163 100644 --- a/scripts/policy/protocols/conn/conn-stats-per-host.bro +++ b/scripts/policy/protocols/conn/conn-stats-per-host.bro @@ -1,4 +1,7 @@ +@load base/protocols/conn +@load base/frameworks/metrics + event bro_init() &priority=5 { Metrics::add_filter("conn.orig.data", diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro index 910ae4aa6e..0fb5fa2134 100644 --- a/scripts/policy/protocols/conn/metrics.bro +++ b/scripts/policy/protocols/conn/metrics.bro @@ -1,9 +1,10 @@ @load base/frameworks/metrics +@load base/utils/site event bro_init() &priority=3 { - Metrics::add_filter("conns.country", [$break_interval=1hr]); - Metrics::add_filter("hosts.active", [$break_interval=1hr]); + Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM)]); + Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM)]); } event connection_established(c: connection) &priority=3 @@ -12,10 +13,10 @@ event connection_established(c: connection) &priority=3 { local loc = lookup_location(c$id$resp_h); if ( loc?$country_code ) - Metrics::add_data("conns.country", [$str=loc$country_code], 1); + Metrics::add_data("conns.country", [$str=loc$country_code], [$num=1]); } local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h; # There is no index for this. - Metrics::add_unique("hosts.active", [], cat(the_host)); + Metrics::add_data("hosts.active", [], [$str=cat(the_host)]); } diff --git a/scripts/policy/protocols/conn/scan.bro b/scripts/policy/protocols/conn/scan.bro index 8795cfda06..503b8c34b4 100644 --- a/scripts/policy/protocols/conn/scan.bro +++ b/scripts/policy/protocols/conn/scan.bro @@ -4,10 +4,12 @@ ##! Seth Hall ##! All the authors of the old scan.bro +@load base/frameworks/notice +@load base/frameworks/metrics + module Scan; export { - redef enum Notice::Type += { AddressScan, PortScan, diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro index c3d1bb1e20..ac803ac621 100644 --- a/scripts/policy/protocols/smtp/metrics.bro +++ b/scripts/policy/protocols/smtp/metrics.bro @@ -2,31 +2,36 @@ ##! "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?" ##! "How much mail is being sent from each local mail server per hour?" +@load base/protocols/smtp @load base/frameworks/metrics +@load base/utils/site +@load base/utils/directions-and-hosts module SMTPMetrics; export { ## Define the break intervals for all of the metrics collected and logged by this script. - const breaks = 1hr &redef; + const breaks=1hr &redef; } event bro_init() &priority=5 { - Metrics::add_filter("smtp.mailfrom", [$pred(index: Metrics::Index) = { - return addr_matches_host(index$host, LOCAL_HOSTS); }, - $break_interval=breaks]); - Metrics::add_filter("smtp.messages", [$pred(index: Metrics::Index) = { - return addr_matches_host(index$host, LOCAL_HOSTS); }, - $break_interval=breaks]); + Metrics::add_filter("smtp.mailfrom", [$every=breaks, + $measure=set(Metrics::SUM), + $pred(index: Metrics::Index, data: Metrics::DataPoint) = { + return addr_matches_host(index$host, LOCAL_HOSTS); + }]); + Metrics::add_filter("smtp.messages", [$every=breaks, + $measure=set(Metrics::SUM), + $pred(index: Metrics::Index, data: Metrics::DataPoint) = { + return addr_matches_host(index$host, LOCAL_HOSTS); + }]); } event SMTP::log_smtp(rec: SMTP::Info) { - Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], 1); + Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], [$num=1]); if ( rec?$mailfrom ) - Metrics::add_unique("smtp.mailfrom", [$host=rec$id$orig_h], rec$mailfrom); + Metrics::add_data("smtp.mailfrom", [$host=rec$id$orig_h], [$str=rec$mailfrom]); } - - diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 7fd3a1bdc8..02b7e51030 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2012-11-05-23-29-45 +#open 2012-11-20-06-11-08 #fields name #types string scripts/base/init-bare.bro @@ -38,6 +38,7 @@ scripts/base/init-default.bro scripts/base/utils/files.bro scripts/base/utils/numbers.bro scripts/base/utils/paths.bro + scripts/base/utils/queue.bro scripts/base/utils/strings.bro scripts/base/utils/thresholds.bro scripts/base/utils/urls.bro @@ -118,4 +119,4 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/./main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2012-11-05-23-29-45 +#close 2012-11-20-06-11-08 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log index cb1bd5af01..e6c33719aa 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log @@ -3,10 +3,10 @@ #empty_field (empty) #unset_field - #path metrics -#open 2012-07-20-01-50-41 -#fields ts metric_id filter_name index.host index.str index.network value -#types time enum string addr string subnet count -1342749041.601712 TEST_METRIC foo-bar 6.5.4.3 - - 4 -1342749041.601712 TEST_METRIC foo-bar 7.2.1.5 - - 2 -1342749041.601712 TEST_METRIC foo-bar 1.2.3.4 - - 6 -#close 2012-07-20-01-50-49 +#open 2012-11-20-06-46-51 +#fields ts ts_delta filter_name metric index.str index.host index.network result.begin result.num result.sum result.min result.max result.avg result.variance result.std_dev result.unique +#types time interval string string string addr subnet time count double double double double double double count +1353394011.192622 3.000000 default test.metric - 6.5.4.3 - - 2 6.0 1.0 5.0 3.0 4.0 2.0 - +1353394011.192622 3.000000 default test.metric - 1.2.3.4 - - 9 437.0 3.0 95.0 48.555556 674.469136 25.970544 - +1353394011.192622 3.000000 default test.metric - 7.2.1.5 - - 2 145.0 54.0 91.0 72.5 342.25 18.5 - +#close 2012-11-20-06-46-51 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 6c93cb875e..a5e28de7f9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-51-18 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749078.270791 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-07-20-01-51-27 +#open 2012-11-20-06-46-22 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double +1353393982.260495 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - +#close 2012-11-20-06-46-22 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 88f25b066f..d657cf0ce8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-51-36 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749096.545663 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-07-20-01-51-45 +#open 2012-11-20-06-45-52 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double +1353393952.489496 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - +#close 2012-11-20-06-45-56 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log index f9292344a8..92206c35ce 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-10-05-21-45-15 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-10-05-21-45-15 +#open 2012-11-20-06-09-07 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double +1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - +#close 2012-11-20-06-09-07