diff --git a/DocSourcesList.cmake b/DocSourcesList.cmake index 5ac87a6305..a9ba4838db 100644 --- a/DocSourcesList.cmake +++ b/DocSourcesList.cmake @@ -46,17 +46,6 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro) rest_target(${psd} base/frameworks/logging/writers/dataseries.bro) rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro) rest_target(${psd} base/frameworks/logging/writers/none.bro) -rest_target(${psd} base/frameworks/measurement/cluster.bro) -rest_target(${psd} base/frameworks/measurement/main.bro) -rest_target(${psd} base/frameworks/measurement/non-cluster.bro) -rest_target(${psd} base/frameworks/measurement/plugins/average.bro) -rest_target(${psd} base/frameworks/measurement/plugins/max.bro) -rest_target(${psd} base/frameworks/measurement/plugins/min.bro) -rest_target(${psd} base/frameworks/measurement/plugins/sample.bro) -rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro) -rest_target(${psd} base/frameworks/measurement/plugins/sum.bro) -rest_target(${psd} base/frameworks/measurement/plugins/unique.bro) -rest_target(${psd} base/frameworks/measurement/plugins/variance.bro) rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro) rest_target(${psd} base/frameworks/notice/actions/drop.bro) rest_target(${psd} base/frameworks/notice/actions/email_admin.bro) @@ -72,6 +61,17 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro) rest_target(${psd} base/frameworks/reporter/main.bro) rest_target(${psd} base/frameworks/signatures/main.bro) rest_target(${psd} base/frameworks/software/main.bro) +rest_target(${psd} base/frameworks/sumstats/cluster.bro) +rest_target(${psd} base/frameworks/sumstats/main.bro) +rest_target(${psd} base/frameworks/sumstats/non-cluster.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/average.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/max.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/min.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro) +rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro) rest_target(${psd} base/frameworks/tunnels/main.bro) rest_target(${psd} base/misc/find-checksum-offloading.bro) rest_target(${psd} base/protocols/conn/contents.bro) @@ -147,7 +147,6 @@ rest_target(${psd} policy/misc/stats.bro) rest_target(${psd} policy/misc/trim-trace-file.bro) rest_target(${psd} policy/protocols/conn/known-hosts.bro) rest_target(${psd} policy/protocols/conn/known-services.bro) -rest_target(${psd} policy/protocols/conn/metrics.bro) rest_target(${psd} policy/protocols/conn/weirds.bro) rest_target(${psd} policy/protocols/dns/auth-addl.bro) rest_target(${psd} policy/protocols/dns/detect-external-names.bro) @@ -166,7 +165,6 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro) rest_target(${psd} policy/protocols/modbus/track-memmap.bro) rest_target(${psd} policy/protocols/smtp/blocklists.bro) rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro) -rest_target(${psd} policy/protocols/smtp/metrics.bro) rest_target(${psd} policy/protocols/smtp/software.bro) rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro) rest_target(${psd} policy/protocols/ssh/geo-data.bro) diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro index ee763c1d9d..4f9743547b 100644 --- a/scripts/base/frameworks/sumstats/cluster.bro +++ b/scripts/base/frameworks/sumstats/cluster.bro @@ -34,15 +34,15 @@ export { const enable_intermediate_updates = T &redef; ## Event sent by the manager in a cluster to initiate the - ## collection of metrics values for a measurement. + ## collection of metrics values for a sumstat. global cluster_ss_request: event(uid: string, ssid: string); ## Event sent by nodes that are collecting metrics after receiving - ## a request for the metric measurement from the manager. + ## a request for the metric sumstat from the manager. global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool); ## This event is sent by the manager in a cluster to initiate the - ## collection of a single key value from a measurement. It's typically + ## collection of a single key value from a sumstat. It's typically ## used to get intermediate updates before the break interval triggers ## to speed detection of a value crossing a threshold. global cluster_key_request: event(uid: string, ssid: string, key: Key); @@ -130,13 +130,13 @@ event SumStats::cluster_ss_request(uid: string, ssid: string) { #print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id); - # Initiate sending all of the data for the requested measurement. + # Initiate sending all of the data for the requested stats. if ( ssid in result_store ) event SumStats::send_data(uid, ssid, result_store[ssid]); else event SumStats::send_data(uid, ssid, table()); - # Lookup the actual measurement and reset it, the reference to the data + # Lookup the actual sumstats and reset it, the reference to the data # currently stored will be maintained internally by the send_data event. if ( ssid in stats_store ) reset(stats_store[ssid]); @@ -181,9 +181,9 @@ global done_with: table[string] of count &read_expire=1min &default=0; global key_requests: table[string] of Result &read_expire=1min; # This variable is maintained by managers to prevent overwhelming communication due -# to too many intermediate updates. Each measurement is tracked separately so that -# one won't overwhelm and degrade other quieter measurements. -# Indexed on a measurement id. +# to too many intermediate updates. Each sumstat is tracked separately so that +# one won't overwhelm and degrade other quieter sumstats. +# Indexed on a sumstat id. global outstanding_global_views: table[string] of count &default=0; const zero_time = double_to_time(0.0); @@ -192,7 +192,7 @@ event SumStats::finish_epoch(ss: SumStat) { if ( network_time() > zero_time ) { - #print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id); + #print fmt("%.6f MANAGER: breaking %s sumstat for %s metric", network_time(), ss$name, ss$id); local uid = unique_id(""); if ( uid in stats_results ) @@ -207,8 +207,8 @@ event SumStats::finish_epoch(ss: SumStat) schedule ss$epoch { SumStats::finish_epoch(ss) }; } -# This is unlikely to be called often, but it's here in case there are measurements -# being collected by managers. +# This is unlikely to be called often, but it's here in +# case there are sumstats being collected by managers. function data_added(ss: SumStat, key: Key, result: Result) { if ( check_thresholds(ss, key, result, 1.0) ) @@ -305,7 +305,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable # Clean up delete stats_results[uid]; delete done_with[uid]; - # Not sure I need to reset the measurement on the manager. + # Not sure I need to reset the sumstat on the manager. reset(ss); } } diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro index 002a0147ff..35228a28f5 100644 --- a/scripts/base/frameworks/sumstats/plugins/average.bro +++ b/scripts/base/frameworks/sumstats/plugins/average.bro @@ -14,7 +14,7 @@ export { }; } -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { if ( AVERAGE in r$apply ) { diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro index 0e377ff320..0a959f2d09 100644 --- a/scripts/base/frameworks/sumstats/plugins/max.bro +++ b/scripts/base/frameworks/sumstats/plugins/max.bro @@ -14,7 +14,7 @@ export { }; } -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { if ( MAX in r$apply ) { diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro index 5e1e3fbbb7..16c2dfc3d7 100644 --- a/scripts/base/frameworks/sumstats/plugins/min.bro +++ b/scripts/base/frameworks/sumstats/plugins/min.bro @@ -14,7 +14,7 @@ export { }; } -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { if ( MIN in r$apply ) { diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index a694296727..622b160cbe 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -29,13 +29,13 @@ function get_samples(rv: ResultVal): vector of Observation return s; } -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { if ( r$samples > 0 ) { if ( ! rv?$samples ) rv$samples = Queue::init([$max_len=r$samples]); - Queue::put(rv$samples, data); + Queue::put(rv$samples, obs); } } diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro index af6eea8cdc..7c2754570a 100644 --- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro +++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro @@ -22,13 +22,10 @@ function calc_std_dev(rv: ResultVal) } # This depends on the variance plugin which uses priority -5 -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-10 +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10 { if ( STD_DEV in r$apply ) - { - if ( rv?$variance ) - calc_std_dev(rv); - } + calc_std_dev(rv); } hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10 diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro index 572402d6c5..8c8c65cd61 100644 --- a/scripts/base/frameworks/sumstats/plugins/sum.bro +++ b/scripts/base/frameworks/sumstats/plugins/sum.bro @@ -34,7 +34,7 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal) rv$sum = 0; } -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { if ( SUM in r$apply ) rv$sum += val; diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro index f260148af4..d3a4464d0d 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/unique.bro @@ -23,13 +23,13 @@ redef record ResultVal += { unique_vals: set[Observation] &optional; }; -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { if ( UNIQUE in r$apply ) { if ( ! rv?$unique_vals ) rv$unique_vals=set(); - add rv$unique_vals[data]; + add rv$unique_vals[obs]; rv$unique = |rv$unique_vals|; } } diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro index a26a2d4095..29118b284b 100644 --- a/scripts/base/frameworks/sumstats/plugins/variance.bro +++ b/scripts/base/frameworks/sumstats/plugins/variance.bro @@ -29,7 +29,7 @@ function calc_variance(rv: ResultVal) } # Reduced priority since this depends on the average -hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-5 +hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5 { if ( VARIANCE in r$apply ) { diff --git a/scripts/policy/misc/app-metrics.bro b/scripts/policy/misc/app-metrics.bro index 53f210b46a..ec2e8f8d48 100644 --- a/scripts/policy/misc/app-metrics.bro +++ b/scripts/policy/misc/app-metrics.bro @@ -32,7 +32,7 @@ redef record connection += { event bro_init() &priority=3 { - Log::create_stream(AppSumStats::LOG, [$columns=Info]); + Log::create_stream(AppStats::LOG, [$columns=Info]); local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)]; local r2: SumStats::Reducer = [$stream="apps.hits", $apply=set(SumStats::UNIQUE)]; diff --git a/scripts/policy/protocols/conn/metrics.bro b/scripts/policy/protocols/conn/metrics.bro deleted file mode 100644 index 62ca96ea0a..0000000000 --- a/scripts/policy/protocols/conn/metrics.bro +++ /dev/null @@ -1,24 +0,0 @@ -@load base/frameworks/measurement -@load base/utils/site - -event bro_init() &priority=3 - { - Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM), - $period_finished=Metrics::write_log]); - Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM), - $period_finished=Metrics::write_log]); - } - -event connection_established(c: connection) &priority=3 - { - if ( Site::is_local_addr(c$id$orig_h) ) - { - local loc = lookup_location(c$id$resp_h); - if ( loc?$country_code ) - Metrics::add_data("conns.country", [$str=loc$country_code], [$num=1]); - } - - local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h; - # There is no index for this. - Metrics::add_data("hosts.active", [], [$str=cat(the_host)]); - } diff --git a/scripts/policy/protocols/smtp/metrics.bro b/scripts/policy/protocols/smtp/metrics.bro deleted file mode 100644 index 04e1185e25..0000000000 --- a/scripts/policy/protocols/smtp/metrics.bro +++ /dev/null @@ -1,37 +0,0 @@ -##! This script is meant to answer the following questions... -##! "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?" -##! "How much mail is being sent from each local mail server per hour?" - -@load base/protocols/smtp -@load base/frameworks/measurement -@load base/utils/site -@load base/utils/directions-and-hosts - -module SMTPMetrics; - -export { - ## Define the break intervals for all of the metrics collected and logged by this script. - const breaks=1hr &redef; -} - -event bro_init() &priority=5 - { - Metrics::add_filter("smtp.mailfrom", [$every=breaks, - $measure=set(Metrics::SUM), - $pred(index: Metrics::Index, data: Metrics::Observation) = { - return addr_matches_host(index$host, LOCAL_HOSTS); - }]); - Metrics::add_filter("smtp.messages", [$every=breaks, - $measure=set(Metrics::SUM), - $pred(index: Metrics::Index, data: Metrics::Observation) = { - return addr_matches_host(index$host, LOCAL_HOSTS); - }]); - } - -event SMTP::log_smtp(rec: SMTP::Info) - { - Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], [$num=1]); - - if ( rec?$mailfrom ) - Metrics::add_data("smtp.mailfrom", [$host=rec$id$orig_h], [$str=rec$mailfrom]); - } diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 2fe32a4788..35d9f89de9 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -32,9 +32,13 @@ @load integration/collective-intel/__load__.bro @load integration/collective-intel/main.bro @load misc/analysis-groups.bro +@load misc/app-metrics.bro @load misc/capture-loss.bro +@load misc/detect-traceroute/__load__.bro +@load misc/detect-traceroute/main.bro @load misc/loaded-scripts.bro @load misc/profiling.bro +@load misc/scan.bro @load misc/stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @@ -42,6 +46,7 @@ @load protocols/conn/weirds.bro @load protocols/dns/auth-addl.bro @load protocols/dns/detect-external-names.bro +@load protocols/ftp/detect-bruteforcing.bro @load protocols/ftp/detect.bro @load protocols/ftp/software.bro @load protocols/http/detect-MHR.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index e691a906c2..a34f4dd3fc 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-04-02-04-22-32 +#open 2013-04-16-03-43-22 #fields name #types string scripts/base/init-bare.bro @@ -68,23 +68,23 @@ scripts/base/init-default.bro scripts/base/frameworks/software/main.bro scripts/base/frameworks/communication/__load__.bro scripts/base/frameworks/communication/main.bro - scripts/base/frameworks/measurement/__load__.bro - scripts/base/frameworks/measurement/main.bro - scripts/base/frameworks/measurement/plugins/__load__.bro - scripts/base/frameworks/measurement/plugins/average.bro - scripts/base/frameworks/measurement/plugins/max.bro - scripts/base/frameworks/measurement/plugins/min.bro - scripts/base/frameworks/measurement/plugins/sample.bro - scripts/base/frameworks/measurement/plugins/std-dev.bro - scripts/base/frameworks/measurement/plugins/variance.bro - scripts/base/frameworks/measurement/plugins/sum.bro - scripts/base/frameworks/measurement/plugins/unique.bro - scripts/base/frameworks/measurement/non-cluster.bro scripts/base/frameworks/intel/__load__.bro scripts/base/frameworks/intel/main.bro scripts/base/frameworks/intel/input.bro scripts/base/frameworks/reporter/__load__.bro scripts/base/frameworks/reporter/main.bro + scripts/base/frameworks/sumstats/__load__.bro + scripts/base/frameworks/sumstats/main.bro + scripts/base/frameworks/sumstats/plugins/__load__.bro + scripts/base/frameworks/sumstats/plugins/average.bro + scripts/base/frameworks/sumstats/plugins/max.bro + scripts/base/frameworks/sumstats/plugins/min.bro + scripts/base/frameworks/sumstats/plugins/sample.bro + scripts/base/frameworks/sumstats/plugins/variance.bro + scripts/base/frameworks/sumstats/plugins/std-dev.bro + scripts/base/frameworks/sumstats/plugins/sum.bro + scripts/base/frameworks/sumstats/plugins/unique.bro + scripts/base/frameworks/sumstats/non-cluster.bro scripts/base/frameworks/tunnels/__load__.bro scripts/base/frameworks/tunnels/main.bro scripts/base/protocols/conn/__load__.bro @@ -130,4 +130,4 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-04-02-04-22-32 +#close 2013-04-16-03-43-22 diff --git a/testing/btest/Baseline/coverage.init-default/missing_loads b/testing/btest/Baseline/coverage.init-default/missing_loads index 554fcf012e..23cae7b694 100644 --- a/testing/btest/Baseline/coverage.init-default/missing_loads +++ b/testing/btest/Baseline/coverage.init-default/missing_loads @@ -3,5 +3,5 @@ -./frameworks/cluster/nodes/worker.bro -./frameworks/cluster/setup-connections.bro -./frameworks/intel/cluster.bro --./frameworks/measurement/cluster.bro -./frameworks/notice/cluster.bro +-./frameworks/sumstats/cluster.bro