mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
SumStats test checkpoint.
This commit is contained in:
Seth Hall
parent
437815454d
commit
1cac89e4f8
16 changed files with 55 additions and 116 deletions
|
|
@ -46,17 +46,6 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
|
|||
rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
|
||||
rest_target(${psd} base/frameworks/logging/writers/none.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/main.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/non-cluster.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/average.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/max.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/min.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/sample.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/std-dev.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/sum.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/unique.bro)
|
||||
rest_target(${psd} base/frameworks/measurement/plugins/variance.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/drop.bro)
|
||||
rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
|
||||
|
|
@ -72,6 +61,17 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
|
|||
rest_target(${psd} base/frameworks/reporter/main.bro)
|
||||
rest_target(${psd} base/frameworks/signatures/main.bro)
|
||||
rest_target(${psd} base/frameworks/software/main.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/cluster.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/main.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
|
||||
rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
|
||||
rest_target(${psd} base/frameworks/tunnels/main.bro)
|
||||
rest_target(${psd} base/misc/find-checksum-offloading.bro)
|
||||
rest_target(${psd} base/protocols/conn/contents.bro)
|
||||
|
|
@ -147,7 +147,6 @@ rest_target(${psd} policy/misc/stats.bro)
|
|||
rest_target(${psd} policy/misc/trim-trace-file.bro)
|
||||
rest_target(${psd} policy/protocols/conn/known-hosts.bro)
|
||||
rest_target(${psd} policy/protocols/conn/known-services.bro)
|
||||
rest_target(${psd} policy/protocols/conn/metrics.bro)
|
||||
rest_target(${psd} policy/protocols/conn/weirds.bro)
|
||||
rest_target(${psd} policy/protocols/dns/auth-addl.bro)
|
||||
rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
|
||||
|
|
@ -166,7 +165,6 @@ rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
|
|||
rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/blocklists.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/metrics.bro)
|
||||
rest_target(${psd} policy/protocols/smtp/software.bro)
|
||||
rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
|
||||
rest_target(${psd} policy/protocols/ssh/geo-data.bro)
|
||||
|
|
|
|||
|
|
@ -34,15 +34,15 @@ export {
|
|||
const enable_intermediate_updates = T &redef;
|
||||
|
||||
## Event sent by the manager in a cluster to initiate the
|
||||
## collection of metrics values for a measurement.
|
||||
## collection of metrics values for a sumstat.
|
||||
global cluster_ss_request: event(uid: string, ssid: string);
|
||||
|
||||
## Event sent by nodes that are collecting metrics after receiving
|
||||
## a request for the metric measurement from the manager.
|
||||
## a request for the metric sumstat from the manager.
|
||||
global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
|
||||
|
||||
## This event is sent by the manager in a cluster to initiate the
|
||||
## collection of a single key value from a measurement. It's typically
|
||||
## collection of a single key value from a sumstat. It's typically
|
||||
## used to get intermediate updates before the break interval triggers
|
||||
## to speed detection of a value crossing a threshold.
|
||||
global cluster_key_request: event(uid: string, ssid: string, key: Key);
|
||||
|
|
@ -130,13 +130,13 @@ event SumStats::cluster_ss_request(uid: string, ssid: string)
|
|||
{
|
||||
#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
|
||||
|
||||
# Initiate sending all of the data for the requested measurement.
|
||||
# Initiate sending all of the data for the requested stats.
|
||||
if ( ssid in result_store )
|
||||
event SumStats::send_data(uid, ssid, result_store[ssid]);
|
||||
else
|
||||
event SumStats::send_data(uid, ssid, table());
|
||||
|
||||
# Lookup the actual measurement and reset it, the reference to the data
|
||||
# Lookup the actual sumstats and reset it, the reference to the data
|
||||
# currently stored will be maintained internally by the send_data event.
|
||||
if ( ssid in stats_store )
|
||||
reset(stats_store[ssid]);
|
||||
|
|
@ -181,9 +181,9 @@ global done_with: table[string] of count &read_expire=1min &default=0;
|
|||
global key_requests: table[string] of Result &read_expire=1min;
|
||||
|
||||
# This variable is maintained by managers to prevent overwhelming communication due
|
||||
# to too many intermediate updates. Each measurement is tracked separately so that
|
||||
# one won't overwhelm and degrade other quieter measurements.
|
||||
# Indexed on a measurement id.
|
||||
# to too many intermediate updates. Each sumstat is tracked separately so that
|
||||
# one won't overwhelm and degrade other quieter sumstats.
|
||||
# Indexed on a sumstat id.
|
||||
global outstanding_global_views: table[string] of count &default=0;
|
||||
|
||||
const zero_time = double_to_time(0.0);
|
||||
|
|
@ -192,7 +192,7 @@ event SumStats::finish_epoch(ss: SumStat)
|
|||
{
|
||||
if ( network_time() > zero_time )
|
||||
{
|
||||
#print fmt("%.6f MANAGER: breaking %s measurement for %s metric", network_time(), measurement$name, measurement$id);
|
||||
#print fmt("%.6f MANAGER: breaking %s sumstat for %s metric", network_time(), ss$name, ss$id);
|
||||
local uid = unique_id("");
|
||||
|
||||
if ( uid in stats_results )
|
||||
|
|
@ -207,8 +207,8 @@ event SumStats::finish_epoch(ss: SumStat)
|
|||
schedule ss$epoch { SumStats::finish_epoch(ss) };
|
||||
}
|
||||
|
||||
# This is unlikely to be called often, but it's here in case there are measurements
|
||||
# being collected by managers.
|
||||
# This is unlikely to be called often, but it's here in
|
||||
# case there are sumstats being collected by managers.
|
||||
function data_added(ss: SumStat, key: Key, result: Result)
|
||||
{
|
||||
if ( check_thresholds(ss, key, result, 1.0) )
|
||||
|
|
@ -305,7 +305,7 @@ event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable
|
|||
# Clean up
|
||||
delete stats_results[uid];
|
||||
delete done_with[uid];
|
||||
# Not sure I need to reset the measurement on the manager.
|
||||
# Not sure I need to reset the sumstat on the manager.
|
||||
reset(ss);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export {
|
|||
};
|
||||
}
|
||||
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( AVERAGE in r$apply )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export {
|
|||
};
|
||||
}
|
||||
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( MAX in r$apply )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ export {
|
|||
};
|
||||
}
|
||||
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( MIN in r$apply )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -29,13 +29,13 @@ function get_samples(rv: ResultVal): vector of Observation
|
|||
return s;
|
||||
}
|
||||
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( r$samples > 0 )
|
||||
{
|
||||
if ( ! rv?$samples )
|
||||
rv$samples = Queue::init([$max_len=r$samples]);
|
||||
Queue::put(rv$samples, data);
|
||||
Queue::put(rv$samples, obs);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,13 +22,10 @@ function calc_std_dev(rv: ResultVal)
|
|||
}
|
||||
|
||||
# This depends on the variance plugin which uses priority -5
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-10
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
|
||||
{
|
||||
if ( STD_DEV in r$apply )
|
||||
{
|
||||
if ( rv?$variance )
|
||||
calc_std_dev(rv);
|
||||
}
|
||||
calc_std_dev(rv);
|
||||
}
|
||||
|
||||
hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ hook init_resultval_hook(r: Reducer, rv: ResultVal)
|
|||
rv$sum = 0;
|
||||
}
|
||||
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( SUM in r$apply )
|
||||
rv$sum += val;
|
||||
|
|
|
|||
|
|
@ -23,13 +23,13 @@ redef record ResultVal += {
|
|||
unique_vals: set[Observation] &optional;
|
||||
};
|
||||
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal)
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
|
||||
{
|
||||
if ( UNIQUE in r$apply )
|
||||
{
|
||||
if ( ! rv?$unique_vals )
|
||||
rv$unique_vals=set();
|
||||
add rv$unique_vals[data];
|
||||
add rv$unique_vals[obs];
|
||||
rv$unique = |rv$unique_vals|;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ function calc_variance(rv: ResultVal)
|
|||
}
|
||||
|
||||
# Reduced priority since this depends on the average
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, data: Observation, rv: ResultVal) &priority=-5
|
||||
hook add_to_reducer_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
|
||||
{
|
||||
if ( VARIANCE in r$apply )
|
||||
{
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ redef record connection += {
|
|||
|
||||
event bro_init() &priority=3
|
||||
{
|
||||
Log::create_stream(AppSumStats::LOG, [$columns=Info]);
|
||||
Log::create_stream(AppStats::LOG, [$columns=Info]);
|
||||
|
||||
local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
|
||||
local r2: SumStats::Reducer = [$stream="apps.hits", $apply=set(SumStats::UNIQUE)];
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
@load base/frameworks/measurement
|
||||
@load base/utils/site
|
||||
|
||||
event bro_init() &priority=3
|
||||
{
|
||||
Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM),
|
||||
$period_finished=Metrics::write_log]);
|
||||
Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM),
|
||||
$period_finished=Metrics::write_log]);
|
||||
}
|
||||
|
||||
event connection_established(c: connection) &priority=3
|
||||
{
|
||||
if ( Site::is_local_addr(c$id$orig_h) )
|
||||
{
|
||||
local loc = lookup_location(c$id$resp_h);
|
||||
if ( loc?$country_code )
|
||||
Metrics::add_data("conns.country", [$str=loc$country_code], [$num=1]);
|
||||
}
|
||||
|
||||
local the_host = Site::is_local_addr(c$id$orig_h) ? c$id$orig_h : c$id$resp_h;
|
||||
# There is no index for this.
|
||||
Metrics::add_data("hosts.active", [], [$str=cat(the_host)]);
|
||||
}
|
||||
|
|
@ -1,37 +0,0 @@
|
|||
##! This script is meant to answer the following questions...
|
||||
##! "How many unique 'MAIL FROM' addresses are being used by local mail servers per hour?"
|
||||
##! "How much mail is being sent from each local mail server per hour?"
|
||||
|
||||
@load base/protocols/smtp
|
||||
@load base/frameworks/measurement
|
||||
@load base/utils/site
|
||||
@load base/utils/directions-and-hosts
|
||||
|
||||
module SMTPMetrics;
|
||||
|
||||
export {
|
||||
## Define the break intervals for all of the metrics collected and logged by this script.
|
||||
const breaks=1hr &redef;
|
||||
}
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Metrics::add_filter("smtp.mailfrom", [$every=breaks,
|
||||
$measure=set(Metrics::SUM),
|
||||
$pred(index: Metrics::Index, data: Metrics::Observation) = {
|
||||
return addr_matches_host(index$host, LOCAL_HOSTS);
|
||||
}]);
|
||||
Metrics::add_filter("smtp.messages", [$every=breaks,
|
||||
$measure=set(Metrics::SUM),
|
||||
$pred(index: Metrics::Index, data: Metrics::Observation) = {
|
||||
return addr_matches_host(index$host, LOCAL_HOSTS);
|
||||
}]);
|
||||
}
|
||||
|
||||
event SMTP::log_smtp(rec: SMTP::Info)
|
||||
{
|
||||
Metrics::add_data("smtp.messages", [$host=rec$id$orig_h], [$num=1]);
|
||||
|
||||
if ( rec?$mailfrom )
|
||||
Metrics::add_data("smtp.mailfrom", [$host=rec$id$orig_h], [$str=rec$mailfrom]);
|
||||
}
|
||||
|
|
@ -32,9 +32,13 @@
|
|||
@load integration/collective-intel/__load__.bro
|
||||
@load integration/collective-intel/main.bro
|
||||
@load misc/analysis-groups.bro
|
||||
@load misc/app-metrics.bro
|
||||
@load misc/capture-loss.bro
|
||||
@load misc/detect-traceroute/__load__.bro
|
||||
@load misc/detect-traceroute/main.bro
|
||||
@load misc/loaded-scripts.bro
|
||||
@load misc/profiling.bro
|
||||
@load misc/scan.bro
|
||||
@load misc/stats.bro
|
||||
@load misc/trim-trace-file.bro
|
||||
@load protocols/conn/known-hosts.bro
|
||||
|
|
@ -42,6 +46,7 @@
|
|||
@load protocols/conn/weirds.bro
|
||||
@load protocols/dns/auth-addl.bro
|
||||
@load protocols/dns/detect-external-names.bro
|
||||
@load protocols/ftp/detect-bruteforcing.bro
|
||||
@load protocols/ftp/detect.bro
|
||||
@load protocols/ftp/software.bro
|
||||
@load protocols/http/detect-MHR.bro
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path loaded_scripts
|
||||
#open 2013-04-02-04-22-32
|
||||
#open 2013-04-16-03-43-22
|
||||
#fields name
|
||||
#types string
|
||||
scripts/base/init-bare.bro
|
||||
|
|
@ -68,23 +68,23 @@ scripts/base/init-default.bro
|
|||
scripts/base/frameworks/software/main.bro
|
||||
scripts/base/frameworks/communication/__load__.bro
|
||||
scripts/base/frameworks/communication/main.bro
|
||||
scripts/base/frameworks/measurement/__load__.bro
|
||||
scripts/base/frameworks/measurement/main.bro
|
||||
scripts/base/frameworks/measurement/plugins/__load__.bro
|
||||
scripts/base/frameworks/measurement/plugins/average.bro
|
||||
scripts/base/frameworks/measurement/plugins/max.bro
|
||||
scripts/base/frameworks/measurement/plugins/min.bro
|
||||
scripts/base/frameworks/measurement/plugins/sample.bro
|
||||
scripts/base/frameworks/measurement/plugins/std-dev.bro
|
||||
scripts/base/frameworks/measurement/plugins/variance.bro
|
||||
scripts/base/frameworks/measurement/plugins/sum.bro
|
||||
scripts/base/frameworks/measurement/plugins/unique.bro
|
||||
scripts/base/frameworks/measurement/non-cluster.bro
|
||||
scripts/base/frameworks/intel/__load__.bro
|
||||
scripts/base/frameworks/intel/main.bro
|
||||
scripts/base/frameworks/intel/input.bro
|
||||
scripts/base/frameworks/reporter/__load__.bro
|
||||
scripts/base/frameworks/reporter/main.bro
|
||||
scripts/base/frameworks/sumstats/__load__.bro
|
||||
scripts/base/frameworks/sumstats/main.bro
|
||||
scripts/base/frameworks/sumstats/plugins/__load__.bro
|
||||
scripts/base/frameworks/sumstats/plugins/average.bro
|
||||
scripts/base/frameworks/sumstats/plugins/max.bro
|
||||
scripts/base/frameworks/sumstats/plugins/min.bro
|
||||
scripts/base/frameworks/sumstats/plugins/sample.bro
|
||||
scripts/base/frameworks/sumstats/plugins/variance.bro
|
||||
scripts/base/frameworks/sumstats/plugins/std-dev.bro
|
||||
scripts/base/frameworks/sumstats/plugins/sum.bro
|
||||
scripts/base/frameworks/sumstats/plugins/unique.bro
|
||||
scripts/base/frameworks/sumstats/non-cluster.bro
|
||||
scripts/base/frameworks/tunnels/__load__.bro
|
||||
scripts/base/frameworks/tunnels/main.bro
|
||||
scripts/base/protocols/conn/__load__.bro
|
||||
|
|
@ -130,4 +130,4 @@ scripts/base/init-default.bro
|
|||
scripts/base/protocols/syslog/main.bro
|
||||
scripts/base/misc/find-checksum-offloading.bro
|
||||
scripts/policy/misc/loaded-scripts.bro
|
||||
#close 2013-04-02-04-22-32
|
||||
#close 2013-04-16-03-43-22
|
||||
|
|
|
|||
|
|
@ -3,5 +3,5 @@
|
|||
-./frameworks/cluster/nodes/worker.bro
|
||||
-./frameworks/cluster/setup-connections.bro
|
||||
-./frameworks/intel/cluster.bro
|
||||
-./frameworks/measurement/cluster.bro
|
||||
-./frameworks/notice/cluster.bro
|
||||
-./frameworks/sumstats/cluster.bro
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue