Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-12 03:28:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

API updates for metrics framework.

Browse source 
- Removed default logging.  Now a function is available for the new
  $period_finished filter field to get the same behavior for logging
  named Metrics::write_log.

- Added index rollups for getting multiple metrics result values
  as the same time.
This commit is contained in:
Seth Hall 2012-12-18 01:08:59 -05:00
parent 69030fdff3
commit 69b7ce12d2
17 changed files with 304 additions and 162 deletions
Download patch file Download diff file Expand all files Collapse all files

9
scripts/policy/frameworks/metrics/conn-example.bro
View file

@ -7,11 +7,16 @@
event bro_init()
{
#Metrics::add_filter("conns.originated", [$aggregation_mask=24, $break_interval=1mins]);
Metrics::add_filter("conns.originated", [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
Metrics::add_filter("conns.originated", [$every=1mins, $measure=set(Metrics::SUM),
$aggregation_table=Site::local_nets_table,
$period_finished=Metrics::write_log]);
# Site::local_nets must be defined in order for this to actually do anything.
Metrics::add_filter("conns.responded", [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
Metrics::add_filter("conns.responded", [$every=1mins, $measure=set(Metrics::SUM),
$aggregation_table=Site::local_nets_table,
$period_finished=Metrics::write_log]);
}
event connection_established(c: connection)

15
scripts/policy/frameworks/metrics/http-example.bro
View file

@ -8,15 +8,16 @@
event bro_init()
{
# TODO: these are waiting on a fix with table vals + records before they will work.
#Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER,
# [$pred(index: Metrics::Index) = { return Site::is_local_addr(index$host); },
# $aggregation_mask=24,
# $break_interval=1min]);
Metrics::add_filter("http.request.by_host_header",
[$every=1min, $measure=set(Metrics::SUM),
$pred(index: Metrics::Index, data: Metrics::DataPoint) = { return T; return Site::is_local_addr(index$host); },
$aggregation_mask=24,
$period_finished=Metrics::write_log]);
# Site::local_nets must be defined in order for this to actually do anything.
Metrics::add_filter("http.request.by_status_code", [$aggregation_table=Site::local_nets_table,
$break_interval=1min]);
Metrics::add_filter("http.request.by_status_code", [$every=1min, $measure=set(Metrics::SUM),
$aggregation_table=Site::local_nets_table,
$period_finished=Metrics::write_log]);
}
event HTTP::log_http(rec: HTTP::Info)

5
scripts/policy/frameworks/metrics/ssl-example.bro
View file

@ -10,11 +10,10 @@ event bro_init()
{
Metrics::add_filter("ssl.by_servername",
[$name="no-google-ssl-servers",
$every=10secs, $measure=set(Metrics::SUM),
$pred(index: Metrics::Index, data: Metrics::DataPoint) = {
return (/google\.com$/ !in index$str);
},
$break_interval=10secs
]);
}]);
}
event SSL::log_ssl(rec: SSL::Info)

55
scripts/policy/misc/app-metrics.bro
View file

@ -1,49 +1,80 @@
@load base/protocols/http
@load base/protocols/ssl
@load base/frameworks/metrics
module AppMetrics;
export {
## The metric break interval for the default stats collected by this script.
const break_interval = 1hr &redef;
redef enum Log::ID += { LOG };
type Info: record {
ts: time &log;
app: string &log;
uniq_hosts: count &log;
hits: count &log;
bytes: count &log;
};
## The frequency of logging the stats collected by this script.
const break_interval = 1min &redef;
}
function app_metrics_rollup(index: Metrics::Index, vals: table[string, string] of Metrics::ResultVal)
{
local l: Info;
l$ts = network_time();
for ( [metric_name, filter_name] in vals )
{
local val = vals[metric_name, filter_name];
l$app = index$str;
if ( metric_name == "apps.bytes" )
l$bytes = double_to_count(floor(val$sum));
else if ( metric_name == "apps.hits" )
{
l$hits = val$num;
l$uniq_hosts = val$unique;
}
}
Log::write(LOG, l);
}
event bro_init() &priority=3
{
Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM)]);
Metrics::add_filter("apps.hits", [$every=break_interval, $measure=set(Metrics::SUM, Metrics::UNIQUE)]);
Log::create_stream(AppMetrics::LOG, [$columns=Info]);
Metrics::create_index_rollup("AppMetrics", app_metrics_rollup);
Metrics::add_filter("apps.bytes", [$every=break_interval, $measure=set(Metrics::SUM), $period_finished=Metrics::write_log, $rollup="AppMetrics"]);
Metrics::add_filter("apps.hits", [$every=break_interval, $measure=set(Metrics::UNIQUE), $rollup="AppMetrics"]);
}
function do_metric(id: conn_id, hostname: string, size: count)
{
if ( /youtube/ in hostname && size > 512*1024 )
if ( /youtube\.com$/ in hostname && size > 512*1024 )
{
Metrics::add_data("apps.bytes", [$str="youtube"], [$num=size]);
Metrics::add_data("apps.hits", [$str="youtube"], [$str=cat(id$orig_h)]);
}
else if ( /facebook.com|fbcdn.net/ in hostname && size > 20 )
else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
{
Metrics::add_data("apps.bytes", [$str="facebook"], [$num=size]);
Metrics::add_data("apps.hits", [$str="facebook"], [$str=cat(id$orig_h)]);
}
else if ( /google.com/ in hostname && size > 20 )
else if ( /\.google\.com$/ in hostname && size > 20 )
{
Metrics::add_data("apps.bytes", [$str="google"], [$num=size]);
Metrics::add_data("apps.hits", [$str="google"], [$str=cat(id$orig_h)]);
}
else if ( /nflximg.com/ in hostname && size > 200*1024 )
else if ( /nflximg\.com$/ in hostname && size > 200*1024 )
{
Metrics::add_data("apps.bytes", [$str="netflix"], [$num=size]);
Metrics::add_data("apps.hits", [$str="netflix"], [$str=cat(id$orig_h)]);
}
else if ( /pandora.com/ in hostname && size > 512*1024 )
else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
{
Metrics::add_data("apps.bytes", [$str="pandora"], [$num=size]);
Metrics::add_data("apps.hits", [$str="pandora"], [$str=cat(id$orig_h)]);
}
else if ( /gmail.com/ in hostname && size > 20 )
else if ( /gmail\.com$/ in hostname && size > 20 )
{
Metrics::add_data("apps.bytes", [$str="gmail"], [$num=size]);
Metrics::add_data("apps.hits", [$str="gmail"], [$str=cat(id$orig_h)]);
@ -63,7 +94,7 @@ event ssl_established(c: connection)
event connection_finished(c: connection)
{
if ( c?$resp_hostname )
do_metric(c$id, c$resp_hostname, c$resp$num_bytes_ip);
do_metric(c$id, c$resp_hostname, c$resp$size);
}
event HTTP::log_http(rec: HTTP::Info)

1
scripts/policy/misc/capture-loss.bro
View file

@ -8,7 +8,6 @@
##! for a sequence number that's above a gap).
@load base/frameworks/notice
@load base/frameworks/metrics
module CaptureLoss;

18
scripts/policy/misc/scan.bro
View file

@ -43,6 +43,10 @@ export {
## Custom threholds based on service for address scan. This is primarily
## useful for setting reduced thresholds for specific ports.
const addr_scan_custom_thresholds: table[port] of count &redef;
global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
}
@ -94,16 +98,14 @@ function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::Result
event bro_init() &priority=5
{
# Note: addr scans are trcked similar to: table[src_ip, port] of set(dst);
Metrics::add_filter("scan.addr.fail", [$log=F,
$every=addr_scan_interval,
Metrics::add_filter("scan.addr.fail", [$every=addr_scan_interval,
$measure=set(Metrics::UNIQUE),
$threshold_func=check_addr_scan_threshold,
$threshold=addr_scan_threshold,
$threshold_crossed=addr_scan_threshold_crossed]);
# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
Metrics::add_filter("scan.port.fail", [$log=F,
$every=port_scan_interval,
Metrics::add_filter("scan.port.fail", [$every=port_scan_interval,
$measure=set(Metrics::UNIQUE),
$threshold=port_scan_threshold,
$threshold_crossed=port_scan_threshold_crossed]);
@ -146,11 +148,11 @@ function add_metrics(id: conn_id, reverse: bool)
#if ( |analyze_subnets| > 0 && host !in analyze_subnets )
# return F;
# Probably do a hook point here?
Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
# Probably do a hook point here?
Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
}
function is_failed_conn(c: connection): bool

6
scripts/policy/protocols/conn/conn-stats-per-host.bro
View file

@ -6,10 +6,12 @@ event bro_init() &priority=5
{
Metrics::add_filter("conn.orig.data",
[$every=5mins,
$measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
$measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
$period_finished=Metrics::write_log]);
Metrics::add_filter("conn.resp.data",
[$every=5mins,
$measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV)]);
$measure=set(Metrics::VARIANCE, Metrics::AVG, Metrics::MAX, Metrics::MIN, Metrics::STD_DEV),
$period_finished=Metrics::write_log]);
}

6
scripts/policy/protocols/conn/metrics.bro
View file

@ -3,8 +3,10 @@
event bro_init() &priority=3
{
Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM)]);
Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM)]);
Metrics::add_filter("conns.country", [$every=1hr, $measure=set(Metrics::SUM),
$period_finished=Metrics::write_log]);
Metrics::add_filter("hosts.active", [$every=1hr, $measure=set(Metrics::SUM),
$period_finished=Metrics::write_log]);
}
event connection_established(c: connection) &priority=3