mirror of
https://github.com/zeek/zeek.git
synced 2025-10-15 13:08:20 +00:00
Measurement framework is ready for testing.
- New, expanded API. - Calculations moved into plugins. - Scripts using measurement framework ported. - Updated the script-land queue implementation to make it more generic. -
This commit is contained in:
Seth Hall
parent
93eca70e6b
commit
b477d2b02d
11 changed files with 183 additions and 186 deletions
|
|
@ -8,22 +8,28 @@ export {
|
|||
redef enum Log::ID += { LOG };
|
||||
|
||||
type Info: record {
|
||||
## Timestamp when the log line was finished and written.
|
||||
ts: time &log;
|
||||
## Time interval that the log line covers.
|
||||
ts_delta: interval &log;
|
||||
## The name of the "app", like "facebook" or "netflix".
|
||||
app: string &log;
|
||||
## The number of unique local hosts using the app.
|
||||
uniq_hosts: count &log;
|
||||
## The number of hits to the app in total.
|
||||
hits: count &log;
|
||||
## The total number of bytes received by users of the app.
|
||||
bytes: count &log;
|
||||
};
|
||||
|
||||
## The frequency of logging the stats collected by this script.
|
||||
const break_interval = 1min &redef;
|
||||
const break_interval = 15mins &redef;
|
||||
}
|
||||
|
||||
redef record connection += {
|
||||
resp_hostname: string &optional;
|
||||
};
|
||||
|
||||
|
||||
event bro_init() &priority=3
|
||||
{
|
||||
Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
|
||||
|
|
@ -32,10 +38,11 @@ event bro_init() &priority=3
|
|||
local r2: Measurement::Reducer = [$stream="apps.hits", $apply=set(Measurement::UNIQUE)];
|
||||
Measurement::create([$epoch=break_interval,
|
||||
$reducers=set(r1, r2),
|
||||
$period_finished(data: Measurement::ResultTable) =
|
||||
$epoch_finished(data: Measurement::ResultTable) =
|
||||
{
|
||||
local l: Info;
|
||||
l$ts = network_time();
|
||||
l$ts_delta = break_interval;
|
||||
for ( key in data )
|
||||
{
|
||||
local result = data[key];
|
||||
|
|
@ -48,7 +55,7 @@ event bro_init() &priority=3
|
|||
}]);
|
||||
}
|
||||
|
||||
function do_metric(id: conn_id, hostname: string, size: count)
|
||||
function do_measurement(id: conn_id, hostname: string, size: count)
|
||||
{
|
||||
if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
|
||||
{
|
||||
|
|
@ -92,11 +99,11 @@ event ssl_established(c: connection)
|
|||
event connection_finished(c: connection)
|
||||
{
|
||||
if ( c?$resp_hostname )
|
||||
do_metric(c$id, c$resp_hostname, c$resp$size);
|
||||
do_measurement(c$id, c$resp_hostname, c$resp$size);
|
||||
}
|
||||
|
||||
event HTTP::log_http(rec: HTTP::Info)
|
||||
{
|
||||
if( rec?$host )
|
||||
do_metric(rec$id, rec$host, rec$response_body_len);
|
||||
do_measurement(rec$id, rec$host, rec$response_body_len);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,53 +49,45 @@ export {
|
|||
global log_traceroute: event(rec: Traceroute::Info);
|
||||
}
|
||||
|
||||
# Track hosts that have sent low TTL packets and which hosts they
|
||||
# sent them to.
|
||||
global low_ttlers: set[addr, addr] = {} &create_expire=2min &synchronized;
|
||||
|
||||
function traceroute_detected(src: addr, dst: addr)
|
||||
{
|
||||
Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
|
||||
NOTICE([$note=Traceroute::Detected,
|
||||
$msg=fmt("%s seems to be running traceroute", src),
|
||||
$src=src, $dst=dst,
|
||||
$identifier=cat(src)]);
|
||||
}
|
||||
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
|
||||
|
||||
Metrics::add_filter("traceroute.time_exceeded",
|
||||
[$log=F,
|
||||
$every=icmp_time_exceeded_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
local r1: Measurement::Reducer = [$stream="traceroute.time_exceeded", $apply=set(Measurement::UNIQUE)];
|
||||
local r2: Measurement::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(Measurement::SUM)];
|
||||
Measurement::create([$epoch=icmp_time_exceeded_interval,
|
||||
$reducers=set(r1, r2),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
# Give a threshold value of zero depending on if the host
|
||||
# sends a low ttl packet.
|
||||
if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
|
||||
return 0;
|
||||
else
|
||||
return result["traceroute.time_exceeded"]$unique;
|
||||
},
|
||||
$threshold=icmp_time_exceeded_threshold,
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
|
||||
local parts = split1(index$str, /-/);
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local parts = split1(key$str, /-/);
|
||||
local src = to_addr(parts[1]);
|
||||
local dst = to_addr(parts[2]);
|
||||
if ( require_low_ttl_packets )
|
||||
{
|
||||
when ( [src, dst] in low_ttlers )
|
||||
{
|
||||
traceroute_detected(src, dst);
|
||||
}
|
||||
}
|
||||
else
|
||||
traceroute_detected(src, dst);
|
||||
}]);
|
||||
Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
|
||||
NOTICE([$note=Traceroute::Detected,
|
||||
$msg=fmt("%s seems to be running traceroute", src),
|
||||
$src=src, $dst=dst,
|
||||
$identifier=cat(src)]);
|
||||
}]);
|
||||
}
|
||||
|
||||
# Low TTL packets are detected with a signature.
|
||||
event signature_match(state: signature_state, msg: string, data: string)
|
||||
{
|
||||
if ( state$sig_id == /traceroute-detector.*/ )
|
||||
add low_ttlers[state$conn$id$orig_h, state$conn$id$resp_h];
|
||||
Measurement::add_data("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
|
||||
}
|
||||
|
||||
event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
|
||||
{
|
||||
Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
|
||||
Measurement::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,59 +52,64 @@ export {
|
|||
}
|
||||
|
||||
|
||||
function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
|
||||
{
|
||||
# We don't need to do this if no custom thresholds are defined.
|
||||
if ( |addr_scan_custom_thresholds| == 0 )
|
||||
return F;
|
||||
|
||||
local service = to_port(index$str);
|
||||
return ( service in addr_scan_custom_thresholds &&
|
||||
val$sum > addr_scan_custom_thresholds[service] );
|
||||
}
|
||||
|
||||
function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
|
||||
{
|
||||
local side = Site::is_local_addr(index$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(val$end-val$begin);
|
||||
local message=fmt("%s scanned at least %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
|
||||
|
||||
NOTICE([$note=Address_Scan,
|
||||
$src=index$host,
|
||||
$p=to_port(index$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(index$host)]);
|
||||
}
|
||||
|
||||
function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
|
||||
{
|
||||
local side = Site::is_local_addr(index$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(val$end-val$begin);
|
||||
local message = fmt("%s scanned at least %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
|
||||
|
||||
NOTICE([$note=Port_Scan,
|
||||
$src=index$host,
|
||||
$dst=to_addr(index$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(index$host)]);
|
||||
}
|
||||
#function check_addr_scan_threshold(key: Measurement::Key, val: Measurement::Result): bool
|
||||
# {
|
||||
# # We don't need to do this if no custom thresholds are defined.
|
||||
# if ( |addr_scan_custom_thresholds| == 0 )
|
||||
# return F;
|
||||
#
|
||||
# local service = to_port(key$str);
|
||||
# return ( service in addr_scan_custom_thresholds &&
|
||||
# val$sum > addr_scan_custom_thresholds[service] );
|
||||
# }
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
# Note: addr scans are trcked similar to: table[src_ip, port] of set(dst);
|
||||
Metrics::add_filter("scan.addr.fail", [$every=addr_scan_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
$threshold_func=check_addr_scan_threshold,
|
||||
$threshold=addr_scan_threshold,
|
||||
$threshold_crossed=addr_scan_threshold_crossed]);
|
||||
local r1: Measurement::Reducer = [$stream="scan.addr.fail", $apply=set(Measurement::UNIQUE)];
|
||||
Measurement::create([$epoch=addr_scan_interval,
|
||||
$reducers=set(r1),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["scan.addr.fail"]$unique);
|
||||
},
|
||||
#$threshold_func=check_addr_scan_threshold,
|
||||
$threshold=addr_scan_threshold,
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["scan.addr.fail"];
|
||||
local side = Site::is_local_addr(key$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(r$end-r$begin);
|
||||
local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
|
||||
NOTICE([$note=Address_Scan,
|
||||
$src=key$host,
|
||||
$p=to_port(key$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(key$host)]);
|
||||
}]);
|
||||
|
||||
# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
|
||||
Metrics::add_filter("scan.port.fail", [$every=port_scan_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
$threshold=port_scan_threshold,
|
||||
$threshold_crossed=port_scan_threshold_crossed]);
|
||||
local r2: Measurement::Reducer = [$stream="scan.port.fail", $apply=set(Measurement::UNIQUE)];
|
||||
Measurement::create([$epoch=port_scan_interval,
|
||||
$reducers=set(r2),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["scan.port.fail"]$unique);
|
||||
},
|
||||
$threshold=port_scan_threshold,
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["scan.port.fail"];
|
||||
local side = Site::is_local_addr(key$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(r$end-r$begin);
|
||||
local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
|
||||
NOTICE([$note=Port_Scan,
|
||||
$src=key$host,
|
||||
$dst=to_addr(key$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(key$host)]);
|
||||
}]);
|
||||
}
|
||||
|
||||
function add_metrics(id: conn_id, reverse: bool)
|
||||
|
|
@ -145,10 +150,10 @@ function add_metrics(id: conn_id, reverse: bool)
|
|||
# return F;
|
||||
|
||||
if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
|
||||
Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
|
||||
Measurement::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
|
||||
|
||||
if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
|
||||
Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
|
||||
Measurement::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
|
||||
}
|
||||
|
||||
function is_failed_conn(c: connection): bool
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue