mirror of
https://github.com/zeek/zeek.git
synced 2025-10-10 02:28:21 +00:00
Measurement framework is ready for testing.
- New, expanded API. - Calculations moved into plugins. - Scripts using measurement framework ported. - Updated the script-land queue implementation to make it more generic. -
This commit is contained in:
Seth Hall
parent
93eca70e6b
commit
b477d2b02d
11 changed files with 183 additions and 186 deletions
|
|
@ -8,22 +8,28 @@ export {
|
|||
redef enum Log::ID += { LOG };
|
||||
|
||||
type Info: record {
|
||||
## Timestamp when the log line was finished and written.
|
||||
ts: time &log;
|
||||
## Time interval that the log line covers.
|
||||
ts_delta: interval &log;
|
||||
## The name of the "app", like "facebook" or "netflix".
|
||||
app: string &log;
|
||||
## The number of unique local hosts using the app.
|
||||
uniq_hosts: count &log;
|
||||
## The number of hits to the app in total.
|
||||
hits: count &log;
|
||||
## The total number of bytes received by users of the app.
|
||||
bytes: count &log;
|
||||
};
|
||||
|
||||
## The frequency of logging the stats collected by this script.
|
||||
const break_interval = 1min &redef;
|
||||
const break_interval = 15mins &redef;
|
||||
}
|
||||
|
||||
redef record connection += {
|
||||
resp_hostname: string &optional;
|
||||
};
|
||||
|
||||
|
||||
event bro_init() &priority=3
|
||||
{
|
||||
Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
|
||||
|
|
@ -32,10 +38,11 @@ event bro_init() &priority=3
|
|||
local r2: Measurement::Reducer = [$stream="apps.hits", $apply=set(Measurement::UNIQUE)];
|
||||
Measurement::create([$epoch=break_interval,
|
||||
$reducers=set(r1, r2),
|
||||
$period_finished(data: Measurement::ResultTable) =
|
||||
$epoch_finished(data: Measurement::ResultTable) =
|
||||
{
|
||||
local l: Info;
|
||||
l$ts = network_time();
|
||||
l$ts_delta = break_interval;
|
||||
for ( key in data )
|
||||
{
|
||||
local result = data[key];
|
||||
|
|
@ -48,7 +55,7 @@ event bro_init() &priority=3
|
|||
}]);
|
||||
}
|
||||
|
||||
function do_metric(id: conn_id, hostname: string, size: count)
|
||||
function do_measurement(id: conn_id, hostname: string, size: count)
|
||||
{
|
||||
if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
|
||||
{
|
||||
|
|
@ -92,11 +99,11 @@ event ssl_established(c: connection)
|
|||
event connection_finished(c: connection)
|
||||
{
|
||||
if ( c?$resp_hostname )
|
||||
do_metric(c$id, c$resp_hostname, c$resp$size);
|
||||
do_measurement(c$id, c$resp_hostname, c$resp$size);
|
||||
}
|
||||
|
||||
event HTTP::log_http(rec: HTTP::Info)
|
||||
{
|
||||
if( rec?$host )
|
||||
do_metric(rec$id, rec$host, rec$response_body_len);
|
||||
do_measurement(rec$id, rec$host, rec$response_body_len);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -49,53 +49,45 @@ export {
|
|||
global log_traceroute: event(rec: Traceroute::Info);
|
||||
}
|
||||
|
||||
# Track hosts that have sent low TTL packets and which hosts they
|
||||
# sent them to.
|
||||
global low_ttlers: set[addr, addr] = {} &create_expire=2min &synchronized;
|
||||
|
||||
function traceroute_detected(src: addr, dst: addr)
|
||||
{
|
||||
Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
|
||||
NOTICE([$note=Traceroute::Detected,
|
||||
$msg=fmt("%s seems to be running traceroute", src),
|
||||
$src=src, $dst=dst,
|
||||
$identifier=cat(src)]);
|
||||
}
|
||||
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
|
||||
|
||||
Metrics::add_filter("traceroute.time_exceeded",
|
||||
[$log=F,
|
||||
$every=icmp_time_exceeded_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
local r1: Measurement::Reducer = [$stream="traceroute.time_exceeded", $apply=set(Measurement::UNIQUE)];
|
||||
local r2: Measurement::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(Measurement::SUM)];
|
||||
Measurement::create([$epoch=icmp_time_exceeded_interval,
|
||||
$reducers=set(r1, r2),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
# Give a threshold value of zero depending on if the host
|
||||
# sends a low ttl packet.
|
||||
if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
|
||||
return 0;
|
||||
else
|
||||
return result["traceroute.time_exceeded"]$unique;
|
||||
},
|
||||
$threshold=icmp_time_exceeded_threshold,
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
|
||||
local parts = split1(index$str, /-/);
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local parts = split1(key$str, /-/);
|
||||
local src = to_addr(parts[1]);
|
||||
local dst = to_addr(parts[2]);
|
||||
if ( require_low_ttl_packets )
|
||||
{
|
||||
when ( [src, dst] in low_ttlers )
|
||||
{
|
||||
traceroute_detected(src, dst);
|
||||
}
|
||||
}
|
||||
else
|
||||
traceroute_detected(src, dst);
|
||||
}]);
|
||||
Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
|
||||
NOTICE([$note=Traceroute::Detected,
|
||||
$msg=fmt("%s seems to be running traceroute", src),
|
||||
$src=src, $dst=dst,
|
||||
$identifier=cat(src)]);
|
||||
}]);
|
||||
}
|
||||
|
||||
# Low TTL packets are detected with a signature.
|
||||
event signature_match(state: signature_state, msg: string, data: string)
|
||||
{
|
||||
if ( state$sig_id == /traceroute-detector.*/ )
|
||||
add low_ttlers[state$conn$id$orig_h, state$conn$id$resp_h];
|
||||
Measurement::add_data("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
|
||||
}
|
||||
|
||||
event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
|
||||
{
|
||||
Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
|
||||
Measurement::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -52,59 +52,64 @@ export {
|
|||
}
|
||||
|
||||
|
||||
function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
|
||||
{
|
||||
# We don't need to do this if no custom thresholds are defined.
|
||||
if ( |addr_scan_custom_thresholds| == 0 )
|
||||
return F;
|
||||
|
||||
local service = to_port(index$str);
|
||||
return ( service in addr_scan_custom_thresholds &&
|
||||
val$sum > addr_scan_custom_thresholds[service] );
|
||||
}
|
||||
|
||||
function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
|
||||
{
|
||||
local side = Site::is_local_addr(index$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(val$end-val$begin);
|
||||
local message=fmt("%s scanned at least %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
|
||||
|
||||
NOTICE([$note=Address_Scan,
|
||||
$src=index$host,
|
||||
$p=to_port(index$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(index$host)]);
|
||||
}
|
||||
|
||||
function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
|
||||
{
|
||||
local side = Site::is_local_addr(index$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(val$end-val$begin);
|
||||
local message = fmt("%s scanned at least %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
|
||||
|
||||
NOTICE([$note=Port_Scan,
|
||||
$src=index$host,
|
||||
$dst=to_addr(index$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(index$host)]);
|
||||
}
|
||||
#function check_addr_scan_threshold(key: Measurement::Key, val: Measurement::Result): bool
|
||||
# {
|
||||
# # We don't need to do this if no custom thresholds are defined.
|
||||
# if ( |addr_scan_custom_thresholds| == 0 )
|
||||
# return F;
|
||||
#
|
||||
# local service = to_port(key$str);
|
||||
# return ( service in addr_scan_custom_thresholds &&
|
||||
# val$sum > addr_scan_custom_thresholds[service] );
|
||||
# }
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
# Note: addr scans are trcked similar to: table[src_ip, port] of set(dst);
|
||||
Metrics::add_filter("scan.addr.fail", [$every=addr_scan_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
$threshold_func=check_addr_scan_threshold,
|
||||
$threshold=addr_scan_threshold,
|
||||
$threshold_crossed=addr_scan_threshold_crossed]);
|
||||
local r1: Measurement::Reducer = [$stream="scan.addr.fail", $apply=set(Measurement::UNIQUE)];
|
||||
Measurement::create([$epoch=addr_scan_interval,
|
||||
$reducers=set(r1),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["scan.addr.fail"]$unique);
|
||||
},
|
||||
#$threshold_func=check_addr_scan_threshold,
|
||||
$threshold=addr_scan_threshold,
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["scan.addr.fail"];
|
||||
local side = Site::is_local_addr(key$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(r$end-r$begin);
|
||||
local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
|
||||
NOTICE([$note=Address_Scan,
|
||||
$src=key$host,
|
||||
$p=to_port(key$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(key$host)]);
|
||||
}]);
|
||||
|
||||
# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
|
||||
Metrics::add_filter("scan.port.fail", [$every=port_scan_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
$threshold=port_scan_threshold,
|
||||
$threshold_crossed=port_scan_threshold_crossed]);
|
||||
local r2: Measurement::Reducer = [$stream="scan.port.fail", $apply=set(Measurement::UNIQUE)];
|
||||
Measurement::create([$epoch=port_scan_interval,
|
||||
$reducers=set(r2),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["scan.port.fail"]$unique);
|
||||
},
|
||||
$threshold=port_scan_threshold,
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["scan.port.fail"];
|
||||
local side = Site::is_local_addr(key$host) ? "local" : "remote";
|
||||
local dur = duration_to_mins_secs(r$end-r$begin);
|
||||
local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
|
||||
NOTICE([$note=Port_Scan,
|
||||
$src=key$host,
|
||||
$dst=to_addr(key$str),
|
||||
$sub=side,
|
||||
$msg=message,
|
||||
$identifier=cat(key$host)]);
|
||||
}]);
|
||||
}
|
||||
|
||||
function add_metrics(id: conn_id, reverse: bool)
|
||||
|
|
@ -145,10 +150,10 @@ function add_metrics(id: conn_id, reverse: bool)
|
|||
# return F;
|
||||
|
||||
if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
|
||||
Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
|
||||
Measurement::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
|
||||
|
||||
if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
|
||||
Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
|
||||
Measurement::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
|
||||
}
|
||||
|
||||
function is_failed_conn(c: connection): bool
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@ event bro_init()
|
|||
{
|
||||
Metrics::add_filter("ftp.failed_auth", [$every=bruteforce_measurement_interval,
|
||||
$measure=set(Metrics::UNIQUE),
|
||||
$threshold_val_func(val: Metrics::ResultVal) = { return val$num; },
|
||||
$threshold_val_func(val: Metrics::Result) = { return val$num; },
|
||||
$threshold=bruteforce_threshold,
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) =
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::Result) =
|
||||
{
|
||||
local dur = duration_to_mins_secs(val$end-val$begin);
|
||||
local plural = val$unique>1 ? "s" : "";
|
||||
|
|
|
|||
|
|
@ -50,11 +50,11 @@ export {
|
|||
| /\/\*![[:digit:]]{5}.*?\*\// &redef;
|
||||
}
|
||||
|
||||
function format_sqli_samples(samples: vector of string): string
|
||||
function format_sqli_samples(samples: vector of Measurement::DataPoint): string
|
||||
{
|
||||
local ret = "SQL Injection samples\n---------------------";
|
||||
for ( i in samples )
|
||||
ret += "\n" + samples[i];
|
||||
ret += "\n" + samples[i]$str;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
|
@ -63,31 +63,41 @@ event bro_init() &priority=3
|
|||
# Add filters to the metrics so that the metrics framework knows how to
|
||||
# determine when it looks like an actual attack and how to respond when
|
||||
# thresholds are crossed.
|
||||
Metrics::add_filter("http.sqli.attacker",
|
||||
[$every=sqli_requests_interval,
|
||||
$measure=set(Metrics::SUM),
|
||||
local r1: Measurement::Reducer = [$stream="http.sqli.attacker", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
|
||||
Measurement::create([$epoch=sqli_requests_interval,
|
||||
$reducers=set(r1),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["http.sqli.attacker"]$sum);
|
||||
},
|
||||
$threshold=sqli_requests_threshold,
|
||||
$samples=collect_SQLi_samples,
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["http.sqli.attacker"];
|
||||
NOTICE([$note=SQL_Injection_Attacker,
|
||||
$msg="An SQL injection attacker was discovered!",
|
||||
$email_body_sections=vector(format_sqli_samples(val$samples)),
|
||||
$src=index$host,
|
||||
$identifier=cat(index$host)]);
|
||||
}]);
|
||||
$email_body_sections=vector(format_sqli_samples(r$samples)),
|
||||
$src=key$host,
|
||||
$identifier=cat(key$host)]);
|
||||
}]);
|
||||
|
||||
Metrics::add_filter("http.sqli.victim",
|
||||
[$every=sqli_requests_interval,
|
||||
$measure=set(Metrics::SUM),
|
||||
local r2: Measurement::Reducer = [$stream="http.sqli.victim", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
|
||||
Measurement::create([$epoch=sqli_requests_interval,
|
||||
$reducers=set(r2),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["http.sqli.victim"]$sum);
|
||||
},
|
||||
$threshold=sqli_requests_threshold,
|
||||
$samples=collect_SQLi_samples,
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["http.sqli.victim"];
|
||||
NOTICE([$note=SQL_Injection_Victim,
|
||||
$msg="An SQL injection victim was discovered!",
|
||||
$email_body_sections=vector(format_sqli_samples(val$samples)),
|
||||
$src=index$host,
|
||||
$identifier=cat(index$host)]);
|
||||
}]);
|
||||
$email_body_sections=vector(format_sqli_samples(r$samples)),
|
||||
$src=key$host,
|
||||
$identifier=cat(key$host)]);
|
||||
}]);
|
||||
}
|
||||
|
||||
event http_request(c: connection, method: string, original_URI: string,
|
||||
|
|
@ -97,7 +107,7 @@ event http_request(c: connection, method: string, original_URI: string,
|
|||
{
|
||||
add c$http$tags[URI_SQLI];
|
||||
|
||||
Metrics::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
|
||||
Metrics::add_data("http.sqli.victim", [$host=c$id$resp_h], [$str=original_URI]);
|
||||
Measurement::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
|
||||
Measurement::add_data("http.sqli.victim", [$host=c$id$resp_h], [$str=original_URI]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,21 +42,27 @@ export {
|
|||
|
||||
event bro_init()
|
||||
{
|
||||
Metrics::add_filter("ssh.login.failure", [$name="detect-bruteforcing", $log=F,
|
||||
$every=guessing_timeout,
|
||||
$measure=set(Metrics::SUM),
|
||||
$threshold=password_guesses_limit,
|
||||
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
|
||||
# Generate the notice.
|
||||
NOTICE([$note=Password_Guessing,
|
||||
$msg=fmt("%s appears to be guessing SSH passwords (seen in %.0f connections).", index$host, val$sum),
|
||||
$src=index$host,
|
||||
$identifier=cat(index$host)]);
|
||||
# Insert the guesser into the intel framework.
|
||||
Intel::insert([$host=index$host,
|
||||
$meta=[$source="local",
|
||||
$desc=fmt("Bro observed %0.f apparently failed SSH connections.", val$sum)]]);
|
||||
}]);
|
||||
local r1: Measurement::Reducer = [$stream="ssh.login.failure", $apply=set(Measurement::SUM)];
|
||||
Measurement::create([$epoch=guessing_timeout,
|
||||
$reducers=set(r1),
|
||||
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
return double_to_count(result["ssh.login.failure"]$sum);
|
||||
},
|
||||
$threshold=password_guesses_limit,
|
||||
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
|
||||
{
|
||||
local r = result["ssh.login.failure"];
|
||||
# Generate the notice.
|
||||
NOTICE([$note=Password_Guessing,
|
||||
$msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
|
||||
$src=key$host,
|
||||
$identifier=cat(key$host)]);
|
||||
# Insert the guesser into the intel framework.
|
||||
Intel::insert([$host=key$host,
|
||||
$meta=[$source="local",
|
||||
$desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]);
|
||||
}]);
|
||||
}
|
||||
|
||||
event SSH::heuristic_successful_login(c: connection)
|
||||
|
|
@ -76,5 +82,5 @@ event SSH::heuristic_failed_login(c: connection)
|
|||
# be ignored.
|
||||
if ( ! (id$orig_h in ignore_guessers &&
|
||||
id$resp_h in ignore_guessers[id$orig_h]) )
|
||||
Metrics::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
|
||||
Measurement::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue