Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Measurement framework is ready for testing.

Browse source 
- New, expanded API.
 - Calculations moved into plugins.
 - Scripts using measurement framework ported.
 - Updated the script-land queue implementation to make it more generic.
 -
This commit is contained in:
Seth Hall 2013-04-01 17:04:15 -04:00
parent 93eca70e6b
commit b477d2b02d
11 changed files with 183 additions and 186 deletions
Download patch file Download diff file Expand all files Collapse all files

8
scripts/base/frameworks/measurement/main.bro
View file

@ -255,6 +255,11 @@ function reset(m: Measurement)
function create(m: Measurement)
{
if ( (m?$threshold || m?$threshold_series) && ! m?$threshold_val )
{
Reporter::error("Measurement given a threshold with no $threshold_val function");
}
if ( ! m?$id )
m$id=unique_id("");
local tmp: table[Key] of Thresholding = table();
@ -365,9 +370,6 @@ function threshold_crossed(m: Measurement, key: Key, result: Result)
if ( ! m?$threshold_crossed )
return;
#if ( val?$sample_queue )
# val$samples = Queue::get_str_vector(val$sample_queue);
# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
if ( |m$reducers| != |result| )
{

6
scripts/base/frameworks/measurement/plugins/sample.bro
View file

@ -1,3 +1,4 @@
@load base/utils/queue
module Measurement;
@ -29,7 +30,10 @@ hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal
{
if ( ! rv?$sample_queue )
rv$sample_queue = Queue::init([$max_len=r$samples]);
Queue::push(rv$sample_queue, data$str);
if ( ! rv?$samples )
rv$samples = vector();
Queue::put(rv$sample_queue, data);
Queue::get_vector(rv$sample_queue, rv$samples);
}
}

6
scripts/base/frameworks/measurement/plugins/sum.bro
View file

@ -27,6 +27,12 @@ function sum_threshold(data_id: string): threshold_function
};
}
hook init_resultval_hook(r: Reducer, rv: ResultVal)
{
if ( SUM in r$apply && ! rv?$sum )
rv$sum = 0;
}
hook add_to_reducer_hook(r: Reducer, val: double, data: DataPoint, rv: ResultVal)
{
if ( SUM in r$apply )

2
scripts/base/frameworks/measurement/plugins/unique.bro
View file

@ -10,7 +10,7 @@ export {
redef record ResultVal += {
## If cardinality is being tracked, the number of unique
## items is tracked here.
unique: count &optional;
unique: count &default=0;
};
}

71
scripts/base/utils/queue.bro
View file

@ -1,4 +1,4 @@
##! A FIFO string queue.
##! A FIFO queue.
module Queue;
@ -23,17 +23,17 @@ export {
## Push a string onto the top of a queue.
##
## q: The queue to push the string into.
## q: The queue to put the value into.
##
## val: The string to push
global push: function(q: Queue, val: any);
## val: The value to insert into the queue.
global put: function(q: Queue, val: any);
## Pop a string from the bottom of a queue.
##
## q: The queue to pop the string from.
## q: The queue to get the string from.
##
## Returns: The string popped from the queue.
global pop: function(q: Queue): any;
## Returns: The value gotten from the queue.
global get: function(q: Queue): any;
## Merge two queue's together. If any settings are applied
## to the queues, the settings from q1 are used for the new
@ -53,23 +53,14 @@ export {
## Returns: The length of the queue.
global len: function(q: Queue): count;
## Get the contents of the queue as a string vector.
## Get the contents of the queue as a vector.
##
## q: The queue.
##
## Returns: A :bro:type:`vector of string` containing the
## current contents of q.
global get_str_vector: function(q: Queue): vector of string;
## ret: A vector containing the
## current contents of q as the type of ret.
global get_vector: function(q: Queue, ret: vector of any);
## Get the contents of the queue as a count vector. Use care
## with this function. If the data put into the queue wasn't
## integers you will get conversion errors.
##
## q: The queue.
##
## Returns: A :bro:type:`vector of count` containing the
## current contents of q.
global get_cnt_vector: function(q: Queue): vector of count;
}
redef record Queue += {
@ -96,15 +87,15 @@ function init(s: Settings): Queue
return q;
}
function push(q: Queue, val: any)
function put(q: Queue, val: any)
{
if ( q$settings?$max_len && len(q) >= q$settings$max_len )
pop(q);
get(q);
q$vals[q$top] = val;
++q$top;
}
function pop(q: Queue): any
function get(q: Queue): any
{
local ret = q$vals[q$bottom];
delete q$vals[q$bottom];
@ -120,9 +111,9 @@ function merge(q1: Queue, q2: Queue): Queue
for ( ignored_val in q1$vals )
{
if ( i in q1$vals )
push(ret, q1$vals[i]);
put(ret, q1$vals[i]);
if ( j in q2$vals )
push(ret, q2$vals[j]);
put(ret, q2$vals[j]);
++i;
++j;
}
@ -134,9 +125,8 @@ function len(q: Queue): count
return |q$vals|;
}
function get_str_vector(q: Queue): vector of string
function get_vector(q: Queue, ret: vector of any)
{
local ret: vector of string;
local i = q$bottom;
local j = 0;
# Really dumb hack, this is only to provide
@ -147,32 +137,7 @@ function get_str_vector(q: Queue): vector of string
if ( i >= q$top )
break;
ret[j] = cat(q$vals[i]);
ret[j] = q$vals[i];
++j; ++i;
}
return ret;
}
function get_cnt_vector(q: Queue): vector of count
{
local ret: vector of count;
local i = q$bottom;
local j = 0;
# Really dumb hack, this is only to provide
# the iteration for the correct number of
# values in q$vals.
for ( ignored_val in q$vals )
{
if ( i >= q$top )
break;
# TODO: this is terrible and should be replaced by
# a more generic version of the various
# functions to get vectors of values.
# (the way "any" works right now makes this impossible though)
ret[j] = to_count(cat(q$vals[i]));
++j; ++i;
}
return ret;
}

19
scripts/policy/misc/app-metrics.bro
View file

@ -8,22 +8,28 @@ export {
redef enum Log::ID += { LOG };
type Info: record {
## Timestamp when the log line was finished and written.
ts: time &log;
## Time interval that the log line covers.
ts_delta: interval &log;
## The name of the "app", like "facebook" or "netflix".
app: string &log;
## The number of unique local hosts using the app.
uniq_hosts: count &log;
## The number of hits to the app in total.
hits: count &log;
## The total number of bytes received by users of the app.
bytes: count &log;
};
## The frequency of logging the stats collected by this script.
const break_interval = 1min &redef;
const break_interval = 15mins &redef;
}
redef record connection += {
resp_hostname: string &optional;
};
event bro_init() &priority=3
{
Log::create_stream(AppMeasurement::LOG, [$columns=Info]);
@ -32,10 +38,11 @@ event bro_init() &priority=3
local r2: Measurement::Reducer = [$stream="apps.hits", $apply=set(Measurement::UNIQUE)];
Measurement::create([$epoch=break_interval,
$reducers=set(r1, r2),
$period_finished(data: Measurement::ResultTable) =
$epoch_finished(data: Measurement::ResultTable) =
{
local l: Info;
l$ts = network_time();
l$ts_delta = break_interval;
for ( key in data )
{
local result = data[key];
@ -48,7 +55,7 @@ event bro_init() &priority=3
}]);
}
function do_metric(id: conn_id, hostname: string, size: count)
function do_measurement(id: conn_id, hostname: string, size: count)
{
if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
{
@ -92,11 +99,11 @@ event ssl_established(c: connection)
event connection_finished(c: connection)
{
if ( c?$resp_hostname )
do_metric(c$id, c$resp_hostname, c$resp$size);
do_measurement(c$id, c$resp_hostname, c$resp$size);
}
event HTTP::log_http(rec: HTTP::Info)
{
if( rec?$host )
do_metric(rec$id, rec$host, rec$response_body_len);
do_measurement(rec$id, rec$host, rec$response_body_len);
}

56
scripts/policy/misc/detect-traceroute/main.bro
View file

@ -49,53 +49,45 @@ export {
global log_traceroute: event(rec: Traceroute::Info);
}
# Track hosts that have sent low TTL packets and which hosts they
# sent them to.
global low_ttlers: set[addr, addr] = {} &create_expire=2min &synchronized;
function traceroute_detected(src: addr, dst: addr)
{
Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
NOTICE([$note=Traceroute::Detected,
$msg=fmt("%s seems to be running traceroute", src),
$src=src, $dst=dst,
$identifier=cat(src)]);
}
event bro_init() &priority=5
{
Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
Metrics::add_filter("traceroute.time_exceeded",
[$log=F,
$every=icmp_time_exceeded_interval,
$measure=set(Metrics::UNIQUE),
local r1: Measurement::Reducer = [$stream="traceroute.time_exceeded", $apply=set(Measurement::UNIQUE)];
local r2: Measurement::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(Measurement::SUM)];
Measurement::create([$epoch=icmp_time_exceeded_interval,
$reducers=set(r1, r2),
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
{
# Give a threshold value of zero depending on if the host
# sends a low ttl packet.
if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
return 0;
else
return result["traceroute.time_exceeded"]$unique;
},
$threshold=icmp_time_exceeded_threshold,
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
local parts = split1(index$str, /-/);
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
{
local parts = split1(key$str, /-/);
local src = to_addr(parts[1]);
local dst = to_addr(parts[2]);
if ( require_low_ttl_packets )
{
when ( [src, dst] in low_ttlers )
{
traceroute_detected(src, dst);
}
}
else
traceroute_detected(src, dst);
}]);
Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst]);
NOTICE([$note=Traceroute::Detected,
$msg=fmt("%s seems to be running traceroute", src),
$src=src, $dst=dst,
$identifier=cat(src)]);
}]);
}
# Low TTL packets are detected with a signature.
event signature_match(state: signature_state, msg: string, data: string)
{
if ( state$sig_id == /traceroute-detector.*/ )
add low_ttlers[state$conn$id$orig_h, state$conn$id$resp_h];
Measurement::add_data("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h)], [$num=1]);
}
event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
{
Metrics::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
Measurement::add_data("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h)], [$str=cat(c$id$orig_h)]);
}

105
scripts/policy/misc/scan.bro
View file

@ -52,59 +52,64 @@ export {
}
function check_addr_scan_threshold(index: Metrics::Index, val: Metrics::ResultVal): bool
{
# We don't need to do this if no custom thresholds are defined.
if ( |addr_scan_custom_thresholds| == 0 )
return F;
local service = to_port(index$str);
return ( service in addr_scan_custom_thresholds &&
val$sum > addr_scan_custom_thresholds[service] );
}
function addr_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
{
local side = Site::is_local_addr(index$host) ? "local" : "remote";
local dur = duration_to_mins_secs(val$end-val$begin);
local message=fmt("%s scanned at least %d unique hosts on port %s in %s", index$host, val$unique, index$str, dur);
NOTICE([$note=Address_Scan,
$src=index$host,
$p=to_port(index$str),
$sub=side,
$msg=message,
$identifier=cat(index$host)]);
}
function port_scan_threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal)
{
local side = Site::is_local_addr(index$host) ? "local" : "remote";
local dur = duration_to_mins_secs(val$end-val$begin);
local message = fmt("%s scanned at least %d unique ports of host %s in %s", index$host, val$unique, index$str, dur);
NOTICE([$note=Port_Scan,
$src=index$host,
$dst=to_addr(index$str),
$sub=side,
$msg=message,
$identifier=cat(index$host)]);
}
#function check_addr_scan_threshold(key: Measurement::Key, val: Measurement::Result): bool
# {
# # We don't need to do this if no custom thresholds are defined.
# if ( |addr_scan_custom_thresholds| == 0 )
# return F;
#
# local service = to_port(key$str);
# return ( service in addr_scan_custom_thresholds &&
# val$sum > addr_scan_custom_thresholds[service] );
# }
event bro_init() &priority=5
{
# Note: addr scans are trcked similar to: table[src_ip, port] of set(dst);
Metrics::add_filter("scan.addr.fail", [$every=addr_scan_interval,
$measure=set(Metrics::UNIQUE),
$threshold_func=check_addr_scan_threshold,
$threshold=addr_scan_threshold,
$threshold_crossed=addr_scan_threshold_crossed]);
local r1: Measurement::Reducer = [$stream="scan.addr.fail", $apply=set(Measurement::UNIQUE)];
Measurement::create([$epoch=addr_scan_interval,
$reducers=set(r1),
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
{
return double_to_count(result["scan.addr.fail"]$unique);
},
#$threshold_func=check_addr_scan_threshold,
$threshold=addr_scan_threshold,
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
{
local r = result["scan.addr.fail"];
local side = Site::is_local_addr(key$host) ? "local" : "remote";
local dur = duration_to_mins_secs(r$end-r$begin);
local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
NOTICE([$note=Address_Scan,
$src=key$host,
$p=to_port(key$str),
$sub=side,
$msg=message,
$identifier=cat(key$host)]);
}]);
# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
Metrics::add_filter("scan.port.fail", [$every=port_scan_interval,
$measure=set(Metrics::UNIQUE),
$threshold=port_scan_threshold,
$threshold_crossed=port_scan_threshold_crossed]);
local r2: Measurement::Reducer = [$stream="scan.port.fail", $apply=set(Measurement::UNIQUE)];
Measurement::create([$epoch=port_scan_interval,
$reducers=set(r2),
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
{
return double_to_count(result["scan.port.fail"]$unique);
},
$threshold=port_scan_threshold,
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
{
local r = result["scan.port.fail"];
local side = Site::is_local_addr(key$host) ? "local" : "remote";
local dur = duration_to_mins_secs(r$end-r$begin);
local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
NOTICE([$note=Port_Scan,
$src=key$host,
$dst=to_addr(key$str),
$sub=side,
$msg=message,
$identifier=cat(key$host)]);
}]);
}
function add_metrics(id: conn_id, reverse: bool)
@ -145,10 +150,10 @@ function add_metrics(id: conn_id, reverse: bool)
# return F;
if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
Metrics::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
Measurement::add_data("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
Metrics::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
Measurement::add_data("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
}
function is_failed_conn(c: connection): bool

4
scripts/policy/protocols/ftp/detect-bruteforcing.bro
View file

@ -27,9 +27,9 @@ event bro_init()
{
Metrics::add_filter("ftp.failed_auth", [$every=bruteforce_measurement_interval,
$measure=set(Metrics::UNIQUE),
$threshold_val_func(val: Metrics::ResultVal) = { return val$num; },
$threshold_val_func(val: Metrics::Result) = { return val$num; },
$threshold=bruteforce_threshold,
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) =
$threshold_crossed(index: Metrics::Index, val: Metrics::Result) =
{
local dur = duration_to_mins_secs(val$end-val$begin);
local plural = val$unique>1 ? "s" : "";

54
scripts/policy/protocols/http/detect-sqli.bro
View file

@ -50,11 +50,11 @@ export {
| /\/\*![[:digit:]]{5}.*?\*\// &redef;
}
function format_sqli_samples(samples: vector of string): string
function format_sqli_samples(samples: vector of Measurement::DataPoint): string
{
local ret = "SQL Injection samples\n---------------------";
for ( i in samples )
ret += "\n" + samples[i];
ret += "\n" + samples[i]$str;
return ret;
}
@ -63,31 +63,41 @@ event bro_init() &priority=3
# Add filters to the metrics so that the metrics framework knows how to
# determine when it looks like an actual attack and how to respond when
# thresholds are crossed.
Metrics::add_filter("http.sqli.attacker",
[$every=sqli_requests_interval,
$measure=set(Metrics::SUM),
local r1: Measurement::Reducer = [$stream="http.sqli.attacker", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
Measurement::create([$epoch=sqli_requests_interval,
$reducers=set(r1),
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
{
return double_to_count(result["http.sqli.attacker"]$sum);
},
$threshold=sqli_requests_threshold,
$samples=collect_SQLi_samples,
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
{
local r = result["http.sqli.attacker"];
NOTICE([$note=SQL_Injection_Attacker,
$msg="An SQL injection attacker was discovered!",
$email_body_sections=vector(format_sqli_samples(val$samples)),
$src=index$host,
$identifier=cat(index$host)]);
}]);
$email_body_sections=vector(format_sqli_samples(r$samples)),
$src=key$host,
$identifier=cat(key$host)]);
}]);
Metrics::add_filter("http.sqli.victim",
[$every=sqli_requests_interval,
$measure=set(Metrics::SUM),
local r2: Measurement::Reducer = [$stream="http.sqli.victim", $apply=set(Measurement::SUM), $samples=collect_SQLi_samples];
Measurement::create([$epoch=sqli_requests_interval,
$reducers=set(r2),
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
{
return double_to_count(result["http.sqli.victim"]$sum);
},
$threshold=sqli_requests_threshold,
$samples=collect_SQLi_samples,
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
{
local r = result["http.sqli.victim"];
NOTICE([$note=SQL_Injection_Victim,
$msg="An SQL injection victim was discovered!",
$email_body_sections=vector(format_sqli_samples(val$samples)),
$src=index$host,
$identifier=cat(index$host)]);
}]);
$email_body_sections=vector(format_sqli_samples(r$samples)),
$src=key$host,
$identifier=cat(key$host)]);
}]);
}
event http_request(c: connection, method: string, original_URI: string,
@ -97,7 +107,7 @@ event http_request(c: connection, method: string, original_URI: string,
{
add c$http$tags[URI_SQLI];
Metrics::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
Metrics::add_data("http.sqli.victim", [$host=c$id$resp_h], [$str=original_URI]);
Measurement::add_data("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
Measurement::add_data("http.sqli.victim", [$host=c$id$resp_h], [$str=original_URI]);
}
}

38
scripts/policy/protocols/ssh/detect-bruteforcing.bro
View file

@ -42,21 +42,27 @@ export {
event bro_init()
{
Metrics::add_filter("ssh.login.failure", [$name="detect-bruteforcing", $log=F,
$every=guessing_timeout,
$measure=set(Metrics::SUM),
$threshold=password_guesses_limit,
$threshold_crossed(index: Metrics::Index, val: Metrics::ResultVal) = {
# Generate the notice.
NOTICE([$note=Password_Guessing,
$msg=fmt("%s appears to be guessing SSH passwords (seen in %.0f connections).", index$host, val$sum),
$src=index$host,
$identifier=cat(index$host)]);
# Insert the guesser into the intel framework.
Intel::insert([$host=index$host,
$meta=[$source="local",
$desc=fmt("Bro observed %0.f apparently failed SSH connections.", val$sum)]]);
}]);
local r1: Measurement::Reducer = [$stream="ssh.login.failure", $apply=set(Measurement::SUM)];
Measurement::create([$epoch=guessing_timeout,
$reducers=set(r1),
$threshold_val(key: Measurement::Key, result: Measurement::Result) =
{
return double_to_count(result["ssh.login.failure"]$sum);
},
$threshold=password_guesses_limit,
$threshold_crossed(key: Measurement::Key, result: Measurement::Result) =
{
local r = result["ssh.login.failure"];
# Generate the notice.
NOTICE([$note=Password_Guessing,
$msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
$src=key$host,
$identifier=cat(key$host)]);
# Insert the guesser into the intel framework.
Intel::insert([$host=key$host,
$meta=[$source="local",
$desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]);
}]);
}
event SSH::heuristic_successful_login(c: connection)
@ -76,5 +82,5 @@ event SSH::heuristic_failed_login(c: connection)
# be ignored.
if ( ! (id$orig_h in ignore_guessers &&
id$resp_h in ignore_guessers[id$orig_h]) )
Metrics::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
Measurement::add_data("ssh.login.failure", [$host=id$orig_h], [$num=1]);
}