mirror of
https://github.com/zeek/zeek.git
synced 2025-10-09 18:18:19 +00:00
Merge remote-tracking branch 'origin/topic/johanna/conn-threshold'
* origin/topic/johanna/conn-threshold: Wrap threshold stuff up - fix two small bugs and update baselines. update GridFTP analyzer to use connection thresholding instead of polling Add high level api for thresholding that holds lists of thresholds and raises an event for each threshold exactly once. Allow setting packet and byte thresholds for connections. BIT-1377 #merged
This commit is contained in:
Robin Sommer
commit
1e010fbb76
23 changed files with 877 additions and 218 deletions
|
|
@ -2,3 +2,4 @@
|
|||
@load ./contents
|
||||
@load ./inactivity
|
||||
@load ./polling
|
||||
@load ./thresholds
|
||||
|
|
|
|||
274
scripts/base/protocols/conn/thresholds.bro
Normal file
274
scripts/base/protocols/conn/thresholds.bro
Normal file
|
|
@ -0,0 +1,274 @@
|
|||
##! Implements a generic API to throw events when a connection crosses a
|
||||
##! fixed threshold of bytes or packets.
|
||||
|
||||
module ConnThreshold;
|
||||
|
||||
export {
|
||||
|
||||
type Thresholds: record {
|
||||
orig_byte: set[count] &default=count_set(); ##< current originator byte thresholds we watch for
|
||||
resp_byte: set[count] &default=count_set(); ##< current responder byte thresholds we watch for
|
||||
orig_packet: set[count] &default=count_set(); ##< corrent originator packet thresholds we watch for
|
||||
resp_packet: set[count] &default=count_set(); ##< corrent responder packet thresholds we watch for
|
||||
};
|
||||
|
||||
## Sets a byte threshold for connection sizes, adding it to potentially already existing thresholds.
|
||||
## conn_bytes_threshold_crossed will be raised for each set threshold.
|
||||
##
|
||||
## cid: The connection id.
|
||||
##
|
||||
## threshold: Threshold in bytes.
|
||||
##
|
||||
## is_orig: If true, threshold is set for bytes from originator, otherwise for bytes from responder.
|
||||
##
|
||||
## Returns: T on success, F on failure.
|
||||
##
|
||||
## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_packets_threshold
|
||||
## delete_bytes_threshold delete_packets_threshold
|
||||
global set_bytes_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
|
||||
|
||||
## Sets a packet threshold for connection sizes, adding it to potentially already existing thresholds.
|
||||
## conn_packets_threshold_crossed will be raised for each set threshold.
|
||||
##
|
||||
## cid: The connection id.
|
||||
##
|
||||
## threshold: Threshold in packets.
|
||||
##
|
||||
## is_orig: If true, threshold is set for packets from originator, otherwise for packets from responder.
|
||||
##
|
||||
## Returns: T on success, F on failure.
|
||||
##
|
||||
## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_bytes_threshold
|
||||
## delete_bytes_threshold delete_packets_threshold
|
||||
global set_packets_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
|
||||
|
||||
## Deletes a byte threshold for connection sizes.
|
||||
##
|
||||
## cid: The connection id.
|
||||
##
|
||||
## threshold: Threshold in bytes to remove.
|
||||
##
|
||||
## is_orig: If true, threshold is removed for packets from originator, otherwhise for packets from responder.
|
||||
##
|
||||
## Returns: T on success, F on failure.
|
||||
##
|
||||
## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_bytes_threshold set_packets_threshold
|
||||
## delete_packets_threshold
|
||||
global delete_bytes_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
|
||||
|
||||
## Deletes a packet threshold for connection sizes.
|
||||
##
|
||||
## cid: The connection id.
|
||||
##
|
||||
## threshold: Threshold in packets.
|
||||
##
|
||||
## is_orig: If true, threshold is removed for packets from originator, otherwise for packets from responder.
|
||||
##
|
||||
## Returns: T on success, F on failure.
|
||||
##
|
||||
## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_bytes_threshold set_packets_threshold
|
||||
## delete_bytes_threshold
|
||||
global delete_packets_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
|
||||
|
||||
## Generated for a connection that crossed a set byte threshold
|
||||
##
|
||||
## c: the connection
|
||||
##
|
||||
## threshold: the threshold that was set
|
||||
##
|
||||
## is_orig: True if the threshold was crossed by the originator of the connection
|
||||
##
|
||||
## .. bro:see:: packets_threshold_crossed set_bytes_threshold set_packets_threshold
|
||||
## delete_bytes_threshold delete_packets_threshold
|
||||
global bytes_threshold_crossed: event(c: connection, threshold: count, is_orig: bool);
|
||||
|
||||
## Generated for a connection that crossed a set byte threshold
|
||||
##
|
||||
## c: the connection
|
||||
##
|
||||
## threshold: the threshold that was set
|
||||
##
|
||||
## is_orig: True if the threshold was crossed by the originator of the connection
|
||||
##
|
||||
## .. bro:see:: bytes_threshold_crossed set_bytes_threshold set_packets_threshold
|
||||
## delete_bytes_threshold delete_packets_threshold
|
||||
global packets_threshold_crossed: event(c: connection, threshold: count, is_orig: bool);
|
||||
}
|
||||
|
||||
redef record connection += {
|
||||
thresholds: ConnThreshold::Thresholds &optional;
|
||||
};
|
||||
|
||||
function set_conn(c: connection)
|
||||
{
|
||||
if ( c?$thresholds )
|
||||
return;
|
||||
|
||||
c$thresholds = Thresholds();
|
||||
}
|
||||
|
||||
function find_min_threshold(t: set[count]): count
|
||||
{
|
||||
if ( |t| == 0 )
|
||||
return 0;
|
||||
|
||||
local first = T;
|
||||
local min: count = 0;
|
||||
|
||||
for ( i in t )
|
||||
{
|
||||
if ( first )
|
||||
{
|
||||
min = i;
|
||||
first = F;
|
||||
}
|
||||
else
|
||||
{
|
||||
if ( i < min )
|
||||
min = i;
|
||||
}
|
||||
}
|
||||
|
||||
return min;
|
||||
}
|
||||
|
||||
function set_current_threshold(c: connection, bytes: bool, is_orig: bool): bool
|
||||
{
|
||||
local t: count = 0;
|
||||
local cur: count = 0;
|
||||
|
||||
if ( bytes && is_orig )
|
||||
{
|
||||
t = find_min_threshold(c$thresholds$orig_byte);
|
||||
cur = get_current_conn_bytes_threshold(c$id, is_orig);
|
||||
}
|
||||
else if ( bytes && ! is_orig )
|
||||
{
|
||||
t = find_min_threshold(c$thresholds$resp_byte);
|
||||
cur = get_current_conn_bytes_threshold(c$id, is_orig);
|
||||
}
|
||||
else if ( ! bytes && is_orig )
|
||||
{
|
||||
t = find_min_threshold(c$thresholds$orig_packet);
|
||||
cur = get_current_conn_packets_threshold(c$id, is_orig);
|
||||
}
|
||||
else if ( ! bytes && ! is_orig )
|
||||
{
|
||||
t = find_min_threshold(c$thresholds$resp_packet);
|
||||
cur = get_current_conn_packets_threshold(c$id, is_orig);
|
||||
}
|
||||
|
||||
if ( t == cur )
|
||||
return T;
|
||||
|
||||
if ( bytes && is_orig )
|
||||
return set_current_conn_bytes_threshold(c$id, t, T);
|
||||
else if ( bytes && ! is_orig )
|
||||
return set_current_conn_bytes_threshold(c$id, t, F);
|
||||
else if ( ! bytes && is_orig )
|
||||
return set_current_conn_packets_threshold(c$id, t, T);
|
||||
else if ( ! bytes && ! is_orig )
|
||||
return set_current_conn_packets_threshold(c$id, t, F);
|
||||
}
|
||||
|
||||
function set_bytes_threshold(c: connection, threshold: count, is_orig: bool): bool
|
||||
{
|
||||
set_conn(c);
|
||||
|
||||
if ( threshold == 0 )
|
||||
return F;
|
||||
|
||||
if ( is_orig )
|
||||
add c$thresholds$orig_byte[threshold];
|
||||
else
|
||||
add c$thresholds$resp_byte[threshold];
|
||||
|
||||
return set_current_threshold(c, T, is_orig);
|
||||
}
|
||||
|
||||
function set_packets_threshold(c: connection, threshold: count, is_orig: bool): bool
|
||||
{
|
||||
set_conn(c);
|
||||
|
||||
if ( threshold == 0 )
|
||||
return F;
|
||||
|
||||
if ( is_orig )
|
||||
add c$thresholds$orig_packet[threshold];
|
||||
else
|
||||
add c$thresholds$resp_packet[threshold];
|
||||
|
||||
return set_current_threshold(c, F, is_orig);
|
||||
}
|
||||
|
||||
function delete_bytes_threshold(c: connection, threshold: count, is_orig: bool): bool
|
||||
{
|
||||
set_conn(c);
|
||||
|
||||
if ( is_orig && threshold in c$thresholds$orig_byte )
|
||||
{
|
||||
delete c$thresholds$orig_byte[threshold];
|
||||
set_current_threshold(c, T, is_orig);
|
||||
return T;
|
||||
}
|
||||
else if ( ! is_orig && threshold in c$thresholds$resp_byte )
|
||||
{
|
||||
delete c$thresholds$resp_byte[threshold];
|
||||
set_current_threshold(c, T, is_orig);
|
||||
return T;
|
||||
}
|
||||
|
||||
return F;
|
||||
}
|
||||
|
||||
function delete_packets_threshold(c: connection, threshold: count, is_orig: bool): bool
|
||||
{
|
||||
set_conn(c);
|
||||
|
||||
if ( is_orig && threshold in c$thresholds$orig_packet )
|
||||
{
|
||||
delete c$thresholds$orig_packet[threshold];
|
||||
set_current_threshold(c, F, is_orig);
|
||||
return T;
|
||||
}
|
||||
else if ( ! is_orig && threshold in c$thresholds$resp_packet )
|
||||
{
|
||||
delete c$thresholds$resp_packet[threshold];
|
||||
set_current_threshold(c, F, is_orig);
|
||||
return T;
|
||||
}
|
||||
|
||||
return F;
|
||||
}
|
||||
|
||||
event conn_bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool) &priority=5
|
||||
{
|
||||
if ( is_orig && threshold in c$thresholds$orig_byte )
|
||||
{
|
||||
delete c$thresholds$orig_byte[threshold];
|
||||
event ConnThreshold::bytes_threshold_crossed(c, threshold, is_orig);
|
||||
}
|
||||
else if ( ! is_orig && threshold in c$thresholds$resp_byte )
|
||||
{
|
||||
delete c$thresholds$resp_byte[threshold];
|
||||
event ConnThreshold::bytes_threshold_crossed(c, threshold, is_orig);
|
||||
}
|
||||
|
||||
set_current_threshold(c, T, is_orig);
|
||||
}
|
||||
|
||||
event conn_packets_threshold_crossed(c: connection, threshold: count, is_orig: bool) &priority=5
|
||||
{
|
||||
if ( is_orig && threshold in c$thresholds$orig_packet )
|
||||
{
|
||||
delete c$thresholds$orig_packet[threshold];
|
||||
event ConnThreshold::packets_threshold_crossed(c, threshold, is_orig);
|
||||
}
|
||||
else if ( ! is_orig && threshold in c$thresholds$resp_packet )
|
||||
{
|
||||
delete c$thresholds$resp_packet[threshold];
|
||||
event ConnThreshold::packets_threshold_crossed(c, threshold, is_orig);
|
||||
}
|
||||
|
||||
set_current_threshold(c, F, is_orig);
|
||||
}
|
||||
|
|
@ -11,13 +11,13 @@
|
|||
##! GridFTP data channels are identified by a heuristic that relies on
|
||||
##! the fact that default settings for GridFTP clients typically
|
||||
##! mutually authenticate the data channel with TLS/SSL and negotiate a
|
||||
##! NULL bulk cipher (no encryption). Connections with those
|
||||
##! attributes are then polled for two minutes with decreasing frequency
|
||||
##! to check if the transfer sizes are large enough to indicate a
|
||||
##! GridFTP data channel that would be undesirable to analyze further
|
||||
##! (e.g. stop TCP reassembly). A side effect is that true connection
|
||||
##! sizes are not logged, but at the benefit of saving CPU cycles that
|
||||
##! would otherwise go to analyzing the large (and likely benign) connections.
|
||||
##! NULL bulk cipher (no encryption). Connections with those attributes
|
||||
##! are marked as GridFTP if the data transfer within the first two minutes
|
||||
##! is big enough to indicate a GripFTP data channel that would be
|
||||
##! undesirable to analyze further (e.g. stop TCP reassembly). A side
|
||||
##! effect is that true connection sizes are not logged, but at the benefit
|
||||
##! of saving CPU cycles that would otherwise go to analyzing the large
|
||||
##! (and likely benign) connections.
|
||||
|
||||
@load ./info
|
||||
@load ./main
|
||||
|
|
@ -32,23 +32,14 @@ export {
|
|||
## GridFTP data channel.
|
||||
const size_threshold = 1073741824 &redef;
|
||||
|
||||
## Max number of times to check whether a connection's size exceeds the
|
||||
## Time during which we check whether a connection's size exceeds the
|
||||
## :bro:see:`GridFTP::size_threshold`.
|
||||
const max_poll_count = 15 &redef;
|
||||
const max_time = 2 min &redef;
|
||||
|
||||
## Whether to skip further processing of the GridFTP data channel once
|
||||
## detected, which may help performance.
|
||||
const skip_data = T &redef;
|
||||
|
||||
## Base amount of time between checking whether a GridFTP data connection
|
||||
## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
|
||||
const poll_interval = 1sec &redef;
|
||||
|
||||
## The amount of time the base :bro:see:`GridFTP::poll_interval` is
|
||||
## increased by each poll interval. Can be used to make more frequent
|
||||
## checks at the start of a connection and gradually slow down.
|
||||
const poll_interval_increase = 1sec &redef;
|
||||
|
||||
## Raised when a GridFTP data channel is detected.
|
||||
##
|
||||
## c: The connection pertaining to the GridFTP data channel.
|
||||
|
|
@ -79,23 +70,27 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4
|
|||
c$ftp$last_auth_requested = arg;
|
||||
}
|
||||
|
||||
function size_callback(c: connection, cnt: count): interval
|
||||
event ConnThreshold::bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool)
|
||||
{
|
||||
if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
|
||||
if ( threshold < size_threshold || "gridftp-data" in c$service || c$duration > max_time )
|
||||
return;
|
||||
|
||||
add c$service["gridftp-data"];
|
||||
event GridFTP::data_channel_detected(c);
|
||||
|
||||
if ( skip_data )
|
||||
skip_further_processing(c$id);
|
||||
}
|
||||
|
||||
event gridftp_possibility_timeout(c: connection)
|
||||
{
|
||||
# only remove if we did not already detect it and the connection
|
||||
# is not yet at its end.
|
||||
if ( "gridftp-data" !in c$service && ! c$conn?$service )
|
||||
{
|
||||
add c$service["gridftp-data"];
|
||||
event GridFTP::data_channel_detected(c);
|
||||
|
||||
if ( skip_data )
|
||||
skip_further_processing(c$id);
|
||||
|
||||
return -1sec;
|
||||
ConnThreshold::delete_bytes_threshold(c, size_threshold, T);
|
||||
ConnThreshold::delete_bytes_threshold(c, size_threshold, F);
|
||||
}
|
||||
|
||||
if ( cnt >= max_poll_count )
|
||||
return -1sec;
|
||||
|
||||
return poll_interval + poll_interval_increase * cnt;
|
||||
}
|
||||
|
||||
event ssl_established(c: connection) &priority=5
|
||||
|
|
@ -118,5 +113,9 @@ event ssl_established(c: connection) &priority=-3
|
|||
# By default GridFTP data channels do mutual authentication and
|
||||
# negotiate a cipher suite with a NULL bulk cipher.
|
||||
if ( data_channel_initial_criteria(c) )
|
||||
ConnPolling::watch(c, size_callback, 0, 0secs);
|
||||
{
|
||||
ConnThreshold::set_bytes_threshold(c, size_threshold, T);
|
||||
ConnThreshold::set_bytes_threshold(c, size_threshold, F);
|
||||
schedule max_time { gridftp_possibility_timeout(c) };
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue