Merge remote-tracking branch 'origin/master' into topic/johanna/openflow

scripts/base/frameworks/logging/postprocessors/sftp.bro

@@ -37,6 +37,8 @@ export {
 		user: string;
 		## The remote host to which to transfer logs.
 		host: string;
+		## The port to connect to. Defaults to 22
+		host_port: count &default=22;
 		## The path/directory on the remote host to send logs.
 		path: string;
 	};
@@ -63,8 +65,8 @@ function sftp_postprocessor(info: Log::RotationInfo): bool
 		{
 		local dst = fmt("%s/%s.%s.log", d$path, info$path,
 		                strftime(Log::sftp_rotation_date_format, info$open));
-		command += fmt("echo put %s %s | sftp -b - %s@%s;", info$fname, dst,
-		               d$user, d$host);
+		command += fmt("echo put %s %s | sftp -P %d -b - %s@%s;", info$fname, dst,
+		               d$host_port, d$user, d$host);
 		}
 
 	command += fmt("/bin/rm %s", info$fname);

scripts/base/init-bare.bro

@@ -3219,6 +3219,11 @@ const forward_remote_events = F &redef;
 ##    more sophisticated script-level communication framework.
 const forward_remote_state_changes = F &redef;
 
+## The number of IO chunks allowed to be buffered between the child
+## and parent process of remote communication before Bro starts dropping
+## connections to remote peers in an attempt to catch up.
+const chunked_io_buffer_soft_cap = 800000 &redef;
+
 ## Place-holder constant indicating "no peer".
 const PEER_ID_NONE = 0;
 

scripts/base/protocols/conn/thresholds.bro

@@ -1,15 +1,15 @@
-##! Implements a generic way to throw events when a connection crosses a
-##! fixed threshold of bytes or packets
+##! Implements a generic API to throw events when a connection crosses a
+##! fixed threshold of bytes or packets.
 
 module ConnThreshold;
 
 export {
 
 	type Thresholds: record {
-		orig_byte_thresholds: set[count] &default=count_set(); ##< current originator byte thresholds we watch for
-		resp_byte_thresholds: set[count] &default=count_set(); ##< current responder byte thresholds we watch for
-		orig_packet_thresholds: set[count] &default=count_set(); ##< corrent originator packet thresholds we watch for
-		resp_packet_thresholds: set[count] &default=count_set(); ##< corrent responder packet thresholds we watch for
+		orig_byte: set[count] &default=count_set(); ##< current originator byte thresholds we watch for
+		resp_byte: set[count] &default=count_set(); ##< current responder byte thresholds we watch for
+		orig_packet: set[count] &default=count_set(); ##< corrent originator packet thresholds we watch for
+		resp_packet: set[count] &default=count_set(); ##< corrent responder packet thresholds we watch for
 	};
 
 	## Sets a byte threshold for connection sizes, adding it to potentially already existing thresholds.
@@ -19,12 +19,9 @@ export {
 	##
 	## threshold: Threshold in bytes.
 	##
-	## is_orig: If true, threshold is set for bytes from originator, otherwhise for bytes from responder.
+	## is_orig: If true, threshold is set for bytes from originator, otherwise for bytes from responder.
 	##
 	## Returns: T on success, F on failure.
-	##
-	## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_packets_threshold
-	##              delete_bytes_threshold delete_packets_threshold
 	global set_bytes_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
 
 	## Sets a packet threshold for connection sizes, adding it to potentially already existing thresholds.
@@ -34,12 +31,9 @@ export {
 	##
 	## threshold: Threshold in packets.
 	##
-	## is_orig: If true, threshold is set for packets from originator, otherwhise for packets from responder.
+	## is_orig: If true, threshold is set for packets from originator, otherwise for packets from responder.
 	##
 	## Returns: T on success, F on failure.
-	##
-	## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_bytes_threshold
-	##              delete_bytes_threshold delete_packets_threshold
 	global set_packets_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
 
 	## Deletes a byte threshold for connection sizes.
@@ -51,9 +45,6 @@ export {
 	## is_orig: If true, threshold is removed for packets from originator, otherwhise for packets from responder.
 	##
 	## Returns: T on success, F on failure.
-	##
-	## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_bytes_threshold set_packets_threshold
-	##              delete_packets_threshold
 	global delete_bytes_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
 
 	## Deletes a packet threshold for connection sizes.
@@ -62,12 +53,9 @@ export {
 	##
 	## threshold: Threshold in packets.
 	##
-	## is_orig: If true, threshold is removed for packets from originator, otherwhise for packets from responder.
+	## is_orig: If true, threshold is removed for packets from originator, otherwise for packets from responder.
 	##
 	## Returns: T on success, F on failure.
-	##
-	## .. bro:see:: bytes_threshold_crossed packets_threshold_crossed set_bytes_threshold set_packets_threshold
-	##              delete_bytes_threshold
 	global delete_packets_threshold: function(c: connection, threshold: count, is_orig: bool): bool;
 
 	## Generated for a connection that crossed a set byte threshold
@@ -77,9 +65,6 @@ export {
 	## threshold: the threshold that was set
 	##
 	## is_orig: True if the threshold was crossed by the originator of the connection
-	##
-	## .. bro:see:: packets_threshold_crossed set_bytes_threshold set_packets_threshold
-	##              delete_bytes_threshold delete_packets_threshold
 	global bytes_threshold_crossed: event(c: connection, threshold: count, is_orig: bool);
 
 	## Generated for a connection that crossed a set byte threshold
@@ -89,9 +74,6 @@ export {
 	## threshold: the threshold that was set
 	##
 	## is_orig: True if the threshold was crossed by the originator of the connection
-	##
-	## .. bro:see:: bytes_threshold_crossed  set_bytes_threshold set_packets_threshold
-	##              delete_bytes_threshold delete_packets_threshold
 	global packets_threshold_crossed: event(c: connection, threshold: count, is_orig: bool);
 }
 
@@ -99,7 +81,7 @@ redef record connection += {
 	thresholds: ConnThreshold::Thresholds &optional;
 };
 
-function set_conn_thresholds(c: connection)
+function set_conn(c: connection)
 	{
 	if ( c?$thresholds )
 		return;
@@ -139,22 +121,22 @@ function set_current_threshold(c: connection, bytes: bool, is_orig: bool): bool
 
 	if ( bytes && is_orig )
 		{
-		t = find_min_threshold(c$thresholds$orig_byte_thresholds);
+		t = find_min_threshold(c$thresholds$orig_byte);
 		cur = get_current_conn_bytes_threshold(c$id, is_orig);
 		}
 	else if ( bytes && ! is_orig )
 		{
-		t = find_min_threshold(c$thresholds$resp_byte_thresholds);
+		t = find_min_threshold(c$thresholds$resp_byte);
 		cur = get_current_conn_bytes_threshold(c$id, is_orig);
 		}
 	else if ( ! bytes && is_orig )
 		{
-		t = find_min_threshold(c$thresholds$orig_packet_thresholds);
+		t = find_min_threshold(c$thresholds$orig_packet);
 		cur = get_current_conn_packets_threshold(c$id, is_orig);
 		}
 	else if ( ! bytes && ! is_orig )
 		{
-		t = find_min_threshold(c$thresholds$resp_packet_thresholds);
+		t = find_min_threshold(c$thresholds$resp_packet);
 		cur = get_current_conn_packets_threshold(c$id, is_orig);
 		}
 
@@ -173,47 +155,47 @@ function set_current_threshold(c: connection, bytes: bool, is_orig: bool): bool
 
 function set_bytes_threshold(c: connection, threshold: count, is_orig: bool): bool
 	{
-	set_conn_thresholds(c);
+	set_conn(c);
 
 	if ( threshold == 0 )
 		return F;
 
 	if ( is_orig )
-		add c$thresholds$orig_byte_thresholds[threshold];
+		add c$thresholds$orig_byte[threshold];
 	else
-		add c$thresholds$resp_byte_thresholds[threshold];
+		add c$thresholds$resp_byte[threshold];
 
 	return set_current_threshold(c, T, is_orig);
 	}
 
 function set_packets_threshold(c: connection, threshold: count, is_orig: bool): bool
 	{
-	set_conn_thresholds(c);
+	set_conn(c);
 
 	if ( threshold == 0 )
 		return F;
 
 	if ( is_orig )
-		add c$thresholds$orig_packet_thresholds[threshold];
+		add c$thresholds$orig_packet[threshold];
 	else
-		add c$thresholds$resp_packet_thresholds[threshold];
+		add c$thresholds$resp_packet[threshold];
 
 	return set_current_threshold(c, F, is_orig);
 	}
 
 function delete_bytes_threshold(c: connection, threshold: count, is_orig: bool): bool
 	{
-	set_conn_thresholds(c);
+	set_conn(c);
 
-	if ( is_orig && threshold in c$thresholds$orig_byte_thresholds )
+	if ( is_orig && threshold in c$thresholds$orig_byte )
 		{
-		delete c$thresholds$orig_byte_thresholds[threshold];
+		delete c$thresholds$orig_byte[threshold];
 		set_current_threshold(c, T, is_orig);
 		return T;
 		}
-	else if ( ! is_orig && threshold in c$thresholds$resp_byte_thresholds )
+	else if ( ! is_orig && threshold in c$thresholds$resp_byte )
 		{
-		delete c$thresholds$resp_byte_thresholds[threshold];
+		delete c$thresholds$resp_byte[threshold];
 		set_current_threshold(c, T, is_orig);
 		return T;
 		}
@@ -223,17 +205,17 @@ function delete_bytes_threshold(c: connection, threshold: count, is_orig: bool):
 
 function delete_packets_threshold(c: connection, threshold: count, is_orig: bool): bool
 	{
-	set_conn_thresholds(c);
+	set_conn(c);
 
-	if ( is_orig && threshold in c$thresholds$orig_packet_thresholds )
+	if ( is_orig && threshold in c$thresholds$orig_packet )
 		{
-		delete c$thresholds$orig_packet_thresholds[threshold];
+		delete c$thresholds$orig_packet[threshold];
 		set_current_threshold(c, F, is_orig);
 		return T;
 		}
-	else if ( ! is_orig && threshold in c$thresholds$resp_packet_thresholds )
+	else if ( ! is_orig && threshold in c$thresholds$resp_packet )
 		{
-		delete c$thresholds$resp_packet_thresholds[threshold];
+		delete c$thresholds$resp_packet[threshold];
 		set_current_threshold(c, F, is_orig);
 		return T;
 		}
@@ -243,14 +225,14 @@ function delete_packets_threshold(c: connection, threshold: count, is_orig: bool
 
 event conn_bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool) &priority=5
 	{
-	if ( is_orig && threshold in c$thresholds$orig_byte_thresholds )
+	if ( is_orig && threshold in c$thresholds$orig_byte )
 		{
-		delete c$thresholds$orig_byte_thresholds[threshold];
+		delete c$thresholds$orig_byte[threshold];
 		event ConnThreshold::bytes_threshold_crossed(c, threshold, is_orig);
 		}
-	else if ( ! is_orig && threshold in c$thresholds$resp_byte_thresholds )
+	else if ( ! is_orig && threshold in c$thresholds$resp_byte )
 		{
-		delete c$thresholds$resp_byte_thresholds[threshold];
+		delete c$thresholds$resp_byte[threshold];
 		event ConnThreshold::bytes_threshold_crossed(c, threshold, is_orig);
 		}
 
@@ -259,14 +241,14 @@ event conn_bytes_threshold_crossed(c: connection, threshold: count, is_orig: boo
 
 event conn_packets_threshold_crossed(c: connection, threshold: count, is_orig: bool) &priority=5
 	{
-	if ( is_orig && threshold in c$thresholds$orig_packet_thresholds )
+	if ( is_orig && threshold in c$thresholds$orig_packet )
 		{
-		delete c$thresholds$orig_packet_thresholds[threshold];
+		delete c$thresholds$orig_packet[threshold];
 		event ConnThreshold::packets_threshold_crossed(c, threshold, is_orig);
 		}
-	else if ( ! is_orig && threshold in c$thresholds$resp_packet_thresholds )
+	else if ( ! is_orig && threshold in c$thresholds$resp_packet )
 		{
-		delete c$thresholds$resp_packet_thresholds[threshold];
+		delete c$thresholds$resp_packet[threshold];
 		event ConnThreshold::packets_threshold_crossed(c, threshold, is_orig);
 		}
 

scripts/base/protocols/ftp/gridftp.bro

@@ -11,13 +11,13 @@
 ##! GridFTP data channels are identified by a heuristic that relies on
 ##! the fact that default settings for GridFTP clients typically
 ##! mutually authenticate the data channel with TLS/SSL and negotiate a
-##! NULL bulk cipher (no encryption).  Connections with those
-##! attributes are then polled for two minutes with decreasing frequency
-##! to check if the transfer sizes are large enough to indicate a
-##! GridFTP data channel that would be undesirable to analyze further
-##! (e.g. stop TCP reassembly).  A side effect is that true connection
-##! sizes are not logged, but at the benefit of saving CPU cycles that
-##! would otherwise go to analyzing the large (and likely benign) connections.
+##! NULL bulk cipher (no encryption). Connections with those attributes
+##! are marked as GridFTP if the data transfer within the first two minutes
+##! is big enough to indicate a GripFTP data channel that would be
+##! undesirable to analyze further (e.g. stop TCP reassembly).  A side
+##! effect is that true connection sizes are not logged, but at the benefit
+##! of saving CPU cycles that would otherwise go to analyzing the large
+##! (and likely benign) connections.
 
 @load ./info
 @load ./main
@@ -32,23 +32,14 @@ export {
 	## GridFTP data channel.
 	const size_threshold = 1073741824 &redef;
 
-	## Max number of times to check whether a connection's size exceeds the
+	## Time during which we check whether a connection's size exceeds the
 	## :bro:see:`GridFTP::size_threshold`.
-	const max_poll_count = 15 &redef;
+	const max_time = 2 min &redef;
 
 	## Whether to skip further processing of the GridFTP data channel once
 	## detected, which may help performance.
 	const skip_data = T &redef;
 
-	## Base amount of time between checking whether a GridFTP data connection
-	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
-	const poll_interval = 1sec &redef;
-
-	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
-	## increased by each poll interval.  Can be used to make more frequent
-	## checks at the start of a connection and gradually slow down.
-	const poll_interval_increase = 1sec &redef;
-
 	## Raised when a GridFTP data channel is detected.
 	##
 	## c: The connection pertaining to the GridFTP data channel.
@@ -79,23 +70,27 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4
 		c$ftp$last_auth_requested = arg;
 	}
 
-function size_callback(c: connection, cnt: count): interval
+event ConnThreshold::bytes_threshold_crossed(c: connection, threshold: count, is_orig: bool)
 	{
-	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+	if ( threshold < size_threshold || "gridftp-data" in c$service || c$duration > max_time )
+		return;
+
+	add c$service["gridftp-data"];
+	event GridFTP::data_channel_detected(c);
+
+	if ( skip_data )
+		skip_further_processing(c$id);
+	}
+
+event gridftp_possibility_timeout(c: connection)
+	{
+	# only remove if we did not already detect it and the connection
+	# is not yet at its end.
+	if ( "gridftp-data" !in c$service && ! c$conn?$service )
 		{
-		add c$service["gridftp-data"];
-		event GridFTP::data_channel_detected(c);
-
-		if ( skip_data )
-			skip_further_processing(c$id);
-
-		return -1sec;
+		ConnThreshold::delete_bytes_threshold(c, size_threshold, T);
+		ConnThreshold::delete_bytes_threshold(c, size_threshold, F);
 		}
-
-	if ( cnt >= max_poll_count )
-		return -1sec;
-
-	return poll_interval + poll_interval_increase * cnt;
 	}
 
 event ssl_established(c: connection) &priority=5
@@ -118,5 +113,9 @@ event ssl_established(c: connection) &priority=-3
 	# By default GridFTP data channels do mutual authentication and
 	# negotiate a cipher suite with a NULL bulk cipher.
 	if ( data_channel_initial_criteria(c) )
-		ConnPolling::watch(c, size_callback, 0, 0secs);
+		{
+		ConnThreshold::set_bytes_threshold(c, size_threshold, T);
+		ConnThreshold::set_bytes_threshold(c, size_threshold, F);
+		schedule max_time { gridftp_possibility_timeout(c) };
+		}
 	}