Merge remote-tracking branch 'origin/topic/jsiwek/gridftp'

* origin/topic/jsiwek/gridftp:
  Add memory leak unit test for GridFTP.
  Enable GridFTP detection by default.  Track/log SSL client certs.
  Add analyzer for GSI mechanism of GSSAPI FTP AUTH method.
  Add an example of a GridFTP data channel detection script.

scripts/base/protocols/conn/__load__.bro

@@ -1,3 +1,4 @@
 @load ./main
 @load ./contents
 @load ./inactivity
+@load ./polling

scripts/base/protocols/conn/polling.bro

@@ -0,0 +1,49 @@
+##! Implements a generic way to poll connections looking for certain features
+##! (e.g. monitor bytes transferred).  The specific feature of a connection
+##! to look for, the polling interval, and the code to execute if the feature
+##! is found are all controlled by user-defined callback functions.
+
+module ConnPolling;
+
+export {
+	## Starts monitoring a given connection.
+	##
+	## c: The connection to watch.
+	##
+	## callback: A callback function that takes as arguments the monitored
+	##           *connection*, and counter *cnt* that increments each time the
+	##           callback is called.  It returns an interval indicating how long
+	##           in the future to schedule an event which will call the
+	##           callback.  A negative return interval causes polling to stop.
+	##
+	## cnt: The initial value of a counter which gets passed to *callback*.
+	##
+	## i: The initial interval at which to schedule the next callback.
+	##    May be ``0secs`` to poll right away.
+	global watch: function(c: connection,
+			       callback: function(c: connection, cnt: count): interval,
+			       cnt: count, i: interval);
+}
+
+event ConnPolling::check(c: connection,
+			 callback: function(c: connection, cnt: count): interval,
+			 cnt: count)
+	{
+	if ( ! connection_exists(c$id) )
+		return;
+
+	lookup_connection(c$id); # updates the conn val
+
+	local next_interval = callback(c, cnt);
+	if ( next_interval < 0secs )
+		return;
+
+	watch(c, callback, cnt + 1, next_interval);
+	}
+
+function watch(c: connection,
+	       callback: function(c: connection, cnt: count): interval,
+	       cnt: count, i: interval)
+	{
+	schedule i { ConnPolling::check(c, callback, cnt) };
+	}

scripts/base/protocols/ftp/__load__.bro

@@ -1,3 +1,4 @@
 @load ./utils-commands
 @load ./main
-@load ./file-extract
+@load ./file-extract
+@load ./gridftp

scripts/base/protocols/ftp/gridftp.bro

@@ -0,0 +1,109 @@
+##! A detection script for GridFTP data and control channels.
+##!
+##! GridFTP control channels are identified by FTP control channels
+##! that successfully negotiate the GSSAPI method of an AUTH request
+##! and for which the exchange involved an encoded TLS/SSL handshake,
+##! indicating the GSI mechanism for GSSAPI was used.  This analysis
+##! is all supported internally, this script simple adds the "gridftp"
+##! label to the *service* field of the control channel's
+##! :bro:type:`connection` record.
+##!
+##! GridFTP data channels are identified by a heuristic that relies on
+##! the fact that default settings for GridFTP clients typically
+##! mutally authenticate the data channel with TLS/SSL and negotiate a
+##! NULL bulk cipher (no encryption).  Connections with those
+##! attributes are then polled for two minutes with decreasing frequency
+##! to check if the transfer sizes are large enough to indicate a
+##! GridFTP data channel that would be undesireable to analyze further
+##! (e.g. stop TCP reassembly).  A side effect is that true connection
+##! sizes are not logged, but at the benefit of saving CPU cycles that
+##! otherwise go to analyzing the large (and likely benign) connections.
+
+@load ./main
+@load base/protocols/conn
+@load base/protocols/ssl
+@load base/frameworks/notice
+
+module GridFTP;
+
+export {
+	## Number of bytes transferred before guessing a connection is a
+	## GridFTP data channel.
+	const size_threshold = 1073741824 &redef;
+
+	## Max number of times to check whether a connection's size exceeds the
+	## :bro:see:`GridFTP::size_threshold`.
+	const max_poll_count = 15 &redef;
+
+	## Whether to skip further processing of the GridFTP data channel once
+	## detected, which may help performance.
+	const skip_data = T &redef;
+
+	## Base amount of time between checking whether a GridFTP data connection
+	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
+	const poll_interval = 1sec &redef;
+
+	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
+	## increased by each poll interval.  Can be used to make more frequent
+	## checks at the start of a connection and gradually slow down.
+	const poll_interval_increase = 1sec &redef;
+
+	## Raised when a GridFTP data channel is detected.
+	##
+	## c: The connection pertaining to the GridFTP data channel.
+	global data_channel_detected: event(c: connection);
+
+	## The initial criteria used to determine whether to start polling
+	## the connection for the :bro:see:`GridFTP::size_threshold` to have
+	## been exceeded.  This is called in a :bro:see:`ssl_established` event
+	## handler and by default looks for both a client and server certificate
+	## and for a NULL bulk cipher.  One way in which this function could be
+	## redefined is to make it also consider client/server certificate issuer
+	## subjects.
+	##
+	## c: The connection which may possibly be a GridFTP data channel.
+	##
+	## Returns: true if the connection should be further polled for an
+	##          exceeded :bro:see:`GridFTP::size_threshold`, else false.
+	const data_channel_initial_criteria: function(c: connection): bool &redef;
+}
+
+function size_callback(c: connection, cnt: count): interval
+	{
+	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+		{
+		add c$service["gridftp-data"];
+		event GridFTP::data_channel_detected(c);
+
+		if ( skip_data )
+			skip_further_processing(c$id);
+
+		return -1sec;
+		}
+
+	if ( cnt >= max_poll_count )
+		return -1sec;
+
+	return poll_interval + poll_interval_increase * cnt;
+	}
+
+event ssl_established(c: connection) &priority=5
+	{
+	# Add service label to control channels.
+	if ( "FTP" in c$service )
+		add c$service["gridftp"];
+	}
+
+function data_channel_initial_criteria(c: connection): bool
+	{
+	return ( c?$ssl && c$ssl?$client_subject && c$ssl?$subject &&
+	         c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher );
+	}
+
+event ssl_established(c: connection) &priority=-3
+	{
+	# By default GridFTP data channels do mutual authentication and
+	# negotiate a cipher suite with a NULL bulk cipher.
+	if ( data_channel_initial_criteria(c) )
+		ConnPolling::watch(c, size_callback, 0, 0secs);
+	}

scripts/base/protocols/ftp/main.bro

@@ -96,11 +96,11 @@ redef record connection += {
 };
 
 # Configure DPD
-const ports = { 21/tcp } &redef;
-redef capture_filters += { ["ftp"] = "port 21" };
+const ports = { 21/tcp, 2811/tcp } &redef; # 2811/tcp is GridFTP.
+redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
 redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
 
-redef likely_server_ports += { 21/tcp };
+redef likely_server_ports += { 21/tcp, 2811/tcp };
 
 # Establish the variable for tracking expected connections.
 global ftp_data_expected: table[addr, port] of Info &create_expire=5mins;

scripts/base/protocols/ssl/main.bro

@@ -19,7 +19,7 @@ export {
 		version:          string           &log &optional;
 		## SSL/TLS cipher suite that the server chose.
 		cipher:           string           &log &optional;
-		## Value of the Server Name Indicator SSL/TLS extension.  It 
+		## Value of the Server Name Indicator SSL/TLS extension.  It
 		## indicates the server name that the client was requesting.
 		server_name:      string           &log &optional;
 		## Session ID offered by the client for session resumption.
@@ -30,37 +30,48 @@ export {
 		issuer_subject:   string           &log &optional;
 		## NotValidBefore field value from the server certificate.
 		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the serve certificate.
+		## NotValidAfter field value from the server certificate.
 		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
-		
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the client.
+		client_issuer_subject:   string           &log &optional;
+
 		## Full binary server certificate stored in DER format.
 		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its 
+		## Chain of certificates offered by the server to validate its
 		## complete signing chain.
 		cert_chain:       vector of string &optional;
 
+		## Full binary client certificate stored in DER format.
+		client_cert:             string           &optional;
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain:       vector of string &optional;
+
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
 		analyzer_id:      count            &optional;
 	};
-	
+
 	## The default root CA bundle.  By loading the
 	## mozilla-ca-list.bro script it will be set to Mozilla's root CA list.
 	const root_certs: table[string] of string = {} &redef;
-	
+
 	## If true, detach the SSL analyzer from the connection to prevent
 	## continuing to process encrypted traffic. Helps with performance
 	## (especially with large file transfers).
 	const disable_analyzer_after_detection = T &redef;
-	
+
 	## The openssl command line utility.  If it's in the path the default
 	## value will work, otherwise a full path string can be supplied for the
 	## utility.
 	const openssl_util = "openssl" &redef;
-	
+
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
@@ -107,7 +118,8 @@ redef likely_server_ports += {
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
+		         $client_cert_chain=vector()];
 	}
 
 function finish(c: connection)
@@ -141,23 +153,40 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 
 	# We aren't doing anything with client certificates yet.
 	if ( is_orig )
-		return;
-
-	if ( chain_idx == 0 )
 		{
-		# Save the primary cert.
-		c$ssl$cert = der_cert;
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$client_cert = der_cert;
 
-		# Also save other certificate information about the primary cert.
-		c$ssl$subject = cert$subject;
-		c$ssl$issuer_subject = cert$issuer;
-		c$ssl$not_valid_before = cert$not_valid_before;
-		c$ssl$not_valid_after = cert$not_valid_after;
+			# Also save other certificate information about the primary cert.
+			c$ssl$client_subject = cert$subject;
+			c$ssl$client_issuer_subject = cert$issuer;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
+			}
 		}
 	else
 		{
-		# Otherwise, add it to the cert validation chain.
-		c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$cert = der_cert;
+
+			# Also save other certificate information about the primary cert.
+			c$ssl$subject = cert$subject;
+			c$ssl$issuer_subject = cert$issuer;
+			c$ssl$not_valid_before = cert$not_valid_before;
+			c$ssl$not_valid_after = cert$not_valid_after;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
+			}
 		}
 	}