Merge remote-tracking branch 'origin/topic/jsiwek/gridftp'

* origin/topic/jsiwek/gridftp:
  Add memory leak unit test for GridFTP.
  Enable GridFTP detection by default.  Track/log SSL client certs.
  Add analyzer for GSI mechanism of GSSAPI FTP AUTH method.
  Add an example of a GridFTP data channel detection script.

CHANGES

@@ -1,4 +1,37 @@
 
+2.1-76 | 2012-10-12 10:32:39 -0700
+
+  * Add support for recognizing GridFTP connections as an extension to
+    the standard FTP analyzer. (Jon Siwek)
+
+    This is enabled by default and includes:
+
+      - An analyzer for GSI mechanism of GSSAPI FTP AUTH method. GSI
+        authentication involves an encoded TLS/SSL handshake over the
+        FTP control session. For FTP sessions that attempt GSI
+        authentication, the *service* field of the connection log will
+        include "gridftp" (as well as also "ftp" and "ssl").
+
+      - Add an example of a GridFTP data channel detection script. It
+        relies on the heuristics of GridFTP data channels commonly
+        default to SSL mutual authentication with a NULL bulk cipher
+        and that they usually transfer large datasets (default
+        threshold of script is 1 GB).  The script also defaults to
+        skip_further_processing() after detection to try to save
+        cycles analyzing the large, benign connection.
+
+        For identified GridFTP data channels, the *services* fields of
+        the connection log will include "gridftp-data".
+
+  * Add *client_subject* and *client_issuer_subject* as &log'd fields
+    to SSL::Info record.  Also add *client_cert* and
+    *client_cert_chain* fields to track client cert chain. (Jon Siwek)
+
+  * Add a script in base/protocols/conn/polling that generalizes the
+    process of polling a connection for interesting features. The
+    GridFTP data channel detection script depends on it to monitor
+    bytes transferred. (Jon Siwek)
+
 2.1-68 | 2012-10-12 09:46:41 -0700
 
   * Rename the Input Framework's update_finished event to end_of_data.

NEWS

@@ -13,7 +13,9 @@ Bro 2.2
 New Functionality
 ~~~~~~~~~~~~~~~~~
 
-- TODO: Update.
+- GridFTP support. TODO: Extend.
+
+- ssl.log now also records the subject client and issuer certificates.
 
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

VERSION

@@ -1 +1 @@
-2.1-68
+2.1-76

doc/scripts/DocSourcesList.cmake

@@ -65,9 +65,11 @@ rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
+rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/gridftp.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
 rest_target(${psd} base/protocols/http/file-extract.bro)

scripts/base/protocols/conn/__load__.bro

@@ -1,3 +1,4 @@
 @load ./main
 @load ./contents
 @load ./inactivity
+@load ./polling

scripts/base/protocols/conn/polling.bro

@@ -0,0 +1,49 @@
+##! Implements a generic way to poll connections looking for certain features
+##! (e.g. monitor bytes transferred).  The specific feature of a connection
+##! to look for, the polling interval, and the code to execute if the feature
+##! is found are all controlled by user-defined callback functions.
+
+module ConnPolling;
+
+export {
+	## Starts monitoring a given connection.
+	##
+	## c: The connection to watch.
+	##
+	## callback: A callback function that takes as arguments the monitored
+	##           *connection*, and counter *cnt* that increments each time the
+	##           callback is called.  It returns an interval indicating how long
+	##           in the future to schedule an event which will call the
+	##           callback.  A negative return interval causes polling to stop.
+	##
+	## cnt: The initial value of a counter which gets passed to *callback*.
+	##
+	## i: The initial interval at which to schedule the next callback.
+	##    May be ``0secs`` to poll right away.
+	global watch: function(c: connection,
+			       callback: function(c: connection, cnt: count): interval,
+			       cnt: count, i: interval);
+}
+
+event ConnPolling::check(c: connection,
+			 callback: function(c: connection, cnt: count): interval,
+			 cnt: count)
+	{
+	if ( ! connection_exists(c$id) )
+		return;
+
+	lookup_connection(c$id); # updates the conn val
+
+	local next_interval = callback(c, cnt);
+	if ( next_interval < 0secs )
+		return;
+
+	watch(c, callback, cnt + 1, next_interval);
+	}
+
+function watch(c: connection,
+	       callback: function(c: connection, cnt: count): interval,
+	       cnt: count, i: interval)
+	{
+	schedule i { ConnPolling::check(c, callback, cnt) };
+	}

scripts/base/protocols/ftp/__load__.bro

@@ -1,3 +1,4 @@
 @load ./utils-commands
 @load ./main
 @load ./file-extract
+@load ./gridftp

scripts/base/protocols/ftp/gridftp.bro

@@ -0,0 +1,109 @@
+##! A detection script for GridFTP data and control channels.
+##!
+##! GridFTP control channels are identified by FTP control channels
+##! that successfully negotiate the GSSAPI method of an AUTH request
+##! and for which the exchange involved an encoded TLS/SSL handshake,
+##! indicating the GSI mechanism for GSSAPI was used.  This analysis
+##! is all supported internally, this script simple adds the "gridftp"
+##! label to the *service* field of the control channel's
+##! :bro:type:`connection` record.
+##!
+##! GridFTP data channels are identified by a heuristic that relies on
+##! the fact that default settings for GridFTP clients typically
+##! mutally authenticate the data channel with TLS/SSL and negotiate a
+##! NULL bulk cipher (no encryption).  Connections with those
+##! attributes are then polled for two minutes with decreasing frequency
+##! to check if the transfer sizes are large enough to indicate a
+##! GridFTP data channel that would be undesireable to analyze further
+##! (e.g. stop TCP reassembly).  A side effect is that true connection
+##! sizes are not logged, but at the benefit of saving CPU cycles that
+##! otherwise go to analyzing the large (and likely benign) connections.
+
+@load ./main
+@load base/protocols/conn
+@load base/protocols/ssl
+@load base/frameworks/notice
+
+module GridFTP;
+
+export {
+	## Number of bytes transferred before guessing a connection is a
+	## GridFTP data channel.
+	const size_threshold = 1073741824 &redef;
+
+	## Max number of times to check whether a connection's size exceeds the
+	## :bro:see:`GridFTP::size_threshold`.
+	const max_poll_count = 15 &redef;
+
+	## Whether to skip further processing of the GridFTP data channel once
+	## detected, which may help performance.
+	const skip_data = T &redef;
+
+	## Base amount of time between checking whether a GridFTP data connection
+	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
+	const poll_interval = 1sec &redef;
+
+	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
+	## increased by each poll interval.  Can be used to make more frequent
+	## checks at the start of a connection and gradually slow down.
+	const poll_interval_increase = 1sec &redef;
+
+	## Raised when a GridFTP data channel is detected.
+	##
+	## c: The connection pertaining to the GridFTP data channel.
+	global data_channel_detected: event(c: connection);
+
+	## The initial criteria used to determine whether to start polling
+	## the connection for the :bro:see:`GridFTP::size_threshold` to have
+	## been exceeded.  This is called in a :bro:see:`ssl_established` event
+	## handler and by default looks for both a client and server certificate
+	## and for a NULL bulk cipher.  One way in which this function could be
+	## redefined is to make it also consider client/server certificate issuer
+	## subjects.
+	##
+	## c: The connection which may possibly be a GridFTP data channel.
+	##
+	## Returns: true if the connection should be further polled for an
+	##          exceeded :bro:see:`GridFTP::size_threshold`, else false.
+	const data_channel_initial_criteria: function(c: connection): bool &redef;
+}
+
+function size_callback(c: connection, cnt: count): interval
+	{
+	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+		{
+		add c$service["gridftp-data"];
+		event GridFTP::data_channel_detected(c);
+
+		if ( skip_data )
+			skip_further_processing(c$id);
+
+		return -1sec;
+		}
+
+	if ( cnt >= max_poll_count )
+		return -1sec;
+
+	return poll_interval + poll_interval_increase * cnt;
+	}
+
+event ssl_established(c: connection) &priority=5
+	{
+	# Add service label to control channels.
+	if ( "FTP" in c$service )
+		add c$service["gridftp"];
+	}
+
+function data_channel_initial_criteria(c: connection): bool
+	{
+	return ( c?$ssl && c$ssl?$client_subject && c$ssl?$subject &&
+	         c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher );
+	}
+
+event ssl_established(c: connection) &priority=-3
+	{
+	# By default GridFTP data channels do mutual authentication and
+	# negotiate a cipher suite with a NULL bulk cipher.
+	if ( data_channel_initial_criteria(c) )
+		ConnPolling::watch(c, size_callback, 0, 0secs);
+	}

scripts/base/protocols/ftp/main.bro

@@ -96,11 +96,11 @@ redef record connection += {
 };
 
 # Configure DPD
-const ports = { 21/tcp } &redef;
-redef capture_filters += { ["ftp"] = "port 21" };
+const ports = { 21/tcp, 2811/tcp } &redef; # 2811/tcp is GridFTP.
+redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
 redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
 
-redef likely_server_ports += { 21/tcp };
+redef likely_server_ports += { 21/tcp, 2811/tcp };
 
 # Establish the variable for tracking expected connections.
 global ftp_data_expected: table[addr, port] of Info &create_expire=5mins;

scripts/base/protocols/ssl/main.bro

@@ -30,17 +30,28 @@ export {
 		issuer_subject:   string           &log &optional;
 		## NotValidBefore field value from the server certificate.
 		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the serve certificate.
+		## NotValidAfter field value from the server certificate.
 		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the client.
+		client_issuer_subject:   string           &log &optional;
+
 		## Full binary server certificate stored in DER format.
 		cert:             string           &optional;
 		## Chain of certificates offered by the server to validate its
 		## complete signing chain.
 		cert_chain:       vector of string &optional;
 
+		## Full binary client certificate stored in DER format.
+		client_cert:             string           &optional;
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain:       vector of string &optional;
+
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -107,7 +118,8 @@ redef likely_server_ports += {
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
+		         $client_cert_chain=vector()];
 	}
 
 function finish(c: connection)
@@ -141,23 +153,40 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 
 	# We aren't doing anything with client certificates yet.
 	if ( is_orig )
-		return;
-
-	if ( chain_idx == 0 )
 		{
-		# Save the primary cert.
-		c$ssl$cert = der_cert;
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$client_cert = der_cert;
 
-		# Also save other certificate information about the primary cert.
-		c$ssl$subject = cert$subject;
-		c$ssl$issuer_subject = cert$issuer;
-		c$ssl$not_valid_before = cert$not_valid_before;
-		c$ssl$not_valid_after = cert$not_valid_after;
+			# Also save other certificate information about the primary cert.
+			c$ssl$client_subject = cert$subject;
+			c$ssl$client_issuer_subject = cert$issuer;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
+			}
 		}
 	else
 		{
-		# Otherwise, add it to the cert validation chain.
-		c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$cert = der_cert;
+
+			# Also save other certificate information about the primary cert.
+			c$ssl$subject = cert$subject;
+			c$ssl$issuer_subject = cert$issuer;
+			c$ssl$not_valid_before = cert$not_valid_before;
+			c$ssl$not_valid_after = cert$not_valid_after;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
+			}
 		}
 	}
 

src/Analyzer.cc

@@ -171,6 +171,7 @@ const Analyzer::Config Analyzer::analyzer_configs[] = {
 	{ AnalyzerTag::Contents_SMB, "CONTENTS_SMB", 0, 0, 0, false },
 	{ AnalyzerTag::Contents_RPC, "CONTENTS_RPC", 0, 0, 0, false },
 	{ AnalyzerTag::Contents_NFS, "CONTENTS_NFS", 0, 0, 0, false },
+	{ AnalyzerTag::FTP_ADAT, "FTP_ADAT", 0, 0, 0, false },
 };
 
 AnalyzerTimer::~AnalyzerTimer()

src/AnalyzerTags.h

@@ -46,6 +46,7 @@ namespace AnalyzerTag {
 		Contents, ContentLine, NVT, Zip, Contents_DNS, Contents_NCP,
 		Contents_NetbiosSSN, Contents_Rlogin, Contents_Rsh,
 		Contents_DCE_RPC, Contents_SMB, Contents_RPC, Contents_NFS,
+		FTP_ADAT,
 		// End-marker.
 		LastAnalyzer
 	};

src/FTP.cc

@@ -8,6 +8,8 @@
 #include "FTP.h"
 #include "NVT.h"
 #include "Event.h"
+#include "SSL.h"
+#include "Base64.h"
 
 FTP_Analyzer::FTP_Analyzer(Connection* conn)
 : TCP_ApplicationAnalyzer(AnalyzerTag::FTP, conn)
@@ -44,6 +46,14 @@ void FTP_Analyzer::Done()
 		Weird("partial_ftp_request");
 	}
 
+static uint32 get_reply_code(int len, const char* line)
+	{
+	if ( len >= 3 && isdigit(line[0]) && isdigit(line[1]) && isdigit(line[2]) )
+		return (line[0] - '0') * 100 + (line[1] - '0') * 10 + (line[2] - '0');
+	else
+		return 0;
+	}
+
 void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 	{
 	TCP_ApplicationAnalyzer::DeliverStream(length, data, orig);
@@ -93,16 +103,7 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 		}
 	else
 		{
-		uint32 reply_code;
-		if ( length >= 3 &&
-		     isdigit(line[0]) && isdigit(line[1]) && isdigit(line[2]) )
-			{
-			reply_code = (line[0] - '0') * 100 +
-					(line[1] - '0') * 10 +
-					(line[2] - '0');
-			}
-		else
-			reply_code = 0;
+		uint32 reply_code = get_reply_code(length, line);
 
 		int cont_resp;
 
@@ -143,19 +144,22 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 				else
 					line = end_of_line;
 
-				if ( auth_requested.size() > 0 &&
-				     (reply_code == 234 || reply_code == 335) )
-					// Server accepted AUTH requested,
-					// which means that very likely we
-					// won't be able to parse the rest
-					// of the session, and thus we stop
-					// here.
-					SetSkip(true);
-
 				cont_resp = 0;
 				}
 			}
 
+		if ( reply_code == 334 && auth_requested.size() > 0 &&
+		     auth_requested == "GSSAPI" )
+			{
+			// Server wants to proceed with an ADAT exchange and we
+			// know how to analyze the GSI mechanism, so attach analyzer
+			// to look for that.
+			SSL_Analyzer* ssl = new SSL_Analyzer(Conn());
+			ssl->AddSupportAnalyzer(new FTP_ADAT_Analyzer(Conn(), true));
+			ssl->AddSupportAnalyzer(new FTP_ADAT_Analyzer(Conn(), false));
+			AddChildAnalyzer(ssl);
+			}
+
 		vl->append(new Val(reply_code, TYPE_COUNT));
 		vl->append(new StringVal(end_of_line - line, line));
 		vl->append(new Val(cont_resp, TYPE_BOOL));
@@ -164,5 +168,140 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 		}
 
 	ConnectionEvent(f, vl);
+
+	ForwardStream(length, data, orig);
 	}
 
+void FTP_ADAT_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	// Don't know how to parse anything but the ADAT exchanges of GSI GSSAPI,
+	// which is basically just TLS/SSL.
+	if ( ! Parent()->GetTag() == AnalyzerTag::SSL )
+		{
+		Parent()->Remove();
+		return;
+		}
+
+	bool done = false;
+	const char* line = (const char*) data;
+	const char* end_of_line = line + len;
+
+	BroString* decoded_adat = 0;
+
+	if ( orig )
+		{
+		int cmd_len;
+		const char* cmd;
+		line = skip_whitespace(line, end_of_line);
+		get_word(len, line, cmd_len, cmd);
+
+		if ( strncmp(cmd, "ADAT", cmd_len) == 0 )
+			{
+			line = skip_whitespace(line + cmd_len, end_of_line);
+			StringVal encoded(end_of_line - line, line);
+			decoded_adat = decode_base64(encoded.AsString());
+
+			if ( first_token )
+				{
+				// RFC 2743 section 3.1 specifies a framing format for tokens
+				// that includes an identifier for the mechanism type.  The
+				// framing is supposed to be required for the initial context
+				// token, but GSI doesn't do that and starts right in on a
+				// TLS/SSL handshake, so look for that to identify it.
+				const u_char* msg = decoded_adat->Bytes();
+				int msg_len = decoded_adat->Len();
+
+				// Just check that it looks like a viable TLS/SSL handshake
+				// record from the first byte (content type of 0x16) and
+				// that the fourth and fifth bytes indicating the length of
+				// the record match the length of the decoded data.
+				if ( msg_len < 5 || msg[0] != 0x16 ||
+				     msg_len - 5 != ntohs(*((uint16*)(msg + 3))) )
+					{
+					// Doesn't look like TLS/SSL, so done analyzing.
+					done = true;
+					delete decoded_adat;
+					decoded_adat = 0;
+					}
+				}
+
+			first_token = false;
+			}
+
+		else if ( strncmp(cmd, "AUTH", cmd_len) == 0 )
+			// Security state will be reset by a reissued AUTH.
+			done = true;
+		}
+
+	else
+		{
+		uint32 reply_code = get_reply_code(len, line);
+
+		switch ( reply_code ) {
+		case 232:
+		case 234:
+			// Indicates security data exchange is complete, but nothing
+			// more to decode in replies.
+			done = true;
+			break;
+
+		case 235:
+			// Security data exchange complete, but may have more to decode
+			// in the reply (same format at 334 and 335).
+			done = true;
+
+			// Fall-through.
+
+		case 334:
+		case 335:
+			// Security data exchange still in progress, and there could be data
+			// to decode in the reply.
+			line += 3;
+			if ( len > 3 && line[0] == '-' )
+				line++;
+
+			line = skip_whitespace(line, end_of_line);
+
+			if ( end_of_line - line >= 5 && strncmp(line, "ADAT=", 5) == 0 )
+				{
+				line += 5;
+				StringVal encoded(end_of_line - line, line);
+				decoded_adat = decode_base64(encoded.AsString());
+				}
+
+			break;
+
+		case 421:
+		case 431:
+		case 500:
+		case 501:
+		case 503:
+		case 535:
+			// Server isn't going to accept named security mechanism.
+			// Client has to restart back at the AUTH.
+			done = true;
+			break;
+
+		case 631:
+		case 632:
+		case 633:
+			// If the server is sending protected replies, the security
+			// data exchange must have already succeeded.  It does have
+			// encoded data in the reply, but 632 and 633 are also encrypted.
+			done = true;
+			break;
+
+		default:
+			break;
+		}
+		}
+
+	if ( decoded_adat )
+		{
+		ForwardStream(decoded_adat->Len(), decoded_adat->Bytes(), orig);
+		delete decoded_adat;
+		}
+
+	if ( done )
+		Parent()->Remove();
+	}

src/FTP.h

@@ -30,4 +30,26 @@ protected:
 	string auth_requested;	// AUTH method requested
 };
 
+/**
+ * Analyzes security data of ADAT exchanges over FTP control session (RFC 2228).
+ * Currently only the GSI mechanism of GSSAPI AUTH method is understood.
+ * The ADAT exchange for GSI is base64 encoded TLS/SSL handshake tokens.  This
+ * analyzer just decodes the tokens and passes them on to the parent, which must
+ * be an SSL analyzer instance.
+ */
+class FTP_ADAT_Analyzer : public SupportAnalyzer {
+public:
+	FTP_ADAT_Analyzer(Connection* conn, bool arg_orig)
+	    : SupportAnalyzer(AnalyzerTag::FTP_ADAT, conn, arg_orig),
+	      first_token(true) { }
+
+	void DeliverStream(int len, const u_char* data, bool orig);
+
+protected:
+	// Used by the client-side analyzer to tell if it needs to peek at the
+	// initial context token and do sanity checking (i.e. does it look like
+	// a TLS/SSL handshake token).
+	bool first_token;
+};
+
 #endif

testing/btest/Baseline/core.print-bpf-filters/output

@@ -3,38 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-29
+#open	2012-10-08-16-16-08
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416469.508262	-	ip or not ip	T	T
-#close	2012-07-27-19-14-29
+1349712968.812610	-	ip or not ip	T	T
+#close	2012-10-08-16-16-08
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-29
+#open	2012-10-08-16-16-09
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416469.888870	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
-#close	2012-07-27-19-14-29
+1349712969.042094	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 995)) or (tcp port 22)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (tcp port 614)) or (tcp port 990)) or (port 6667)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+#close	2012-10-08-16-16-09
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-30
+#open	2012-10-08-16-16-09
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416470.252918	-	port 42	T	T
-#close	2012-07-27-19-14-30
+1349712969.270826	-	port 42	T	T
+#close	2012-10-08-16-16-09
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-30
+#open	2012-10-08-16-16-09
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416470.614962	-	port 56730	T	T
-#close	2012-07-27-19-14-30
+1349712969.499878	-	port 56730	T	T
+#close	2012-10-08-16-16-09

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -77,6 +77,7 @@ scripts/base/init-default.bro
     scripts/base/protocols/conn/./main.bro
     scripts/base/protocols/conn/./contents.bro
     scripts/base/protocols/conn/./inactivity.bro
+    scripts/base/protocols/conn/./polling.bro
   scripts/base/protocols/dns/__load__.bro
     scripts/base/protocols/dns/./consts.bro
     scripts/base/protocols/dns/./main.bro
@@ -84,6 +85,11 @@ scripts/base/init-default.bro
     scripts/base/protocols/ftp/./utils-commands.bro
     scripts/base/protocols/ftp/./main.bro
     scripts/base/protocols/ftp/./file-extract.bro
+    scripts/base/protocols/ftp/./gridftp.bro
+      scripts/base/protocols/ssl/__load__.bro
+        scripts/base/protocols/ssl/./consts.bro
+        scripts/base/protocols/ssl/./main.bro
+        scripts/base/protocols/ssl/./mozilla-ca-list.bro
   scripts/base/protocols/http/__load__.bro
     scripts/base/protocols/http/./main.bro
     scripts/base/protocols/http/./utils.bro
@@ -102,10 +108,6 @@ scripts/base/init-default.bro
     scripts/base/protocols/socks/./main.bro
   scripts/base/protocols/ssh/__load__.bro
     scripts/base/protocols/ssh/./main.bro
-  scripts/base/protocols/ssl/__load__.bro
-    scripts/base/protocols/ssl/./consts.bro
-    scripts/base/protocols/ssl/./main.bro
-    scripts/base/protocols/ssl/./mozilla-ca-list.bro
   scripts/base/protocols/syslog/__load__.bro
     scripts/base/protocols/syslog/./consts.bro
     scripts/base/protocols/syslog/./main.bro

testing/btest/Baseline/scripts.base.protocols.conn.polling/out1

@@ -0,0 +1,7 @@
+new_connection, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp]
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 0
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 1
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 2
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 3
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 4
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 5

testing/btest/Baseline/scripts.base.protocols.conn.polling/out2

@@ -0,0 +1,4 @@
+new_connection, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp]
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 0
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 1
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 2

testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/conn.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2012-10-05-21-45-15
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1348168976.274919	UWkUyAuUGXf	192.168.57.103	60108	192.168.57.101	2811	tcp	ssl,ftp,gridftp	0.294743	4491	6659	SF	-	0	ShAdDaFf	22	5643	21	7759	(empty)
+1348168976.546371	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	ssl,gridftp-data	0.011938	2135	3196	S1	-	0	ShADad	8	2559	6	3516	(empty)
+#close	2012-10-05-21-45-15

testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2012-10-05-21-45-15
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
+1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+#close	2012-10-05-21-45-15

testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log

@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2012-10-05-21-45-15
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+1348168976.508038	UWkUyAuUGXf	192.168.57.103	60108	192.168.57.101	2811	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161979.000000	1379697979.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
+1348168976.551422	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	TLSv10	TLS_RSA_WITH_NULL_SHA	-	-	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348168676.000000	1348206441.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
+#close	2012-10-05-21-45-15

testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2012-04-27-14-53-12
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string
-1335538392.319381	UWkUyAuUGXf	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-
-#close	2012-04-27-14-53-16
+#open	2012-10-08-16-18-56
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+1335538392.319381	UWkUyAuUGXf	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-	-	-
+#close	2012-10-08-16-18-56

testing/btest/core/leaks/gridftp.test

@@ -0,0 +1,24 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/globus-url-copy.trace %INPUT
+
+@load base/protocols/ftp/gridftp
+
+module GridFTP;
+
+redef size_threshold = 2;
+
+redef enum Notice::Type += {
+    Data_Channel
+};
+
+event GridFTP::data_channel_detected(c: connection)
+	{
+	local msg = fmt("GridFTP data channel over threshold %d bytes",
+	                size_threshold);
+	NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
+	}

testing/btest/scripts/base/protocols/conn/polling.test

@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT >out1
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT stop_cnt=2 >out2
+# @TEST-EXEC: btest-diff out2
+
+@load base/protocols/conn
+
+const stop_cnt = 10 &redef;
+
+function callback(c: connection, cnt: count): interval
+	{
+	print "callback", c$id, cnt;
+	return cnt >= stop_cnt ? -1 sec : .2 sec;
+	}
+
+event new_connection(c: connection)
+	{
+	print "new_connection", c$id;
+	ConnPolling::watch(c, callback, 0, 0secs);
+	}

testing/btest/scripts/base/protocols/ftp/gridftp.test

@@ -0,0 +1,21 @@
+# @TEST-EXEC: bro -r $TRACES/globus-url-copy.trace %INPUT
+# @TEST-EXEC: btest-diff notice.log
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ssl.log
+
+@load base/protocols/ftp/gridftp
+
+module GridFTP;
+
+redef size_threshold = 2;
+
+redef enum Notice::Type += {
+    Data_Channel
+};
+
+event GridFTP::data_channel_detected(c: connection)
+	{
+	local msg = fmt("GridFTP data channel over threshold %d bytes",
+	                size_threshold);
+	NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
+	}

testing/scripts/diff-remove-x509-names

@@ -3,7 +3,7 @@
 # A diff canonifier that removes all X.509 Distinguished Name subject fields
 # because that output can differ depending on installed OpenSSL version.
 
-BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1 }
+BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; cs_col = -1; ci_col = -1 }
 
 /^#fields/ {
     for ( i = 2; i < NF; ++i )
@@ -12,6 +12,10 @@ BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1 }
             s_col = i-1;
         if ( $i == "issuer_subject" )
             i_col = i-1;
+        if ( $i == "client_subject" )
+            cs_col = i-1;
+        if ( $i == "client_issuer_subject" )
+            ci_col = i-1;
         }
 }
 
@@ -27,6 +31,18 @@ i_col >= 0 {
         $i_col = "+";
 }
 
+cs_col >= 0 {
+    if ( $cs_col != "-" )
+        # Mark that it's set, but ignore content.
+        $cs_col = "+";
+}
+
+ci_col >= 0 {
+    if ( $ci_col != "-" )
+        # Mark that it's set, but ignore content.
+        $ci_col = "+";
+}
+
 {
     print;
 }