Add an example of a GridFTP data channel detection script.

It relies on the heuristics of GridFTP data channels commonly default to
SSL mutual authentication with a NULL bulk cipher and that they usually
transfer large datasets (default threshold of script is 1 GB).  The
script also defaults to skip_further_processing() after detection to try
to save cycles analyzing the large, benign connection.

Also added a script in base/protocols/conn/polling that generalizes the
process of polling a connection for interesting features.  The GridFTP
data channel detection script depends on it to monitor bytes
transferred.

scripts/base/protocols/conn/__load__.bro

@@ -1,3 +1,4 @@
 @load ./main
 @load ./contents
 @load ./inactivity
+@load ./polling

scripts/base/protocols/conn/polling.bro

@@ -0,0 +1,51 @@
+##! Implements a generic way to poll connections looking for certain features
+##! (e.g. monitor bytes transferred).  The specific feature of a connection
+##! to look for, the polling interval, and the code to execute if the feature
+##! is found are all controlled by user-defined callback functions.
+
+module ConnPolling;
+
+export {
+	## Starts monitoring a given connection.
+	##
+	## c: The connection to watch.
+	##
+	## callback: A callback function that takes as arguments the monitored
+	##           *connection*, and counter *cnt* that increments each time the
+	##           callback is called.  It returns an interval indicating how long
+	##           in the future to schedule an event which will call the
+	##           callback.  A negative return interval causes polling to stop.
+	##
+	## cnt: The initial value of a counter which gets passed to *callback*.
+	##
+	## i: The initial interval at which to schedule the next callback.
+	##    May be ``0secs`` to poll right away.
+	global watch: function(
+	                c: connection,
+	                callback: function(c: connection, cnt: count): interval,
+	                cnt: count,
+	                i: interval);
+}
+
+event ConnPolling::check(
+  c: connection,
+  callback: function(c: connection, cnt: count): interval,
+  cnt: count)
+	{
+	if ( ! connection_exists(c$id) ) return;
+
+	lookup_connection(c$id); # updates the conn val
+
+	local next_interval = callback(c, cnt);
+	if ( next_interval < 0secs ) return;
+	watch(c, callback, cnt + 1, next_interval);
+	}
+
+function watch(
+  c: connection,
+  callback: function(c: connection, cnt: count): interval,
+  cnt: count,
+  i: interval)
+	{
+	schedule i { ConnPolling::check(c, callback, cnt) };
+	}

scripts/policy/protocols/ftp/gridftp-data-detection.bro

@@ -0,0 +1,83 @@
+##! A detection script for GridFTP data channels.  The heuristic used to
+##! identify a GridFTP data channel relies on the fact that default
+##! setting for GridFTP clients typically mutually authenticate the data
+##! channel with SSL and negotiate a NULL bulk cipher (no encryption).
+##! Connections with those attributes are then polled for two minutes
+##! with decreasing frequency to check if the transfer sizes are large
+##! enough to indicate a GridFTP data channel that would be undesireable
+##! to analyze further (e.g. TCP reassembly no longer occurs).  A side
+##! effect is that true connection sizes are not logged, but at the
+##! benefit of saving CPU cycles that otherwise go to analyzing such
+##! large (and hopefully benign) connections.
+
+module GridFTP;
+
+@load base/protocols/conn
+@load base/protocols/ssl
+@load base/frameworks/notice
+
+export {
+	## Number of bytes transferred before guessing a connection is a 
+	## GridFTP data channel.
+	const size_threshold = 1073741824 &redef;
+
+	## Max number of times to check whether a connection's size exceeds the
+	## :bro:see:`GridFTP::size_threshold`.
+	const max_poll_count = 15 &redef;
+
+	## Whether to skip further processing of the GridFTP data channel once
+	## detected, which may help performance.
+	const skip_data = T &redef;
+
+	## Base amount of time between checking whether a GridFTP connection
+	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
+	const poll_interval = 1sec &redef;
+
+	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
+	## increased by each poll interval.  Can be used to make more frequent
+	## checks at the start of a connection and gradually slow down.
+	const poll_interval_increase = 1sec &redef;
+}
+
+redef enum Notice::Type += {
+	Data_Channel
+};
+
+redef record SSL::Info += {
+	## Indicates a client certificate was sent in the SSL handshake.
+	saw_client_cert: bool &optional;
+};
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( is_orig && c?$ssl )
+		c$ssl$saw_client_cert = T;
+	}
+
+function size_callback(c: connection, cnt: count): interval
+	{
+	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+		{
+		local msg = fmt("GridFTP data channel over threshold %d bytes",
+		                size_threshold);
+		NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
+		if ( skip_data )
+			skip_further_processing(c$id);
+		return -1sec;
+		}
+
+	if ( cnt >= max_poll_count ) return -1sec;
+
+	return poll_interval + poll_interval_increase * cnt;
+	}
+
+event ssl_established(c: connection)
+	{
+	# By default GridFTP data channels do mutual authentication and
+	# negotiate a cipher suite with a NULL bulk cipher.
+	if ( c?$ssl && c$ssl?$saw_client_cert && c$ssl?$subject &&
+	     c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher )
+		{
+		ConnPolling::watch(c, size_callback, 0, 0secs);
+		}
+	}

testing/btest/Baseline/scripts.base.protocols.conn.polling/out1

@@ -0,0 +1,7 @@
+new_connection, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp]
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 0
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 1
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 2
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 3
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 4
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 5

testing/btest/Baseline/scripts.base.protocols.conn.polling/out2

@@ -0,0 +1,4 @@
+new_connection, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp]
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 0
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 1
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 2

testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2012-10-01-17-11-05
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
+1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+#close	2012-10-01-17-11-05

testing/btest/scripts/base/protocols/conn/polling.test

@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT >out1
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT stop_cnt=2 >out2
+# @TEST-EXEC: btest-diff out2
+
+@load base/protocols/conn
+
+const stop_cnt = 10 &redef;
+
+function callback(c: connection, cnt: count): interval
+	{
+	print "callback", c$id, cnt;
+	return cnt >= stop_cnt ? -1 sec : .2 sec;
+	}
+
+event new_connection(c: connection)
+	{
+	print "new_connection", c$id;
+	ConnPolling::watch(c, callback, 0, 0secs);
+	}

testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test

@@ -0,0 +1,6 @@
+# @TEST-EXEC: bro -r $TRACES/globus-url-copy.trace %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+@load protocols/ftp/gridftp-data-detection
+
+redef GridFTP::size_threshold = 2;