Checkpointing the merge. Not done yet.

Merge remote-tracking branch 'origin/topic/tunnels' into topic/robin/tunnels-merge

* origin/topic/tunnels: (41 commits)
  Extend weird names that occur in core packet processing during decapsulation.
  Add Teredo analysis option to reduce false positive decapsulation.
  Just some cleanup/documentation of new tunnel-handling code.
  Memory leak fixes
  Add a config.h definition for IPPROTO_IPV4.
  Add AYIYA tunnel decapsulation unit test.
  Add Teredo-specific events.
  Refactor some of the NetSessions routines that recurse on IP packets.
  Add independent options to toggle the different decapsulation methods
  Add more sanity checks before recursing on encapsulated IP packets.
  Suppress Teredo weirds unless decapsulation was successful once before.
  Tunnel support performance optimization.
  Add Teredo tunnel decapsulation.
  Fix for IP tunnel UID persistence.
  Fix AYIYA analyzer tag.
  Add summary documentation to tunnels/main.bro.
  Make tunnels always identifiable by UID, tunnel.log now gets populated.
  Some improvements to the AYIYA analyzer.
  Remove Tunnel::decapsulate_ip option.
  Remove invalid IP-in-IP encapsulated protocol value.
  ...

scripts/base/frameworks/dpd/dpd.sig

@@ -149,3 +149,46 @@ signature dpd_ssl_client {
   payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
   tcp-state originator
 }
+
+signature dpd_ayiya {
+  ip-proto = udp
+  payload /^..\x11\x29/
+  enable "ayiya"
+}
+
+signature dpd_teredo {
+  ip-proto = udp
+  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
+  enable "teredo"
+}
+
+signature dpd_socks_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state originator
+}
+
+signature dpd_socks_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state responder
+	enable "socks"
+}
+
+signature dpd_socks_reverse_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state responder
+}
+
+signature dpd_socks_reverse_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state originator
+	enable "socks"
+}
+

scripts/base/frameworks/tunnels/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/tunnels/main.bro

@@ -0,0 +1,151 @@
+##! This script handles the tracking/logging of tunnels (e.g. Teredo,
+##! AYIYA, or IP-in-IP such as 6to4 where "IP" is either IPv4 or IPv6).
+##!
+##! For any connection that occurs over a tunnel, information about its
+##! encapsulating tunnels is also found in the *tunnel* field of
+##! :bro:type:`connection`.
+
+module Tunnel;
+
+export {
+	## The tunnel logging stream identifier.
+	redef enum Log::ID += { LOG };
+
+	## Types of interesting activity that can occur with a tunnel.
+	type Action: enum {
+		## A new tunnel (encapsulating "connection") has been seen.
+		DISCOVER,
+		## A tunnel connection has closed.
+		CLOSE,
+		## No new connections over a tunnel happened in the past day.
+                ## TODO-Jon: Where is the "past day" coming from? Should be an
+                ## option.
+		EXPIRE,
+	};
+
+	## The record type which contains column fields of the tunnel log.
+	type Info: record {
+		## Time at which some tunnel activity occurred.
+		ts:          time         &log;
+		## The unique identifier for the tunnel, which may correspond
+		## to a :bro:type:`connection`'s *uid* field for non-IP-in-IP tunnels.
+		uid:         string       &log;
+		## The tunnel "connection" 4-tuple of endpoint addresses/ports.
+		## For an IP tunnel, the ports will be 0.
+		id:          conn_id      &log;
+		## The type of activity that occurred.
+		action:      Action       &log;
+		## The type of tunnel.
+		tunnel_type: Tunnel::Type &log;
+	};
+
+	## Logs all tunnels in an ecapsulation chain with action
+	## :bro:see:`Tunnel::DISCOVER` that aren't already in the
+	## :bro:id:`Tunnel::active` table and adds them if not.
+	global register_all: function(ecv: EncapsulatingConnVector);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::DISCOVER` if it's not already in the
+	## :bro:id:`Tunnel::active` table and adds it if not.
+	global register: function(ec: EncapsulatingConn);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::EXPIRE` and removes it from the
+	## :bro:id:`Tunnel::active` table.
+	##
+	## t: A table of tunnels.
+	##
+	## idx: The index of the tunnel table corresponding to the tunnel to expire.
+	##
+	## Returns: 0secs, which when this function is used as an
+	##          :bro:attr:`&expire_func`, indicates to remove the element at
+	##          *idx* immediately.
+	global expire: function(t: table[conn_id] of Info, idx: conn_id): interval;
+
+	## Removes a single tunnel from the :bro:id:`Tunnel::active` table
+	## and logs the closing/expiration of the tunnel.
+	##
+	## tunnel: The tunnel which has closed or expired.
+	##
+	## action: The specific reason for the tunnel ending.
+	global close: function(tunnel: Info, action: Action);
+
+	## Currently active tunnels.  That is, tunnels for which new, encapsulated
+	## connections have been seen in the last day.
+        ## TODO-Jon: Do we we need the &synchronized here?
+	global active: table[conn_id] of Info = table() &synchronized &read_expire=24hrs &expire_func=expire;
+}
+
+const ayiya_ports = { 5072/udp };
+redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] };
+
+const teredo_ports = { 3544/udp };
+redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] };
+
+redef likely_server_ports += { ayiya_ports, teredo_ports };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Tunnel::LOG, [$columns=Info]);
+	}
+
+function register_all(ecv: EncapsulatingConnVector)
+	{
+	for ( i in ecv )
+		register(ecv[i]);
+	}
+
+function register(ec: EncapsulatingConn)
+	{
+	if ( ec$cid !in active )
+		{
+		local tunnel: Info;
+		tunnel$ts = network_time();
+		tunnel$uid = ec$uid;
+		tunnel$id = ec$cid;
+		tunnel$action = DISCOVER;
+		tunnel$tunnel_type = ec$tunnel_type;
+		active[ec$cid] = tunnel;
+		Log::write(LOG, tunnel);
+		}
+	}
+
+function close(tunnel: Info, action: Action)
+	{
+	tunnel$action = action;
+	tunnel$ts = network_time();
+	Log::write(LOG, tunnel);
+	delete active[tunnel$id];
+	}
+
+function expire(t: table[conn_id] of Info, idx: conn_id): interval
+	{
+	close(t[idx], EXPIRE);
+	return 0secs;
+	}
+
+event new_connection(c: connection) &priority=5
+	{
+	if ( c?$tunnel )
+		register_all(c$tunnel);
+	}
+
+event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
+	{
+	## TODO-Jon: Not sure I understand this. Shouldn't c$tunnel already be
+        ## registered? And what if a layer goes way, does that need to be
+        ## removed here? Or is that done separately?
+        ## 
+        ## Also, conn/main.bro has a tunnel_changed handler at the same
+        ## priority that *sets* c$tunnel.  That's seems undefine behaviour. 
+	if ( c?$tunnel )
+		register_all(c$tunnel);
+
+	register_all(e);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c$id in active )
+		close(active[c$id], CLOSE);
+	}