Merge remote-tracking branch 'origin/master' into topic/seth/elasticsearch

Conflicts:
	aux/binpac
	aux/bro-aux
	aux/broccoli
	aux/broctl
	scripts/base/frameworks/logging/__load__.bro
	src/logging.bif

scripts/base/frameworks/communication/main.bro

@@ -11,7 +11,8 @@ export {
 	## The communication logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	## Which interface to listen on (``0.0.0.0`` or ``[::]`` are wildcards).
+	## Which interface to listen on. The addresses ``0.0.0.0`` and ``[::]``
+	## are wildcards.
 	const listen_interface = 0.0.0.0 &redef;
 	
 	## Which port to listen on.

scripts/base/frameworks/dpd/dpd.sig

@@ -149,3 +149,64 @@ signature dpd_ssl_client {
   payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
   tcp-state originator
 }
+
+signature dpd_ayiya {
+  ip-proto = udp
+  payload /^..\x11\x29/
+  enable "ayiya"
+}
+
+signature dpd_teredo {
+  ip-proto = udp
+  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
+  enable "teredo"
+}
+
+signature dpd_socks4_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state originator
+}
+
+signature dpd_socks4_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state responder
+	enable "socks"
+}
+
+signature dpd_socks4_reverse_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state responder
+}
+
+signature dpd_socks4_reverse_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_reverse_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state originator
+	enable "socks"
+}
+
+signature dpd_socks5_client {
+	ip-proto == tcp
+	# Watch for a few authentication methods to reduce false positives.
+	payload /^\x05.[\x00\x01\x02]/
+	tcp-state originator
+}
+
+signature dpd_socks5_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks5_client
+	# Watch for a single authentication method to be chosen by the server or
+	# the server to indicate the no authentication is required.
+	payload /^\x05(\x00|\x01[\x00\x01\x02])/
+	tcp-state responder
+	enable "socks"
+}
+
+

scripts/base/frameworks/input/main.bro

@@ -53,6 +53,11 @@ export {
 		## really be executed. Parameters are the same as for the event. If true is
 		## returned, the update is performed. If false is returned, it is skipped.
 		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+                config: table[string] of string &default=table();
 	};
 
 	## EventFilter description type used for the `event` method.
@@ -85,6 +90,10 @@ export {
 		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
 		ev: any;
 
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
 	};
 
 	## Create a new table input from a given source. Returns true on success.

scripts/base/frameworks/logging/__load__.bro

@@ -2,4 +2,4 @@
 @load ./postprocessors
 @load ./writers/ascii
 @load ./writers/dataseries
-@load ./writers/elasticsearch
+@load ./writers/elasticsearch@load ./writers/none

scripts/base/frameworks/logging/main.bro

@@ -138,6 +138,11 @@ export {
 		## Callback function to trigger for rotated files. If not set, the
 		## default comes out of :bro:id:`Log::default_rotation_postprocessors`.
 		postprocessor: function(info: RotationInfo) : bool &optional;
+
+		## A key/value table that will be passed on to the writer.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
 	};
 
 	## Sentinel value for indicating that a filter was not found when looked up.
@@ -327,6 +332,8 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
 	{
 	if ( info$writer in default_rotation_postprocessors )
 		return default_rotation_postprocessors[info$writer](info);
+
+	return F;
 	}
 
 function default_path_func(id: ID, path: string, rec: any) : string

scripts/base/frameworks/logging/writers/none.bro

@@ -0,0 +1,17 @@
+##! Interface for the None log writer. Thiis writer is mainly for debugging.
+
+module LogNone;
+
+export {
+	## If true, output debugging output that can be useful for unit
+        ## testing the logging framework.
+	const debug = F &redef;
+}
+
+function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
+	{
+	return T;
+	}
+
+redef Log::default_rotation_postprocessors += { [Log::WRITER_NONE] = default_rotation_postprocessor_func };
+

scripts/base/frameworks/tunnels/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/tunnels/main.bro

@@ -0,0 +1,149 @@
+##! This script handles the tracking/logging of tunnels (e.g. Teredo,
+##! AYIYA, or IP-in-IP such as 6to4 where "IP" is either IPv4 or IPv6).
+##!
+##! For any connection that occurs over a tunnel, information about its
+##! encapsulating tunnels is also found in the *tunnel* field of
+##! :bro:type:`connection`.
+
+module Tunnel;
+
+export {
+	## The tunnel logging stream identifier.
+	redef enum Log::ID += { LOG };
+
+	## Types of interesting activity that can occur with a tunnel.
+	type Action: enum {
+		## A new tunnel (encapsulating "connection") has been seen.
+		DISCOVER,
+		## A tunnel connection has closed.
+		CLOSE,
+		## No new connections over a tunnel happened in the amount of
+		## time indicated by :bro:see:`Tunnel::expiration_interval`.
+		EXPIRE,
+	};
+
+	## The record type which contains column fields of the tunnel log.
+	type Info: record {
+		## Time at which some tunnel activity occurred.
+		ts:          time         &log;
+		## The unique identifier for the tunnel, which may correspond
+		## to a :bro:type:`connection`'s *uid* field for non-IP-in-IP tunnels.
+		## This is optional because there could be numerous connections
+		## for payload proxies like SOCKS but we should treat it as a single
+		## tunnel.
+		uid:         string       &log &optional;
+		## The tunnel "connection" 4-tuple of endpoint addresses/ports.
+		## For an IP tunnel, the ports will be 0.
+		id:          conn_id      &log;
+		## The type of tunnel.
+		tunnel_type: Tunnel::Type &log;
+		## The type of activity that occurred.
+		action:      Action       &log;
+	};
+
+	## Logs all tunnels in an encapsulation chain with action
+	## :bro:see:`Tunnel::DISCOVER` that aren't already in the
+	## :bro:id:`Tunnel::active` table and adds them if not.
+	global register_all: function(ecv: EncapsulatingConnVector);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::DISCOVER` if it's not already in the
+	## :bro:id:`Tunnel::active` table and adds it if not.
+	global register: function(ec: EncapsulatingConn);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::EXPIRE` and removes it from the
+	## :bro:id:`Tunnel::active` table.
+	##
+	## t: A table of tunnels.
+	##
+	## idx: The index of the tunnel table corresponding to the tunnel to expire.
+	##
+	## Returns: 0secs, which when this function is used as an
+	##          :bro:attr:`&expire_func`, indicates to remove the element at
+	##          *idx* immediately.
+	global expire: function(t: table[conn_id] of Info, idx: conn_id): interval;
+
+	## Removes a single tunnel from the :bro:id:`Tunnel::active` table
+	## and logs the closing/expiration of the tunnel.
+	##
+	## tunnel: The tunnel which has closed or expired.
+	##
+	## action: The specific reason for the tunnel ending.
+	global close: function(tunnel: Info, action: Action);
+
+	## The amount of time a tunnel is not used in establishment of new
+	## connections before it is considered inactive/expired.
+	const expiration_interval = 1hrs &redef;
+
+	## Currently active tunnels.  That is, tunnels for which new, encapsulated
+	## connections have been seen in the interval indicated by
+	## :bro:see:`Tunnel::expiration_interval`.
+	global active: table[conn_id] of Info = table() &read_expire=expiration_interval &expire_func=expire;
+}
+
+const ayiya_ports = { 5072/udp };
+redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] };
+
+const teredo_ports = { 3544/udp };
+redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] };
+
+redef likely_server_ports += { ayiya_ports, teredo_ports };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Tunnel::LOG, [$columns=Info]);
+	}
+
+function register_all(ecv: EncapsulatingConnVector)
+	{
+	for ( i in ecv )
+		register(ecv[i]);
+	}
+
+function register(ec: EncapsulatingConn)
+	{
+	if ( ec$cid !in active )
+		{
+		local tunnel: Info;
+		tunnel$ts = network_time();
+		if ( ec?$uid )
+			tunnel$uid = ec$uid;
+		tunnel$id = ec$cid;
+		tunnel$action = DISCOVER;
+		tunnel$tunnel_type = ec$tunnel_type;
+		active[ec$cid] = tunnel;
+		Log::write(LOG, tunnel);
+		}
+	}
+
+function close(tunnel: Info, action: Action)
+	{
+	tunnel$action = action;
+	tunnel$ts = network_time();
+	Log::write(LOG, tunnel);
+	delete active[tunnel$id];
+	}
+
+function expire(t: table[conn_id] of Info, idx: conn_id): interval
+	{
+	close(t[idx], EXPIRE);
+	return 0secs;
+	}
+
+event new_connection(c: connection) &priority=5
+	{
+	if ( c?$tunnel )
+		register_all(c$tunnel);
+	}
+
+event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
+	{
+	register_all(e);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c$id in active )
+		close(active[c$id], CLOSE);
+	}