Make tunnels always identifiable by UID, tunnel.log now gets populated.

conn.log now sets a field indicating all the parent tunnel UIDs over
which a connection operated and cross reference the UIDs found in
the tunnel.log.

Also some renaming of tunnel related types at the scripting layer.

doc/scripts/DocSourcesList.cmake

@@ -53,6 +53,7 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
@@ -71,6 +72,7 @@ rest_target(${psd} base/protocols/irc/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
+rest_target(${psd} base/protocols/socks/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
 rest_target(${psd} base/protocols/ssl/main.bro)
@@ -97,7 +99,6 @@ rest_target(${psd} policy/frameworks/metrics/http-example.bro)
 rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/frameworks/tunnel.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)

scripts/base/frameworks/tunnels/main.bro

@@ -1,53 +1,127 @@
-module Tunnels;
+module Tunnel;
 
 export {
+	## The tunnel logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## Types of interesting activity that can occur with a tunnel.
 	type Action: enum {
+		## A new tunnel (encapsulating "connection") has been seen.
 		DISCOVER,
+		## A tunnel connection has closed.
 		CLOSE,
+		## No new connections over a tunnel happened in the past day.
+		EXPIRE,
 	};
 
+	## The record type which contains column fields of the tunnel log.
 	type Info: record {
-		ts:          time    &log;
+		## Time at which some tunnel activity occurred.
-		uid:         string  &log &optional;
+		ts:          time         &log;
-		id:          conn_id &log;
+		## The unique identifier for the tunnel, which may correspond
-		action:      Action  &log;
+		## to a :bro:type:`connection`'s *uid* field for non-IP-in-IP tunnels.
-		tunnel_type: string  &log;
+		uid:         string       &log &optional;
-		user:        string  &log &optional;
+		## The tunnel "connection" 4-tuple of endpoint addresses/ports.
+		## For an IP tunnel, the ports will be 0.
+		id:          conn_id      &log;
+		## The type of activity that occurred.
+		action:      Action       &log;
+		## The type of tunnel.
+		tunnel_type: Tunnel::Type &log;
+		user:        string       &log &optional;
 	};
 
-	global register: function(c: connection, tunnel_type: string);
+	## Logs all tunnels in an ecapsulation chain with action
+	## :bro:see:`Tunnel::DISCOVER` that aren't already in the
+	## :bro:id:`Tunnel::active` table and adds them if not.
+	global register_all: function(ecv: EncapsulatingConnVector);
 
-	global active: table[conn_id] of Tunnels::Info = table();
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::DISCOVER` if it's not already in the
+	## :bro:id:`Tunnel::active` table and adds it if not.
+	global register: function(ec: EncapsulatingConn);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::EXPIRE` and removes it from the
+	## :bro:id:`Tunnel::active` table.
+	##
+	## t: A table of tunnels.
+	##
+	## idx: The index of the tunnel table corresponding to the tunnel to expire.
+	##
+	## Returns: 0secs, which when this function is used as an
+	##          :bro:attr:`&expire_func`, indicates to remove the element at
+	##          *idx* immediately.
+	global expire: function(t: table[conn_id] of Info, idx: conn_id): interval;
+
+	## Removes a single tunnel from the :bro:id:`Tunnel::active` table
+	## and logs the closing/expiration of the tunnel.
+	##
+	## tunnel: The tunnel which has closed or expired.
+	##
+	## action: The specific reason for the tunnel ending.
+	global close: function(tunnel: Info, action: Action);
+
+	## Currently active tunnels.  That is, tunnels for which new, encapsulated
+	## connections have been seen in the last day.
+	global active: table[conn_id] of Info = table() &synchronized &read_expire=24hrs &expire_func=expire;
 }
 
 event bro_init() &priority=5
 	{
-	Log::create_stream(Tunnels::LOG, [$columns=Info]);
+	Log::create_stream(Tunnel::LOG, [$columns=Info]);
 	}
 
-function register(c: connection, tunnel_type: string)
+function register_all(ecv: EncapsulatingConnVector)
 	{
-	local tunnel: Info;
+	for ( i in ecv )
-	tunnel$ts = network_time();
+		register(ecv[i]);
-	tunnel$uid = c$uid;
+	}
-	tunnel$id = c$id;
-	tunnel$action = DISCOVER;
-	tunnel$tunnel_type = tunnel_type;
 
-	active[c$id] = tunnel;
+function register(ec: EncapsulatingConn)
+	{
+	if ( ec$cid !in active )
+		{
+		local tunnel: Info;
+		tunnel$ts = network_time();
+		tunnel$uid = ec$uid;
+		tunnel$id = ec$cid;
+		tunnel$action = DISCOVER;
+		tunnel$tunnel_type = ec$tunnel_type;
+		active[ec$cid] = tunnel;
+		Log::write(LOG, tunnel);
+		}
+	}
+
+function close(tunnel: Info, action: Action)
+	{
+	tunnel$action = action;
+	tunnel$ts = network_time();
 	Log::write(LOG, tunnel);
+	delete active[tunnel$id];
+	}
+
+function expire(t: table[conn_id] of Info, idx: conn_id): interval
+	{
+	close(t[idx], EXPIRE);
+	return 0secs;
+	}
+
+event new_connection(c: connection) &priority=5
+	{
+	if ( c?$tunnel )
+		register_all(c$tunnel);
+	}
+
+event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
+	{
+	if ( c?$tunnel )
+		register_all(c$tunnel);
+	register_all(e);
 	}
 	
 event connection_state_remove(c: connection) &priority=-5
 	{
 	if ( c$id in active )
-		{
+		close(active[c$id], CLOSE);
-		local tunnel = active[c$id];
-		tunnel$action=CLOSE;
-		Log::write(LOG, tunnel);
-		
-		delete active[c$id];
-		}
 	}

scripts/base/init-bare.bro

@@ -188,6 +188,9 @@ export {
 		cid: conn_id;
 		## The type of tunnel.
 		tunnel_type: Tunnel::Type;
+		## A globally unique identifier that, for non-IP-in-IP tunnels,
+		## cross-references the *uid* field of :bro:type:`connection`.
+		uid: string &optional;
 	} &log;
 } # end export
 module GLOBAL;
@@ -198,7 +201,7 @@ module GLOBAL;
 ## .. todo:: We need this type definition only for declaring builtin functions
 ##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
 ##    directly and then remove this alias.
-type encapsulating_conns: vector of Tunnel::EncapsulatingConn;
+type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn;
 
 ## Statistics about an endpoint.
 ##
@@ -251,7 +254,7 @@ type connection: record {
 	## at index zero.  It's also always the first such enapsulation seen
 	## for the connection unless the :bro:id:`tunnel_changed` event is handled
 	## and re-assigns this field to the new encapsulation.
-	tunnel: encapsulating_conns &optional;
+	tunnel: EncapsulatingConnVector &optional;
 };
 
 ## Fields of a SYN packet.

scripts/base/protocols/conn/main.bro

@@ -102,8 +102,9 @@ export {
 		## Number IP level bytes the responder sent. See ``orig_pkts``.
 		resp_ip_bytes: count      &log &optional;
 		## If this connection was over a tunnel, indicate the 
-		## `uid` value for the parent connection or connections.
+		## *uid* values for any encapsulating parent connections
-		parents:        vector of string &log &optional;
+		## used over the lifetime of this inner connection.
+		parents:        set[string] &log;
 	};
 
 	## Event that can be handled to access the :bro:type:`Conn::Info`
@@ -193,15 +194,8 @@ function set_conn(c: connection, eoc: bool)
 	c$conn$ts=c$start_time;
 	c$conn$uid=c$uid;
 	c$conn$id=c$id;
-	if ( ! c$conn?$parents && c?$tunnel )
+	if ( c?$tunnel && |c$tunnel| > 0 )
-		{
+		add c$conn$parents[c$tunnel[|c$tunnel|-1]$uid];
-		c$conn$parents = vector();
-		for ( i in c$tunnel )
-			{
-			# TODO: maybe we should be storing uid's in the $tunnel field?
-			#c$conn$parents[|c$conn$parents|] = lookup_connection(c$tunnel[i]$cid)$uid;
-			}
-		}
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
@@ -240,6 +234,14 @@ event content_gap(c: connection, is_orig: bool, seq: count, length: count) &prio
 	c$conn$missed_bytes = c$conn$missed_bytes + length;
 	}
 
+event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
+	{
+	set_conn(c, F);
+	if ( |e| > 0 )
+		add c$conn$parents[e[|e|-1]$uid];
+	c$tunnel = e;
+	}
+	
 event connection_state_remove(c: connection) &priority=5
 	{
 	set_conn(c, T);

scripts/base/protocols/socks/main.bro

@@ -11,7 +11,7 @@ export {
 
 event socks_request(c: connection, request_type: count, dstaddr: addr, dstname: string, p: port, user: string)
 	{
-	Tunnels::register(c, "SOCKS");
+	Tunnel::register([$cid=c$id, $tunnel_type=Tunnel::SOCKS, $uid=c$uid]);
 	}
 
 #

scripts/policy/frameworks/tunnel.bro

@@ -1,83 +0,0 @@
-##! Handle tunneled connections.  
-##! 
-##! Bro can decapsulate IPinIP and IPinUDP tunnels, were "IP" can be either
-##! IPv4 or IPv6. The most common case will be decapsulating Teredo, 6to4,
-##! 6in4, and AYIAY. When this script is loaded, decapsulation will be 
-##! enabled. "tunnel.log" will log the "parent" for each tunneled 
-##! connection. The identity (and existence) of the tunnel connection
-##! is otherwise lost. 
-##!
-##! Currently handles: 
-##!
-##!   * IP6 in IP{4,6}. (IP4 in IP is easy to add, but omitted due to lack
-##!     of test cases.
-##!   * IP{4,6} in UDP. This decapsulates e.g., standard *Teredo* packets
-##!     (without authentication or origin indicator)
-##!   * IP{4,6} in AYIAY
-##!   * Only checks for UDP tunnels on Teredo's and AYIAY's default 
-##!     ports. See :bro:id:`udp_tunnel_ports` and 
-##!     :bro:id:`udp_tunnel_allports`
-##! 
-##! Decapsulation happens early in a packets processing, right after IP
-##! defragmentation but before there is a connection context. The tunnel
-##! headers are stripped from packet and the identity of the parent is 
-##! is stored as the ``tunnel_parent`` member of :bro:type:`connection`, 
-##! which is of type :bro:type:`Tunnel::Parent`. 
-##! 
-##! *Limitation:* decapsulation happens only on the primary path, i.e.
-##! it's not available for the secondary path.
-
-@load base/protocols/conn
-
-module Tunnel; 
-
-redef Tunnel::decapsulate_ip = T;
-redef Tunnel::decapsulate_udp = T;
-redef Tunnel::udp_tunnel_allports = T;
-
-export {
-	## The Tunnel logging stream identifier.
-	redef enum Log::ID += { LOG };
-
-	## This record describing a tunneled connection will be logged.
-	type Info : record {
-		## This is the time of the first record
-		ts:       time            &log;
-		## The uid of the child connection, i.e. the connection in the tunnel
-		uid:      string          &log;
-		## The connection id of the child
-		id:       conn_id         &log;
-		## The child's transport protocol
-		proto:    transport_proto &log;
-		## The parent connection of IP-pair
-		parent:   Parent          &log;
-	};
-
-	## Event that can be handled to access the :bro:type:`Tunnel::Info`
-	## record as it is sent on to the logging framework.
-	global log_tunnel: event(rec: Info);
-
-	redef record Conn::Info += {
-		## If the connection is tunneled, the type of tunnel.
-		tunnel_type: Tunneltype   &log &optional;
-	};
-}
-
-event bro_init()
-	{
-	Log::create_stream(Tunnel::LOG, [$columns=Info, $ev=log_tunnel]);
-	}
-
-event new_connection(c: connection)
-	{
-	if (c?$tunnel_parent)
-		{
-		local info: Info;
-		info$ts = c$start_time;
-		info$uid = c$uid;
-		info$id = c$id;
-		info$proto = get_port_transport_proto(c$id$resp_p);
-		info$parent = c$tunnel_parent;
-		Log::write(Tunnel::LOG, info);
-		}
-	}

src/Conn.cc

@@ -13,7 +13,6 @@
 #include "Timer.h"
 #include "PIA.h"
 #include "binpac.h"
-#include "Tunnels.h"
 
 void ConnectionTimer::Init(Connection* arg_conn, timer_func arg_timer,
 				int arg_do_expire)
@@ -192,15 +191,6 @@ Connection::~Connection()
 		--external_connections;
 	}
 
-void Connection::CheckEncapsulation(const Encapsulation& arg_encap)
-	{
-	if ( encapsulation != arg_encap )
-		{
-		Event(tunnel_changed, 0, arg_encap.GetVectorVal());
-		encapsulation = arg_encap;
-		}
-	}
-
 void Connection::Done()
 	{
 	finished = 1;

src/Conn.h

@@ -23,7 +23,6 @@ class RuleHdrTest;
 class Specific_RE_Matcher;
 class TransportLayerAnalyzer;
 class RuleEndpointState;
-class TunnelParent;
 
 typedef enum {
 	NUL_IN_LINE,
@@ -56,7 +55,14 @@ public:
 	           const Encapsulation& arg_encap);
 	virtual ~Connection();
 
-	void CheckEncapsulation(const Encapsulation& arg_encap);
+	void CheckEncapsulation(const Encapsulation& arg_encap)
+		{
+		if ( encapsulation != arg_encap )
+			{
+			Event(tunnel_changed, 0, arg_encap.GetVectorVal());
+			encapsulation = arg_encap;
+			}
+		}
 
 	// Invoked when connection is about to be removed.  Use Ref(this)
 	// inside Done to keep the connection object around (though it'll
@@ -246,6 +252,8 @@ public:
 
 	void SetUID(uint64 arg_uid)	 { uid = arg_uid; }
 
+	uint64 GetUID() const { return uid; }
+	
 	const Encapsulation& GetEncapsulation() const
 		{ return encapsulation; }
 	

src/Sessions.cc

@@ -543,11 +543,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		fake_hdr.ts = hdr->ts;
 
 		EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr(),
-		  ip_hdr->IP4_Hdr() ?
+		                     BifEnum::Tunnel::IP);
-		    ( proto == IPPROTO_IPV6 ?
-		      BifEnum::Tunnel::IP6_IN_IP4 : BifEnum::Tunnel::IP4_IN_IP4 ) :
-		    ( proto == IPPROTO_IPV6 ?
-		      BifEnum::Tunnel::IP6_IN_IP6 : BifEnum::Tunnel::IP4_IN_IP6 ));
 		encapsulation.Add(ec);
 
 		DoNextPacket(t, &fake_hdr, inner_ip, data, 0, encapsulation);

src/Tunnels.cc

@@ -1,6 +1,20 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "Tunnels.h"
+#include "util.h"
+#include "Conn.h"
+
+EncapsulatingConn::EncapsulatingConn(Connection* c, BifEnum::Tunnel::Type t)
+		: src_addr(c->OrigAddr()), dst_addr(c->RespAddr()),
+		  src_port(c->OrigPort()), dst_port(c->RespPort()),
+		  type(t), uid(c->GetUID())
+		{
+		if ( ! uid )
+			{
+			uid = calculate_unique_id();
+			c->SetUID(uid);
+			}
+		}
 
 RecordVal* EncapsulatingConn::GetRecordVal() const
 	{
@@ -8,15 +22,17 @@ RecordVal* EncapsulatingConn::GetRecordVal() const
 			new RecordVal(BifType::Record::Tunnel::EncapsulatingConn);
 	TransportProto tproto;
 	switch ( type ) {
-	case BifEnum::Tunnel::IP6_IN_IP4:
+	case BifEnum::Tunnel::AYIYA:
-	case BifEnum::Tunnel::IP4_IN_IP4:
+	case BifEnum::Tunnel::TEREDO:
-	case BifEnum::Tunnel::IP6_IN_IP6:
-	case BifEnum::Tunnel::IP4_IN_IP6:
-		tproto = TRANSPORT_UNKNOWN;
-		break;
-	default:
 		tproto = TRANSPORT_UDP;
 		break;
+	case BifEnum::Tunnel::SOCKS:
+		tproto = TRANSPORT_TCP;
+		break;
+	case BifEnum::Tunnel::IP:
+	default:
+		tproto = TRANSPORT_UNKNOWN;
+		break;
 	} // end switch
 
 	RecordVal* id_val = new RecordVal(conn_id);
@@ -26,6 +42,8 @@ RecordVal* EncapsulatingConn::GetRecordVal() const
 	id_val->Assign(3, new PortVal(ntohs(dst_port), tproto));
 	rv->Assign(0, id_val);
 	rv->Assign(1, new EnumVal(type, BifType::Enum::Tunnel::Type));
+	char tmp[20];
+	rv->Assign(2, new StringVal(uitoa_n(uid, tmp, sizeof(tmp), 62)));
 	return rv;
 	}
 

src/Tunnels.h

@@ -9,25 +9,27 @@
 #include "Val.h"
 #include <vector>
 
+class Connection;
+
 class EncapsulatingConn {
 public:
-	EncapsulatingConn()
-		: src_port(0), dst_port(0), type(BifEnum::Tunnel::NONE) {}
-
 	EncapsulatingConn(const IPAddr& s, const IPAddr& d,
 	                  BifEnum::Tunnel::Type t)
-		: src_addr(s), dst_addr(d), src_port(0), dst_port(0), type(t) {}
+		: src_addr(s), dst_addr(d), src_port(0), dst_port(0), type(t)
+		{
+		uid = calculate_unique_id();
+		}
 
-	EncapsulatingConn(const IPAddr& s, const IPAddr& d, uint16 sp, uint16 dp,
+	EncapsulatingConn(Connection* c, BifEnum::Tunnel::Type t);
-	                  BifEnum::Tunnel::Type t)
-		: src_addr(s), dst_addr(d), src_port(sp), dst_port(dp), type(t) {}
 
 	EncapsulatingConn(const EncapsulatingConn& other)
 		: src_addr(other.src_addr), dst_addr(other.dst_addr),
 		  src_port(other.src_port), dst_port(other.dst_port),
-		  type(other.type) {}
+		  type(other.type), uid(other.uid)
+		{}
 
-	~EncapsulatingConn() {}
+	~EncapsulatingConn()
+		{}
 
 	RecordVal* GetRecordVal() const;
 
@@ -35,7 +37,8 @@ public:
 	                       const EncapsulatingConn& ec2)
 		{
 		return ec1.type == ec2.type && ec1.src_addr == ec2.src_addr &&
-		       ec1.src_port == ec2.src_port && ec1.dst_port == ec2.dst_port;
+		       ec1.src_port == ec2.src_port && ec1.dst_port == ec2.dst_port &&
+		       ec1.uid == ec2.uid;
 		}
 
 	friend bool operator!=(const EncapsulatingConn& ec1,
@@ -49,11 +52,13 @@ public:
 	uint16 src_port;
 	uint16 dst_port;
 	BifEnum::Tunnel::Type type;
+	uint64 uid;
 };
 
 class Encapsulation {
 public:
-	Encapsulation() : conns(0) {}
+	Encapsulation() : conns(0)
+		{}
 
 	Encapsulation(const Encapsulation& other)
 		{

src/ayiya-analyzer.pac

@@ -36,18 +36,14 @@ flow AYIYA_Flow
 			return false;
 			}
 		
-		if ( inner_ip != 0 )
+		connection()->bro_analyzer()->ProtocolConfirmation();
-			connection()->bro_analyzer()->ProtocolConfirmation();
 		
 		struct pcap_pkthdr fake_hdr;
 		fake_hdr.caplen = fake_hdr.len = ${pdu.packet}.length();
-		// Not sure what to do with this timestamp.
+		fake_hdr.ts.tv_sec = fake_hdr.ts.tv_usec = 0;
-		//fake_hdr.ts = network_time();
 		
 		Encapsulation encap(c->GetEncapsulation());
-		EncapsulatingConn ec(c->OrigAddr(), c->RespAddr(),
+		EncapsulatingConn ec(c, BifEnum::Tunnel::AYIYA);
-		                     c->OrigPort(), c->RespPort(),
-		                     BifEnum::Tunnel::AYIYA);
 		encap.Add(ec);
 		
 		sessions->DoNextPacket(network_time(), &fake_hdr, inner_ip, ${pdu.packet}.data(), 0, encap);

src/event.bif

@@ -150,7 +150,7 @@ event new_connection%(c: connection%);
 ## c: The connection whose tunnel/encapsulation changed.
 ##
 ## e: The new encapsulation.
-event tunnel_changed%(c: connection, e: encapsulating_conns%);
+event tunnel_changed%(c: connection, e: EncapsulatingConnVector%);
 
 ## Generated when reassembly starts for a TCP connection. The event is raised
 ## at the moment when Bro's TCP analyzer enables stream reassembly for a

src/types.bif

@@ -172,13 +172,10 @@ enum ID %{
 module Tunnel;
 enum Type %{
 	NONE,
-	IP6_IN_IP4,
+	IP,
-	IP4_IN_IP4,
-	IP6_IN_IP6,
-	IP4_IN_IP6,
-	IP6_IN_UDP,
-	IP4_IN_UDP,
 	AYIYA,
+	TEREDO,
+	SOCKS,
 %}
 
 type EncapsulatingConn: record;

testing/btest/Baseline/core.print-bpf-filters/conn.log

@@ -3,6 +3,6 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945
+1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)

testing/btest/Baseline/core.print-bpf-filters/output

@@ -5,7 +5,7 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.330721	-	ip or not ip	T	T
+1335456050.312960	-	ip or not ip	T	T
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
@@ -13,7 +13,7 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.542418	-	((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+1335456050.557822	-	((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
@@ -21,7 +21,7 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.748480	-	port 42	T	T
+1335456050.805695	-	port 42	T	T
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
@@ -29,4 +29,4 @@
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1328294052.952845	-	port 56730	T	T
+1335456051.042953	-	port 56730	T	T

testing/btest/Baseline/core.tunnels.ip-in-ip/output

@@ -1,22 +1,22 @@
 new_connection: tunnel
     conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP6]]
+    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
 new_connection: tunnel
     conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP6], [cid=[orig_h=babe::beef, orig_p=0/unknown, resp_h=dead::babe, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP6]]
+    encap: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf], [cid=[orig_h=babe::beef, orig_p=0/unknown, resp_h=dead::babe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=arKYeMETxOg]]
 new_connection: tunnel
     conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP4]]
+    encap: [[cid=[orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
 new_connection: tunnel
     conn_id: [orig_h=70.55.213.211, orig_p=31337/tcp, resp_h=192.88.99.1, resp_p=80/tcp]
-    encap: [[cid=[orig_h=2002:4637:d5d3::4637:d5d3, orig_p=0/unknown, resp_h=2001:4860:0:2001::68, resp_p=0/unknown], tunnel_type=Tunnel::IP4_IN_IP6]]
+    encap: [[cid=[orig_h=2002:4637:d5d3::4637:d5d3, orig_p=0/unknown, resp_h=2001:4860:0:2001::68, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
 new_connection: tunnel
     conn_id: [orig_h=10.0.0.1, orig_p=30000/udp, resp_h=10.0.0.2, resp_p=13000/udp]
-    encap: [[cid=[orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], tunnel_type=Tunnel::IP4_IN_IP4]]
+    encap: [[cid=[orig_h=1.2.3.4, orig_p=0/unknown, resp_h=5.6.7.8, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
 new_connection: tunnel
     conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP6]]
+    encap: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
 tunnel_changed:
     conn_id: [orig_h=dead::beef, orig_p=30000/udp, resp_h=cafe::babe, resp_p=13000/udp]
-    old: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP6]]
+    old: [[cid=[orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=0/unknown, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=UWkUyAuUGXf]]
-    new: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP6_IN_IP6]]
+    new: [[cid=[orig_h=feed::beef, orig_p=0/unknown, resp_h=feed::cafe, resp_p=0/unknown], tunnel_type=Tunnel::IP, uid=k6kgXLOoSKl]]

testing/btest/Baseline/core.vlan-mpls/conn.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.102560	26	0	SH	-	0	SADF	11	470	0	0
+952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.102560	26	0	SH	-	0	SADF	11	470	0	0	(empty)
-1128727435.450898	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945
+1128727435.450898	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
-1278600802.069419	k6kgXLOoSKl	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801
+1278600802.069419	k6kgXLOoSKl	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -61,6 +61,8 @@ scripts/base/init-default.bro
     scripts/base/frameworks/intel/./main.bro
   scripts/base/frameworks/reporter/__load__.bro
     scripts/base/frameworks/reporter/./main.bro
+  scripts/base/frameworks/tunnels/__load__.bro
+    scripts/base/frameworks/tunnels/./main.bro
   scripts/base/protocols/conn/__load__.bro
     scripts/base/protocols/conn/./main.bro
     scripts/base/protocols/conn/./contents.bro
@@ -85,6 +87,8 @@ scripts/base/init-default.bro
     scripts/base/protocols/smtp/./main.bro
     scripts/base/protocols/smtp/./entities.bro
     scripts/base/protocols/smtp/./entities-excerpt.bro
+  scripts/base/protocols/socks/__load__.bro
+    scripts/base/protocols/socks/./main.bro
   scripts/base/protocols/ssh/__load__.bro
     scripts/base/protocols/ssh/./main.bro
   scripts/base/protocols/ssl/__load__.bro

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log

@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562
+1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562	(empty)
-1329843179.871641	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	tcp	ftp-data	0.111218	0	77	SF	-	0	ShAdfFa	4	216	4	297
+1329843179.871641	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	tcp	ftp-data	0.111218	0	77	SF	-	0	ShAdfFa	4	216	4	297	(empty)
-1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164
+1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164	(empty)
-1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164
+1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164	(empty)
-1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458
+1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458	(empty)

testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log

@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1329327783.316897	arKYeMETxOg	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49186	2001:470:4867:99::21	57086	tcp	ftp-data	0.219721	0	342	SF	-	0	ShAdfFa	5	372	4	642
+1329327783.316897	arKYeMETxOg	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49186	2001:470:4867:99::21	57086	tcp	ftp-data	0.219721	0	342	SF	-	0	ShAdfFa	5	372	4	642	(empty)
-1329327786.524332	k6kgXLOoSKl	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49187	2001:470:4867:99::21	57087	tcp	ftp-data	0.217501	0	43	SF	-	0	ShAdfFa	5	372	4	343
+1329327786.524332	k6kgXLOoSKl	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49187	2001:470:4867:99::21	57087	tcp	ftp-data	0.217501	0	43	SF	-	0	ShAdfFa	5	372	4	343	(empty)
-1329327787.289095	nQcgTWjvg4c	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49188	2001:470:4867:99::21	57088	tcp	ftp-data	0.217941	0	77	SF	-	0	ShAdfFa	5	372	4	377
+1329327787.289095	nQcgTWjvg4c	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49188	2001:470:4867:99::21	57088	tcp	ftp-data	0.217941	0	77	SF	-	0	ShAdfFa	5	372	4	377	(empty)
-1329327795.571921	j4u32Pc5bif	2001:470:4867:99::21	55785	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49189	tcp	ftp-data	0.109813	77	0	SF	-	0	ShADFaf	5	449	4	300
+1329327795.571921	j4u32Pc5bif	2001:470:4867:99::21	55785	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49189	tcp	ftp-data	0.109813	77	0	SF	-	0	ShADFaf	5	449	4	300	(empty)
-1329327777.822004	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	tcp	ftp	26.658219	310	3448	SF	-	0	ShAdDfFa	57	4426	34	5908
+1329327777.822004	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	tcp	ftp	26.658219	310	3448	SF	-	0	ShAdDfFa	57	4426	34	5908	(empty)
-1329327800.017649	TEfuqmmG4bh	2001:470:4867:99::21	55647	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49190	tcp	ftp-data	0.109181	342	0	SF	-	0	ShADFaf	5	714	4	300
+1329327800.017649	TEfuqmmG4bh	2001:470:4867:99::21	55647	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49190	tcp	ftp-data	0.109181	342	0	SF	-	0	ShADFaf	5	714	4	300	(empty)

testing/btest/core/tunnels/ip-in-ip.test

@@ -20,7 +20,7 @@ event new_connection(c: connection)
 		}
 	}
 
-event tunnel_changed(c: connection, e: encapsulating_conns)
+event tunnel_changed(c: connection, e: EncapsulatingConnVector)
 	{
 	print "tunnel_changed:";
 	print fmt("    conn_id: %s", c$id);