Merge remote-tracking branch 'origin/master' into topic/robin/log-threads

Conflicts:
	aux/broccoli
	src/ConnCompressor.cc

aux/broccoli

@@ -1 +1 @@
-Subproject commit 930e7c78221929849086a578308e2fdc99ac3fb8
+Subproject commit 3b63c3f1e7d915b1bda16862bfa4a8593ffc38f6

scripts/base/init-bare.bro

@@ -2159,26 +2159,6 @@ const forward_remote_state_changes = F &redef;
 ## Place-holder constant indicating "no peer".
 const PEER_ID_NONE = 0;
 
-## Deprecated.
-## 
-## .. todo:: The connection compressor is scheduled to be removed from Bro.
-const use_connection_compressor = F &redef;
-
-## Deprecated.
-## 
-## .. todo:: The connection compressor is scheduled to be removed from Bro.
-const cc_handle_resets = F &redef;
-
-## Deprecated.
-## 
-## .. todo:: The connection compressor is scheduled to be removed from Bro.
-const cc_handle_only_syns = T &redef;
-
-## Deprecated.
-## 
-## .. todo:: The connection compressor is scheduled to be removed from Bro.
-const cc_instantiate_on_data = F &redef;
-
 # Signature payload pattern types.
 # todo::use enum to help autodoc
 # todo::Still used?

src/CMakeLists.txt

@@ -290,7 +290,6 @@ set(bro_SRCS
     ChunkedIO.cc
     CompHash.cc
     Conn.cc
-    ConnCompressor.cc
     ConnSizeAnalyzer.cc
     ContentLine.cc
     DCE_RPC.cc

src/Conn.h

@@ -239,30 +239,6 @@ public:
 	// Sets the transport protocol in use.
 	void SetTransport(TransportProto arg_proto)	{ proto = arg_proto; }
 
-	// If the connection compressor is activated, we need a special memory
-	// layout for connections. (See ConnCompressor.h)
-	void* operator new(size_t size)
-		{
-		if ( ! use_connection_compressor )
-			return ::operator new(size);
-
-		void* c = ::operator new(size + 4);
-
-		// We have to turn off the is_pending bit.  By setting the
-		// first four bytes to zero, we'll achieve this.
-		*((uint32*) c) = 0;
-
-		return ((char *) c) + 4;
-		}
-
-	void operator delete(void* ptr)
-		{
-		if ( ! use_connection_compressor )
-			::operator delete(ptr);
-		else
-			::operator delete(((char*) ptr) - 4);
-		}
-
 	void SetUID(uint64 arg_uid)	 { uid = arg_uid; }
 
 protected:

src/ConnCompressor.h

@@ -1,240 +0,0 @@
-// The ConnCompressor keeps track of the first packet seen for a conn_id using
-// only a minimal amount of memory. This helps us to avoid instantiating
-// full Connection objects for never-established sessions.
-//
-// TCP only.
-
-#ifndef CONNCOMPRESSOR_H
-#define CONNCOMPRESSOR_H
-
-#include "Conn.h"
-#include "Dict.h"
-#include "NetVar.h"
-#include "TCP.h"
-
-class ConnCompressor {
-public:
-	ConnCompressor();
-	~ConnCompressor();
-
-	// Handle next packet.  Returns 0 if packet in handled internally.
-	// Takes ownership of key.
-	Connection* NextPacket(double t, HashKey* k, const IP_Hdr* ip_hdr,
-			const struct pcap_pkthdr* hdr, const u_char* const pkt);
-
-	// Look up a connection.  Returns non-nil for connections for
-	// which a Connection object has already been instantiated.
-	Connection* Lookup(HashKey* k)
-		{
-		ConnData* c = conns.Lookup(k);
-		return c && IsConnPtr(c) ? MakeConnPtr(c) : 0;
-		}
-
-	// Inserts connection into compressor.  If another entry with this key
-	// already exists, it's replaced.  If that was a full connection, it is
-	// also returned.
-	Connection* Insert(Connection* c);
-
-	// Remove all state belonging to the given connection.  Returns
-	// true if the connection was found in the compressor's table,
-	// false if not.
-	bool Remove(HashKey* k);
-
-	// Flush state.
-	void Drain();
-
-	struct Sizes {
-		// Current number of already fully instantiated connections.
-		unsigned int connections;
-
-		// Total number of fully instantiated connections.
-		unsigned int connections_total;
-
-		// Current number of seen but non-yet instantiated connections.
-		unsigned int pending_valid;
-
-		// Total number of seen but non-yet instantiated connections.
-		unsigned int pending_total;
-
-		// Total number of all entries in pending list (some a which
-		// may already been invalid, but not yet removed from memory).
-		unsigned int pending_in_mem;
-
-		// Total number of hash table entires
-		// (should equal connections + pending_valid)
-		unsigned int hash_table_size;
-
-		// Total memory usage;
-		unsigned int memory;
-	};
-
-	const Sizes& Size()
-		{ sizes.hash_table_size = conns.Length(); return sizes; }
-
-	unsigned int MemoryAllocation() const	{ return sizes.memory; }
-
-	// As long as we have only seen packets from one side, we just
-	// store a PendingConn.
-	struct PendingConn {
-		// True if the block is indeed a PendingConn (see below).
-		unsigned int is_pending:1;
-
-		// Whether roles in key are flipped.
-		unsigned int ip1_is_src:1;
-
-		unsigned int invalid:1;	// deleted
-		int window_scale:4;
-		unsigned int SYN:1;
-		unsigned int FIN:1;
-		unsigned int RST:1;
-		unsigned int ACK:1;
-
-		double time;
-		struct Key {
-			uint32 ip1[4];
-			uint32 ip2[4];
-			uint16 port1;
-			uint16 port2;
-		} key;
-		uint32 seq;
-		uint32 ack;
-		hash_t hash;
-		uint16 window;
-		uint64 uid;
-
-		// The following are set if use_conn_size_analyzer is T.
-		uint16 num_pkts;
-		uint16 num_bytes_ip;
-	};
-
-private:
-	// Helpers to extract addrs/ports from PendingConn.
-
-	const uint32* SrcAddr(const PendingConn* c)
-		{ return c->ip1_is_src ? c->key.ip1 : c->key.ip2; }
-	const uint32* DstAddr(const PendingConn* c)
-		{ return c->ip1_is_src ? c->key.ip2 : c->key.ip1; }
-
-	uint16 SrcPort(const PendingConn* c)
-		{ return c->ip1_is_src ? c->key.port1 : c->key.port2; }
-	uint16 DstPort(const PendingConn* c)
-		{ return c->ip1_is_src ? c->key.port2 : c->key.port1; }
-
-
-	// Called for the first packet in a connection.
-	Connection* FirstFromOrig(double t, HashKey* key,
-					const IP_Hdr* ip, const tcphdr* tp);
-
-	// Called for more packets from the orginator w/o seeing a response.
-	Connection* NextFromOrig(PendingConn* pending, double t, HashKey* key,
-					const IP_Hdr* ip, const tcphdr* tp);
-
-	// Called for the first response packet. Instantiates a Connection.
-	Connection* Response(PendingConn* pending, double t, HashKey* key,
-					const IP_Hdr* ip, const tcphdr* tp);
-
-	// Instantiates a full TCP connection (invalidates pending connection).
-	Connection* Instantiate(HashKey* key, PendingConn* pending);
-
-	// Same but based on packet.
-	Connection* Instantiate(double t, HashKey* key, const IP_Hdr* ip);
-
-	// Fills the attributes of a PendingConn based on the given arguments.
-	void PktHdrToPendingConn(double time, const HashKey* key,
-		const IP_Hdr* ip, const struct tcphdr* tp, PendingConn* c);
-
-	// Fakes a TCP packet based on the available information.
-	const IP_Hdr* PendingConnToPacket(const PendingConn* c);
-
-	// Construct a TCP-flags byte.
-	uint8 MakeFlags(const PendingConn* c) const;
-
-	// Allocate room for a new (Ext)PendingConn.
-	PendingConn* MakeNewState(double t);
-
-	// Expire PendingConns.
-	void DoExpire(double t);
-
-	// Remove all state belonging to the given connection.
-	void Invalidate(HashKey* k);
-
-	// Sends the given connection_* event.  If orig_state is
-	// TCP_ENDPOINT__INACTIVE, tries to guess a better one based
-	// on pending.  If arg in non-nil, it will be used as the
-	// *first* argument of the event call (this is for conn_weird()).
-	void Event(const PendingConn* pending, double t,
-			const EventHandlerPtr& event, int orig_state,
-			int orig_size, int resp_state, Val* arg = 0);
-
-	void Weird(const PendingConn* pending, double t, const char* msg)
-		{
-		// This will actually go through the Reporter; Event() takes
-		// care of that.
-		Event(pending, t, conn_weird, TCP_ENDPOINT_INACTIVE, 0,
-		      TCP_ENDPOINT_INACTIVE, new StringVal(msg));
-		}
-
-	static const int BLOCK_SIZE = 16 * 1024;
-
-	// The memory managment for PendConns.
-	struct Block {
-		double time;
-		Block* prev;
-		Block* next;
-		int bytes_used;
-		unsigned char data[BLOCK_SIZE];
-	};
-
-	// In the connection hash table, we store pointers to both PendingConns
-	// and Connections. Thus, we need a way to differentiate between
-	// these two types. To avoid an additional indirection, we use a little
-	// hack: a pointer retrieved from the table is interpreted as a
-	// PendingConn first. However, if is_pending is false, it's in fact a
-	// Connection which starts at offset 4. The methods below help to
-	// implement this scheme transparently. An "operator new" in
-	// Connection takes care of building Connection's accordingly.
-	typedef PendingConn ConnData;
-	declare(PDict, ConnData);
-	typedef PDict(ConnData) ConnMap;
-	ConnMap conns;
-
-	static ConnData* MakeMapPtr(PendingConn* c)
-		{ assert(c->is_pending); return c; }
-
-	static ConnData* MakeMapPtr(Connection* c)
-		{
-		ConnData* p = (ConnData*) (((char*) c) - 4);
-		assert(!p->is_pending);
-		return p;
-		}
-
-	static PendingConn* MakePendingConnPtr(ConnData* c)
-		{ assert(c->is_pending); return c; }
-
-	static Connection* MakeConnPtr(ConnData* c)
-		{
-		assert(!c->is_pending);
-		return (Connection*) (((char*) c) + 4);
-		}
-
-	static bool IsConnPtr(ConnData* c)
-		{ return ! c->is_pending; }
-
-	// New blocks are inserted at the end.
-	Block* first_block;
-	Block* last_block;
-
-	// If we have already expired some entries in a block,
-	// this points to the first non-expired.
-	unsigned char* first_non_expired;
-
-	// Last "connection" that we have build.
-	RecordVal* conn_val;
-
-	// Statistics.
-	Sizes sizes;
-	};
-
-extern ConnCompressor* conn_compressor;
-
-#endif

src/Frag.cc

@@ -125,7 +125,7 @@ void FragReassembler::AddFragment(double t, const IP_Hdr* ip, const u_char* pkt,
 
 void FragReassembler::Overlap(const u_char* b1, const u_char* b2, int n)
 	{
-	IP_Hdr proto_h((const struct ip*) proto_hdr);
+	IP_Hdr proto_h(proto_hdr, false);
 
 	if ( memcmp((const void*) b1, (const void*) b2, n) )
 		s->Weird("fragment_inconsistency", &proto_h);
@@ -157,7 +157,7 @@ void FragReassembler::BlockInserted(DataBlock* /* start_block */)
 			// can happen for benign reasons when we're
 			// intermingling parts of two fragmented packets.
 
-			IP_Hdr proto_h((const struct ip*) proto_hdr);
+			IP_Hdr proto_h(proto_hdr, false);
 			s->Weird("fragment_size_inconsistency", &proto_h);
 
 			// We decide to analyze the contiguous portion now.
@@ -171,7 +171,7 @@ void FragReassembler::BlockInserted(DataBlock* /* start_block */)
 
 	else if ( last_block->upper > frag_size )
 		{
-		IP_Hdr proto_h((const struct ip*) proto_hdr);
+		IP_Hdr proto_h(proto_hdr, false);
 		s->Weird("fragment_size_inconsistency", &proto_h);
 		frag_size = last_block->upper;
 		}
@@ -214,7 +214,7 @@ void FragReassembler::BlockInserted(DataBlock* /* start_block */)
 		}
 
 	delete reassembled_pkt;
-	reassembled_pkt = new IP_Hdr(reassem4);
+	reassembled_pkt = new IP_Hdr(reassem4, true);
 
 	DeleteTimer();
 	}

src/IP.h

@@ -9,23 +9,13 @@
 
 class IP_Hdr {
 public:
-	IP_Hdr(struct ip* arg_ip4)
+	IP_Hdr(const struct ip* arg_ip4, bool arg_del)
-		: ip4(arg_ip4), ip6(0), del(1)
+		: ip4(arg_ip4), ip6(0), del(arg_del)
 		{
 		}
 
-	IP_Hdr(const struct ip* arg_ip4)
+	IP_Hdr(const struct ip6_hdr* arg_ip6, bool arg_del)
-		: ip4(arg_ip4), ip6(0), del(0)
+		: ip4(0), ip6(arg_ip6), del(arg_del)
-		{
-		}
-
-	IP_Hdr(struct ip6_hdr* arg_ip6)
-		: ip4(0), ip6(arg_ip6), del(1)
-		{
-		}
-
-	IP_Hdr(const struct ip6_hdr* arg_ip6)
-		: ip4(0), ip6(arg_ip6), del(0)
 		{
 		}
 
@@ -90,7 +80,7 @@ public:
 private:
 	const struct ip* ip4;
 	const struct ip6_hdr* ip6;
-	int del;
+	bool del;
 };
 
 #endif

src/NetVar.cc

@@ -213,11 +213,6 @@ int sig_max_group_size;
 
 int enable_syslog;
 
-int use_connection_compressor;
-int cc_handle_resets;
-int cc_handle_only_syns;
-int cc_instantiate_on_data;
-
 TableType* irc_join_list;
 RecordType* irc_join_info;
 TableVal* irc_servers;
@@ -525,12 +520,6 @@ void init_net_var()
 
 	gap_report_freq = opt_internal_double("gap_report_freq");
 
-	use_connection_compressor =
-		opt_internal_int("use_connection_compressor");
-	cc_handle_resets = opt_internal_int("cc_handle_resets");
-	cc_handle_only_syns = opt_internal_int("cc_handle_only_syns");
-	cc_instantiate_on_data = opt_internal_int("cc_instantiate_on_data");
-
 	irc_join_info = internal_type("irc_join_info")->AsRecordType();
 	irc_join_list = internal_type("irc_join_list")->AsTableType();
 	irc_servers = internal_val("irc_servers")->AsTableVal();

src/NetVar.h

@@ -216,11 +216,6 @@ extern int sig_max_group_size;
 
 extern int enable_syslog;
 
-extern int use_connection_compressor;
-extern int cc_handle_resets;
-extern int cc_handle_only_syns;
-extern int cc_instantiate_on_data;
-
 extern TableType* irc_join_list;
 extern RecordType* irc_join_info;
 extern TableVal* irc_servers;

src/PIA.cc

@@ -196,7 +196,7 @@ void PIA_TCP::FirstPacket(bool is_orig, const IP_Hdr* ip)
 			ip4->ip_p = IPPROTO_TCP;
 
 			// Cast to const so that it doesn't delete it.
-			ip4_hdr = new IP_Hdr((const struct ip*) ip4);
+			ip4_hdr = new IP_Hdr(ip4, false);
 			}
 
 		if ( is_orig )

src/PacketSort.cc

@@ -27,9 +27,9 @@ PacketSortElement::PacketSortElement(PktSrc* arg_src,
 		{
 		const struct ip* ip = (const struct ip*) (pkt + hdr_size);
 		if ( ip->ip_v == 4 )
-			ip_hdr = new IP_Hdr(ip);
+			ip_hdr = new IP_Hdr(ip, false);
 		else
-			ip_hdr = new IP_Hdr((const struct ip6_hdr*) ip);
+			ip_hdr = new IP_Hdr((const struct ip6_hdr*) ip, false);
 
 		if ( ip_hdr->NextProto() == IPPROTO_TCP &&
 		      // Note: can't sort fragmented packets

src/Serializer.h

@@ -415,7 +415,7 @@ public:
 		}
 
 	const IP_Hdr IP() const
-		{ return IP_Hdr((struct ip *) (pkt + hdr_size)); }
+		{ return IP_Hdr((struct ip *) (pkt + hdr_size), true); }
 
 	void Describe(ODesc* d) const;
 

src/Sessions.cc

@@ -27,7 +27,6 @@
 #include "InterConn.h"
 #include "Discard.h"
 #include "RuleMatcher.h"
-#include "ConnCompressor.h"
 #include "DPM.h"
 
 #include "PacketSort.h"
@@ -275,13 +274,13 @@ void NetSessions::NextPacket(double t, const struct pcap_pkthdr* hdr,
 		const struct ip* ip = (const struct ip*) (pkt + hdr_size);
 		if ( ip->ip_v == 4 )
 			{
-			IP_Hdr ip_hdr(ip);
+			IP_Hdr ip_hdr(ip, false);
 			DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size);
 			}
 
 		else if ( ip->ip_v == 6 )
 			{
-			IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size));
+			IP_Hdr ip_hdr((const struct ip6_hdr*) (pkt + hdr_size), false);
 			DoNextPacket(t, hdr, &ip_hdr, pkt, hdr_size);
 			}
 
@@ -510,7 +509,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	id.src_addr = ip_hdr->SrcAddr();
 	id.dst_addr = ip_hdr->DstAddr();
 	Dictionary* d = 0;
-	bool pass_to_conn_compressor = false;
 
 	switch ( proto ) {
 	case IPPROTO_TCP:
@@ -520,7 +518,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		id.dst_port = tp->th_dport;
 		id.is_one_way = 0;
 		d = &tcp_conns;
-		pass_to_conn_compressor = ip4 && use_connection_compressor;
 		break;
 		}
 
@@ -563,10 +560,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 	// FIXME: The following is getting pretty complex. Need to split up
 	// into separate functions.
-	if ( pass_to_conn_compressor )
-		conn = conn_compressor->NextPacket(t, h, ip_hdr, hdr, pkt);
-	else
-		{
 	conn = (Connection*) d->Lookup(h);
 	if ( ! conn )
 		{
@@ -600,7 +593,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 	if ( ! conn )
 		delete h;
-		}
 
 	if ( ! conn )
 		return;
@@ -838,16 +830,7 @@ Connection* NetSessions::FindConnection(Val* v)
 	Dictionary* d;
 
 	if ( orig_portv->IsTCP() )
-		{
-		if ( use_connection_compressor )
-			{
-			Connection* conn = conn_compressor->Lookup(h);
-			delete h;
-			return conn;
-			}
-		else
 		d = &tcp_conns;
-		}
 	else if ( orig_portv->IsUDP() )
 		d = &udp_conns;
 	else if ( orig_portv->IsICMP() )
@@ -900,17 +883,7 @@ void NetSessions::Remove(Connection* c)
 
 		switch ( c->ConnTransport() ) {
 		case TRANSPORT_TCP:
-			if ( use_connection_compressor &&
+			if ( ! tcp_conns.RemoveEntry(k) )
-			     conn_compressor->Remove(k) )
-				// Note, if the Remove() returned false
-				// then the compressor doesn't know about
-				// this connection, which *should* mean that
-				// we never gave it the connection in the
-				// first place, and thus we should check
-				// the regular TCP table instead.
-				;
-
-			else if ( ! tcp_conns.RemoveEntry(k) )
 				reporter->InternalError("connection missing");
 			break;
 
@@ -957,13 +930,8 @@ void NetSessions::Insert(Connection* c)
 	// reference the old key for already existing connections.
 
 	case TRANSPORT_TCP:
-		if ( use_connection_compressor )
-			old = conn_compressor->Insert(c);
-		else
-			{
 		old = (Connection*) tcp_conns.Remove(c->Key());
 		tcp_conns.Insert(c->Key(), c);
-			}
 		break;
 
 	case TRANSPORT_UDP:
@@ -995,9 +963,6 @@ void NetSessions::Insert(Connection* c)
 
 void NetSessions::Drain()
 	{
-	if ( use_connection_compressor )
-		conn_compressor->Drain();
-
 	IterCookie* cookie = tcp_conns.InitForIteration();
 	Connection* tc;
 
@@ -1110,10 +1075,7 @@ Connection* NetSessions::NewConn(HashKey* k, double t, const ConnID* id,
 		conn->AppendAddl(fmt("tag=%s",
 					conn->GetTimerMgr()->GetTag().c_str()));
 
-	// If the connection compressor is active, it takes care of the
+	if ( new_connection )
-	// new_connection/connection_external events for TCP connections.
-	if ( new_connection &&
-	     (tproto != TRANSPORT_TCP || ! use_connection_compressor) )
 		{
 		conn->Event(new_connection, 0);
 

src/Stats.cc

@@ -6,7 +6,6 @@
 #include "Stats.h"
 #include "Scope.h"
 #include "cq.h"
-#include "ConnCompressor.h"
 #include "DNS_Mgr.h"
 #include "Trigger.h"
 #include "threading/Manager.h"
@@ -129,19 +128,6 @@ void ProfileLogger::Log()
 		expensive ? sessions->ConnectionMemoryUsageConnVals() / 1024 : 0
 		));
 
-	const ConnCompressor::Sizes& cs = conn_compressor->Size();
-
-	file->Write(fmt("%.6f ConnCompressor: pending=%d pending_in_mem=%d full_conns=%d pending+real=%d mem=%dK avg=%.1f/%.1f\n",
-		network_time,
-		cs.pending_valid,
-		cs.pending_in_mem,
-		cs.connections,
-		cs.hash_table_size,
-		cs.memory / 1024,
-		cs.memory / double(cs.pending_valid),
-		cs.memory / double(cs.pending_in_mem)
-		));
-
 	SessionStats s;
 	sessions->GetStats(s);
 

src/main.cc

@@ -43,7 +43,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "PersistenceSerializer.h"
 #include "EventRegistry.h"
 #include "Stats.h"
-#include "ConnCompressor.h"
 #include "DPM.h"
 #include "BroDoc.h"
 #include "Brofiler.h"
@@ -98,7 +97,6 @@ int do_notice_analysis = 0;
 int rule_bench = 0;
 int generate_documentation = 0;
 SecondaryPath* secondary_path = 0;
-ConnCompressor* conn_compressor = 0;
 extern char version[];
 char* command_line_policy = 0;
 vector<string> params;
@@ -301,7 +299,6 @@ void terminate_bro()
 	delete state_serializer;
 	delete event_registry;
 	delete secondary_path;
-	delete conn_compressor;
 	delete remote_serializer;
 	delete dpm;
 	delete log_mgr;
@@ -829,8 +826,6 @@ int main(int argc, char** argv)
 
 	delete [] script_rule_files;
 
-	conn_compressor = new ConnCompressor();
-
 	if ( g_policy_debug )
 		// ### Add support for debug command file.
 		dbg_init_debugger(0);

testing/btest/Baseline/core.conn-uid/output.cc

@@ -1,43 +0,0 @@
-[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf
-[orig_h=fe80::217:f2ff:fed7:cf65, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp], arKYeMETxOg
-[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], k6kgXLOoSKl
-[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], nQcgTWjvg4c
-[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], j4u32Pc5bif
-[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], j4u32Pc5bif
-[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], TEfuqmmG4bh
-[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], FrJExwHcSal
-[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], 5OKnoww6xl4
-[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 3PKsZ2Uye21
-[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], VW0XPVINV8a
-[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], fRFu0wcOle6
-[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], qSsw6ESzHV4
-[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], iE6yhOq3SF
-[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], GSxOnSLghOa
-[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], qCaWGmzFtM5
-[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], 70MGiRM1Qf4
-[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], h5DsfNtYzi1
-[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], P654jzLoe3a
-[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], Tw8jXtpTGu6
-[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], c4Zw9TmAE05
-[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], EAr0uf4mhq
-[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], GvmoxJFXdTa
-[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 0Q4FH8sESw5
-[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], slFea8xwSmb
-[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], UfGkYA2HI2g
-[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], i2rO3KD1Syg
-[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], VW0XPVINV8a
-[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 3PKsZ2Uye21
-[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], GSxOnSLghOa
-[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], Tw8jXtpTGu6
-[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], P654jzLoe3a
-[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 0Q4FH8sESw5
-[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], i2rO3KD1Syg
-[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], 2cx26uAvUPl
-[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], 2cx26uAvUPl
-[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], BWaU4aSuwkc
-[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], 10XodEwRycf
-[orig_h=fe80::3074:17d5:2052:c324, orig_p=65373/udp, resp_h=ff02::1:3, resp_p=5355/udp], zno26fFZkrh
-[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], v5rgkJBig5l
-[orig_h=fe80::3074:17d5:2052:c324, orig_p=54213/udp, resp_h=ff02::1:3, resp_p=5355/udp], eWZCH7OONC1
-[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], 0Pwk3ntf8O3
-[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], 0HKorjr8Zp7

testing/btest/Baseline/core.conn-uid/output.cc2

@@ -1,43 +0,0 @@
-[orig_h=141.142.220.202, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], UWkUyAuUGXf
-[orig_h=fe80::217:f2ff:fed7:cf65, orig_p=5353/udp, resp_h=ff02::fb, resp_p=5353/udp], arKYeMETxOg
-[orig_h=141.142.220.50, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], k6kgXLOoSKl
-[orig_h=141.142.220.118, orig_p=35634/tcp, resp_h=208.80.152.2, resp_p=80/tcp], nQcgTWjvg4c
-[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], j4u32Pc5bif
-[orig_h=141.142.220.118, orig_p=48649/tcp, resp_h=208.80.152.118, resp_p=80/tcp], j4u32Pc5bif
-[orig_h=141.142.220.118, orig_p=43927/udp, resp_h=141.142.2.2, resp_p=53/udp], TEfuqmmG4bh
-[orig_h=141.142.220.118, orig_p=37676/udp, resp_h=141.142.2.2, resp_p=53/udp], FrJExwHcSal
-[orig_h=141.142.220.118, orig_p=40526/udp, resp_h=141.142.2.2, resp_p=53/udp], 5OKnoww6xl4
-[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 3PKsZ2Uye21
-[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], VW0XPVINV8a
-[orig_h=141.142.220.118, orig_p=32902/udp, resp_h=141.142.2.2, resp_p=53/udp], fRFu0wcOle6
-[orig_h=141.142.220.118, orig_p=59816/udp, resp_h=141.142.2.2, resp_p=53/udp], qSsw6ESzHV4
-[orig_h=141.142.220.118, orig_p=59714/udp, resp_h=141.142.2.2, resp_p=53/udp], iE6yhOq3SF
-[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], GSxOnSLghOa
-[orig_h=141.142.220.118, orig_p=58206/udp, resp_h=141.142.2.2, resp_p=53/udp], qCaWGmzFtM5
-[orig_h=141.142.220.118, orig_p=38911/udp, resp_h=141.142.2.2, resp_p=53/udp], 70MGiRM1Qf4
-[orig_h=141.142.220.118, orig_p=59746/udp, resp_h=141.142.2.2, resp_p=53/udp], h5DsfNtYzi1
-[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], P654jzLoe3a
-[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], Tw8jXtpTGu6
-[orig_h=141.142.220.118, orig_p=45000/udp, resp_h=141.142.2.2, resp_p=53/udp], c4Zw9TmAE05
-[orig_h=141.142.220.118, orig_p=48479/udp, resp_h=141.142.2.2, resp_p=53/udp], EAr0uf4mhq
-[orig_h=141.142.220.118, orig_p=48128/udp, resp_h=141.142.2.2, resp_p=53/udp], GvmoxJFXdTa
-[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 0Q4FH8sESw5
-[orig_h=141.142.220.118, orig_p=56056/udp, resp_h=141.142.2.2, resp_p=53/udp], slFea8xwSmb
-[orig_h=141.142.220.118, orig_p=55092/udp, resp_h=141.142.2.2, resp_p=53/udp], UfGkYA2HI2g
-[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], i2rO3KD1Syg
-[orig_h=141.142.220.118, orig_p=49997/tcp, resp_h=208.80.152.3, resp_p=80/tcp], VW0XPVINV8a
-[orig_h=141.142.220.118, orig_p=49996/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 3PKsZ2Uye21
-[orig_h=141.142.220.118, orig_p=49998/tcp, resp_h=208.80.152.3, resp_p=80/tcp], GSxOnSLghOa
-[orig_h=141.142.220.118, orig_p=50000/tcp, resp_h=208.80.152.3, resp_p=80/tcp], Tw8jXtpTGu6
-[orig_h=141.142.220.118, orig_p=49999/tcp, resp_h=208.80.152.3, resp_p=80/tcp], P654jzLoe3a
-[orig_h=141.142.220.118, orig_p=50001/tcp, resp_h=208.80.152.3, resp_p=80/tcp], 0Q4FH8sESw5
-[orig_h=141.142.220.118, orig_p=35642/tcp, resp_h=208.80.152.2, resp_p=80/tcp], i2rO3KD1Syg
-[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], 2cx26uAvUPl
-[orig_h=141.142.220.235, orig_p=6705/tcp, resp_h=173.192.163.128, resp_p=80/tcp], 2cx26uAvUPl
-[orig_h=141.142.220.44, orig_p=5353/udp, resp_h=224.0.0.251, resp_p=5353/udp], BWaU4aSuwkc
-[orig_h=141.142.220.226, orig_p=137/udp, resp_h=141.142.220.255, resp_p=137/udp], 10XodEwRycf
-[orig_h=fe80::3074:17d5:2052:c324, orig_p=65373/udp, resp_h=ff02::1:3, resp_p=5355/udp], zno26fFZkrh
-[orig_h=141.142.220.226, orig_p=55131/udp, resp_h=224.0.0.252, resp_p=5355/udp], v5rgkJBig5l
-[orig_h=fe80::3074:17d5:2052:c324, orig_p=54213/udp, resp_h=ff02::1:3, resp_p=5355/udp], eWZCH7OONC1
-[orig_h=141.142.220.226, orig_p=55671/udp, resp_h=224.0.0.252, resp_p=5355/udp], 0Pwk3ntf8O3
-[orig_h=141.142.220.238, orig_p=56641/udp, resp_h=141.142.220.255, resp_p=137/udp], 0HKorjr8Zp7

testing/btest/Baseline/istate.broccoli-ipv6/bro..stdout

@@ -0,0 +1,9 @@
+handshake done with peer
+bro_addr(1.2.3.4)
+bro_subnet(10.0.0.0/16)
+bro_addr(2607:f8b0:4009:802::1014)
+bro_subnet(2607:f8b0::/32)
+broccoli_addr(1.2.3.4)
+broccoli_subnet(10.0.0.0/16)
+broccoli_addr(2607:f8b0:4009:802::1014)
+broccoli_subnet(2607:f8b0::/32)

testing/btest/Baseline/istate.broccoli-ipv6/broccoli..stdout

@@ -0,0 +1,6 @@
+Connected to Bro instance at: localhost:47757
+Received bro_addr(1.2.3.4)
+Received bro_subnet(10.0.0.0/16)
+Received bro_addr(2607:f8b0:4009:802::1014)
+Received bro_subnet(2607:f8b0::/32)
+Terminating

testing/btest/Baseline/istate.pybroccoli/bro..stdout

@@ -1,14 +1,16 @@
 ==== atomic
 -10
 2
-1313624487.48817
+1330035434.516896
 2.0 mins
 F
 1.5
 Servus
 5555/tcp
 6.7.6.5
+2001:db8:85a3::8a2e:370:7334
 192.168.0.0/16
+2001:db8:85a3::/48
 ==== record
 [a=42, b=6.6.7.7]
 42, 6.6.7.7

testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered

@@ -1,7 +1,7 @@
 ==== atomic a 1 ====
 -4L -4
 42 42
-1313624487.4889
+1330035434.5180
 60.0
 True True
 3.14
@@ -9,10 +9,12 @@ True True
 '12345/udp' 12345/udp
 '1.2.3.4' 1.2.3.4
 '22.33.44.0/24' 22.33.44.0/24
+'2607:f8b0:4009:802::1014' 2607:f8b0:4009:802::1014
+'2607:f8b0::/32' 2607:f8b0::/32
 ==== atomic a 2 ====
 -10L -10
 2 2
-1313624487.4882
+1330035434.5169
 120.0
 False False
 1.5
@@ -20,10 +22,12 @@ False False
 '5555/tcp' 5555/tcp
 '6.7.6.5' 6.7.6.5
 '192.168.0.0/16' 192.168.0.0/16
+'2001:db8:85a3::8a2e:370:7334' 2001:db8:85a3::8a2e:370:7334
+'2001:db8:85a3::/48' 2001:db8:85a3::/48
 ==== atomic b 2 ====
 -10L -10
 <broccoli.count instance at > 2
-<broccoli.time instance at > 1313624487.4882
+<broccoli.time instance at > 1330035434.5169
 <broccoli.interval instance at > 120.0
 False False
 1.5
@@ -31,6 +35,8 @@ False False
 <broccoli.port instance at > 5555/tcp
 <broccoli.addr instance at > 6.7.6.5
 <broccoli.subnet instance at > 192.168.0.0/16
+<broccoli.addr instance at > 2001:db8:85a3::8a2e:370:7334
+<broccoli.subnet instance at > 2001:db8:85a3::/48
 ==== record 1 ====
 <broccoli.record instance at >
 42L 42

testing/btest/analyzers/conn-size.bro

@@ -1,2 +1,2 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T use_connection_compressor=F
+# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
 # @TEST-EXEC: btest-diff conn.log

testing/btest/core/conn-uid.bro

@@ -9,17 +9,6 @@
 # @TEST-EXEC: unset BRO_SEED_FILE &&  bro -C -r $TRACES/wikipedia.trace %INPUT >output2
 # @TEST-EXEC: cat output output2 | sort | uniq -c | wc -l | sed 's/ //g' >counts
 # @TEST-EXEC: btest-diff counts
-#
-# Make sure it works without the connection compressor as well.
-#
-# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT use_connection_compressor=F >output.cc
-# @TEST-EXEC: btest-diff output.cc
-#
-# Make sure it works with the full connection compressor as well.
-#
-# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT cc_handle_only_syns=F >output.cc2
-# @TEST-EXEC: btest-diff output.cc2
-
 
 event new_connection(c: connection)
 	{

testing/btest/istate/broccoli-ipv6.bro

@@ -0,0 +1,13 @@
+# @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
+#
+# @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broccoli-v6addrs.bro
+# @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs
+# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-diff bro/.stdout
+# @TEST-EXEC: btest-diff broccoli/.stdout
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+