Merge remote-tracking branch 'origin/master' into topic/icmp6

CHANGES

@@ -1,4 +1,63 @@
 
+2.0-257 | 2012-04-05 15:32:43 -0700
+
+  * Fix CMake from warning about unused ENABLE_PERFTOOLS_DEBUG
+    variable. (Jon Siwek)
+
+  * Fix handling of IPv6 atomic fragments. (Jon Siwek)
+    
+  * Fix that prevents Bro processes that do neither local logging nor
+    request remote logs from spawning threads. (Robin Sommer)
+
+  * Fixing perftools-debug support. (Robin Sommer)
+
+  * Reverting SocketComm change tuning I/O behaviour. (Robin Sommer)
+    
+  * Adding notice_policy.log canonification for external tests. (Robin Sommer)
+
+
+2.0-245 | 2012-04-04 17:25:20 -0700
+
+  * Internal restructuring of the logging framework: we now spawn
+    threads doing the I/O. From a user's perspective not much should
+    change, except that the OS may now show a bunch of Bro threads.
+    (Gilbert Clark and Robin Sommer).
+
+  * When building Bro, we now always link in tcmalloc if it's found at
+    configure time. If it's installed but not picked up,
+    --with-perftools may help. (Robin Sommer)
+
+  * Renaming the configure option --enable-perftools to
+    --enable-perftool-debug to indicate that the switch is only
+    relevant for debugging the heap. It's not needed to pick up
+    tcmalloc for better performance. (Robin Sommer)
+
+2.0-184 | 2012-03-28 15:11:11 -0700
+
+  * Improve handling of IPv6 Routing Type 0 headers.  (Jon Siwek)
+
+    - For RH0 headers with non-zero segments left, a
+      "routing0_segleft" flow_weird event is raised (with a
+      destination indicating the last address in the routing header),
+      and an "rh0_segleft" event can also be handled if the other
+      contents of the packet header are of interest. No further
+      analysis is done as the complexity required to correctly
+      identify destination endpoints of connections doesn't seem worth
+      it as RH0 has been deprecated by RFC 5095.
+
+    - For RH0 headers without any segments left, a "routing0_header"
+      flow_weird event is raised, but further analysis still occurs as
+      normal.
+
+2.0-182 | 2012-03-28 15:01:57 -0700
+
+  * Remove dead tcp_checksum function from net_util. (Jon Siwek)
+
+  * Change routing0_data_to_addrs BIF to return vector of addresses.
+    The order of addresses in type 0 routing headers is
+    interesting/important. (Jon Siwek)
+
+
 2.0-179 | 2012-03-23 17:43:31 -0700
 
   * Remove the default "tcp or udp or icmp" filter. In default mode,

CMakeLists.txt

@@ -89,15 +89,29 @@ if (LIBGEOIP_FOUND)
 endif ()
 
 set(USE_PERFTOOLS false)
-if (ENABLE_PERFTOOLS)
+set(USE_PERFTOOLS_DEBUG false)
+
 find_package(GooglePerftools)
+
 if (GOOGLEPERFTOOLS_FOUND)
-        set(USE_PERFTOOLS true)
     include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+    set(USE_PERFTOOLS true)
+
+    if (ENABLE_PERFTOOLS_DEBUG)
+        # Enable heap debugging with perftools.
+        set(USE_PERFTOOLS_DEBUG true)
+        list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
+    else ()
+        # Link in tcmalloc for better performance.
         list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
     endif ()
 endif ()
 
+if (ENABLE_PERFTOOLS_DEBUG)
+    # Just a no op to prevent CMake from complaining about manually-specified
+    # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
+endif ()
+
 set(brodeps
     ${BinPAC_LIBRARY}
     ${PCAP_LIBRARY}
@@ -183,6 +197,7 @@ message(
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
     "\nGoogle perftools:  ${USE_PERFTOOLS}"
+    "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
     "\n"
     "\n================================================================\n"
 )

NEWS

@@ -13,6 +13,10 @@ Bro 2.1
 
     * Bro now requires CMake >= 2.6.3.
 
+    * Bro now links in tcmalloc (part of Google perftools) if found at
+      configure time. Doing so can significantly improve memory and
+      CPU use.
+
 - Bro now supports IPv6 out of the box; the configure switch
   --enable-brov6 is gone.
 
@@ -31,6 +35,14 @@ Bro 2.1
 - The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012"
   to "[2607:f8b0:4009:802::1012]".
 
+- Bro now spawn threads for doing its logging. From a user's
+  perspective not much should change, except that the OS may now show
+  a bunch of Bro threads.
+
+- We renamed the configure option --enable-perftools to
+  --enable-perftool-debug to indicate that the switch is only relevant
+  for debugging the heap.
+
 TODO: Extend.
 
 Bro 2.0

VERSION

@@ -1 +1 @@
-2.0-179
+2.0-257

aux/binpac

@@ -1 +1 @@
-Subproject commit dd1a3a95f07082efcd5274b21104a038d523d132
+Subproject commit 56ae73ab995dda665d8918d1a6b3af39b15991e3

aux/bro-aux

@@ -1 +1 @@
-Subproject commit a59b35bdada8f70fb1a59bf7bb2976534c86d378
+Subproject commit 12d32194c19d2dce06818588a2aeccf234de1889

aux/broccoli

@@ -1 +1 @@
-Subproject commit a4046c2f79b6ab0ac19ae8be94b79c6ce578bea7
+Subproject commit 60898666ba1df1913c08ad5045b1e56f974060cc

aux/broctl

@@ -1 +1 @@
-Subproject commit 66e9e87beebce983fa0f479b0284d5690b0290d4
+Subproject commit 4d1a0692a7d7b5229230856a4041f70fd3a6eaa5

aux/btest

@@ -1 +1 @@
-Subproject commit dc78a3ebf5cd8fbd1b3034990e36fa21a51d1a19
+Subproject commit 8da6c55697ff580600cfff474f4ccba2a592f911

cmake

@@ -1 +1 @@
-Subproject commit 550ab2c8d95b1d3e18e40a903152650e6c7a3c45
+Subproject commit 5ddec4556338339fc4d1da27bce766a827990543

config.h.in

@@ -109,7 +109,7 @@
 #cmakedefine HAVE_GEOIP_CITY_EDITION_REV0_V6
 
 /* Use Google's perftools */
-#cmakedefine USE_PERFTOOLS
+#cmakedefine USE_PERFTOOLS_DEBUG
 
 /* Version number of package */
 #define VERSION "@VERSION@"

configure

@@ -27,7 +27,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Optional Features:
     --enable-debug         compile in debugging mode
-    --enable-perftools     use Google's perftools
+    --enable-perftools-debug use Google's perftools for debugging
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxilliary tools
@@ -91,7 +91,7 @@ append_cache_entry BRO_ROOT_DIR         PATH   /usr/local/bro
 append_cache_entry PY_MOD_INSTALL_DIR   PATH   /usr/local/bro/lib/broctl
 append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro
 append_cache_entry ENABLE_DEBUG         BOOL   false
-append_cache_entry ENABLE_PERFTOOLS     BOOL   false
+append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
@@ -132,8 +132,8 @@ while [ $# -ne 0 ]; do
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true
             ;;
-        --enable-perftools)
+        --enable-perftools-debug)
-            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
+            append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
@@ -178,7 +178,6 @@ while [ $# -ne 0 ]; do
             append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg
             ;;
         --with-perftools=*)
-            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg
             ;;
         --with-python=*)

scripts/base/init-bare.bro

@@ -46,6 +46,13 @@ type index_vec: vector of count;
 ##    then remove this alias.
 type string_vec: vector of string;
 
+## A vector of addresses.
+##
+## .. todo:: We need this type definition only for declaring builtin functions via
+##    ``bifcl``. We should extend ``bifcl`` to understand composite types directly and
+##    then remove this alias.
+type addr_vec: vector of addr;
+
 ## A table of strings indexed by strings.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions via

src/Anon.cc

@@ -154,7 +154,7 @@ void AnonymizeIPAddr_A50::init()
 int AnonymizeIPAddr_A50::PreservePrefix(ipaddr32_t input, int num_bits)
 	{
 	DEBUG_MSG("%s/%d\n",
-			IPAddr(IPAddr::IPv4, &input, IPAddr::Network).AsString().c_str(),
+			IPAddr(IPv4, &input, IPAddr::Network).AsString().c_str(),
 			num_bits);
 
 	if ( ! before_anonymization )

src/Attr.cc

@@ -5,7 +5,7 @@
 #include "Attr.h"
 #include "Expr.h"
 #include "Serializer.h"
-#include "LogMgr.h"
+#include "threading/SerialTypes.h"
 
 const char* attr_name(attr_tag t)
 	{
@@ -416,7 +416,7 @@ void Attributes::CheckAttr(Attr* a)
 		break;
 
 	case ATTR_LOG:
-		if ( ! LogVal::IsCompatibleType(type) )
+		if ( ! threading::Value::IsCompatibleType(type) )
 			Error("&log applied to a type that cannot be logged");
 		break;
 

src/CMakeLists.txt

@@ -213,6 +213,8 @@ binpac_target(syslog.pac
 ########################################################################
 ## bro target
 
+find_package (Threads)
+
 # This macro stores associated headers for any C/C++ source files given
 # as arguments (past _var) as a list in the CMake variable named "_var".
 macro(COLLECT_HEADERS _var)
@@ -335,10 +337,6 @@ set(bro_SRCS
     IRC.cc
     List.cc
     Reporter.cc
-    LogMgr.cc
-    LogWriter.cc
-    LogWriterAscii.cc
-    LogWriterNone.cc
     Login.cc
     MIME.cc
     NCP.cc
@@ -411,6 +409,18 @@ set(bro_SRCS
     PacketDumper.cc
     strsep.c
     modp_numtoa.c
+
+    threading/BasicThread.cc
+    threading/Manager.cc
+    threading/MsgThread.cc
+    threading/SerialTypes.cc
+
+    logging/Manager.cc
+    logging/WriterBackend.cc
+    logging/WriterFrontend.cc
+    logging/writers/Ascii.cc
+    logging/writers/None.cc
+
     ${dns_SRCS}
     ${openssl_SRCS}
 )
@@ -423,7 +433,7 @@ add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
-target_link_libraries(bro ${brodeps})
+target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT})
 
 install(TARGETS bro DESTINATION bin)
 install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)

src/CompHash.cc

@@ -709,7 +709,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 		const uint32* const kp = AlignType<uint32>(kp0);
 		kp1 = reinterpret_cast<const char*>(kp+4);
 
-		IPAddr addr(IPAddr::IPv6, kp, IPAddr::Network);
+		IPAddr addr(IPv6, kp, IPAddr::Network);
 
 		switch ( tag ) {
 		case TYPE_ADDR:

src/DCE_RPC.cc

@@ -137,7 +137,7 @@ static bool is_mapped_dce_rpc_endpoint(const dce_rpc_endpoint_addr& addr)
 
 bool is_mapped_dce_rpc_endpoint(const ConnID* id, TransportProto proto)
 	{
-	if ( id->dst_addr.GetFamily() == IPAddr::IPv6 )
+	if ( id->dst_addr.GetFamily() == IPv6 )
 		// TODO: Does the protocol support v6 addresses? #773
 		return false;
 
@@ -414,7 +414,7 @@ void DCE_RPC_Session::DeliverEpmapperMapResponse(
 
 				case binpac::DCE_RPC_Simple::EPM_PROTOCOL_IP:
 					uint32 hostip = floor->rhs()->data()->ip();
-					mapped.addr.addr = IPAddr(IPAddr::IPv4, &hostip, IPAddr::Host);
+					mapped.addr.addr = IPAddr(IPv4, &hostip, IPAddr::Host);
 					break;
 				}
 				}

src/DNS_Mgr.cc

@@ -321,10 +321,10 @@ void DNS_Mapping::Init(struct hostent* h)
 		addrs = new IPAddr[num_addrs];
 		for ( int i = 0; i < num_addrs; ++i )
 			if ( h->h_addrtype == AF_INET )
-				addrs[i] = IPAddr(IPAddr::IPv4, (uint32*)h->h_addr_list[i],
+				addrs[i] = IPAddr(IPv4, (uint32*)h->h_addr_list[i],
 				                  IPAddr::Network);
 			else if ( h->h_addrtype == AF_INET6 )
-				addrs[i] = IPAddr(IPAddr::IPv6, (uint32*)h->h_addr_list[i],
+				addrs[i] = IPAddr(IPv6, (uint32*)h->h_addr_list[i],
 				                  IPAddr::Network);
 		}
 	else

src/DPM.cc

@@ -74,7 +74,7 @@ void DPM::PostScriptInit()
 
 void DPM::AddConfig(const Analyzer::Config& cfg)
 	{
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	HeapLeakChecker::Disabler disabler;
 #endif
 

src/DebugLogger.cc

@@ -15,7 +15,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
-	{ "logging", 0, false }
+	{ "logging", 0, false }, { "threading", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -24,6 +24,7 @@ enum DebugStream {
 	DBG_DPD,	// Dynamic application detection framework
 	DBG_TM,		// Time-machine packet input via Brocolli
 	DBG_LOGGING,	// Logging streams
+	DBG_THREADING,	// Threading system
 
 	NUM_DBGS // Has to be last
 };

src/Desc.cc

@@ -157,6 +157,16 @@ void ODesc::Add(double d)
 		}
 	}
 
+void ODesc::Add(const IPAddr& addr)
+	{
+	Add(addr.AsString());
+	}
+
+void ODesc::Add(const IPPrefix& prefix)
+	{
+	Add(prefix.AsString());
+	}
+
 void ODesc::AddCS(const char* s)
 	{
 	int n = strlen(s);

src/Desc.h

@@ -8,7 +8,6 @@
 #include <utility>
 
 #include "BroString.h"
-#include "IPAddr.h"
 
 typedef enum {
 	DESC_READABLE,
@@ -23,6 +22,8 @@ typedef enum {
 } desc_style;
 
 class BroFile;
+class IPAddr;
+class IPPrefix;
 
 class ODesc {
 public:
@@ -76,8 +77,8 @@ public:
 	void Add(int64 i);
 	void Add(uint64 u);
 	void Add(double d);
-	void Add(const IPAddr& addr)	{ Add(addr.AsString()); }
+	void Add(const IPAddr& addr);
-	void Add(const IPPrefix& prefix)	{ Add(prefix.AsString()); }
+	void Add(const IPPrefix& prefix);
 
 	// Add s as a counted string.
 	void AddCS(const char* s);

src/File.cc

@@ -232,7 +232,7 @@ BroFile::~BroFile()
 	delete [] access;
 	delete [] cipher_buffer;
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	heap_checker->UnIgnoreObject(this);
 #endif
 	}
@@ -255,7 +255,7 @@ void BroFile::Init()
 	cipher_ctx = 0;
 	cipher_buffer = 0;
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	heap_checker->IgnoreObject(this);
 #endif
 	}

src/Frag.cc

@@ -52,8 +52,6 @@ FragReassembler::FragReassembler(NetSessions* arg_s,
 	frag_size = 0;	// flag meaning "not known"
 	next_proto = ip->NextProto();
 
-	AddFragment(t, ip, pkt);
-
 	if ( frag_timeout != 0.0 )
 		{
 		expire_timer = new FragTimer(this, t + frag_timeout);
@@ -61,6 +59,8 @@ FragReassembler::FragReassembler(NetSessions* arg_s,
 		}
 	else
 		expire_timer = 0;
+
+	AddFragment(t, ip, pkt);
 	}
 
 FragReassembler::~FragReassembler()

src/ID.cc

@@ -372,7 +372,7 @@ ID* ID::Unserialize(UnserialInfo* info)
 
 		Ref(id);
 		global_scope()->Insert(id->Name(), id);
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 		heap_checker->IgnoreObject(id);
 #endif
 		}

src/IP.cc

@@ -74,8 +74,8 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const
 		rv->Assign(2, new Val(ntohs(ip6->ip6_plen), TYPE_COUNT));
 		rv->Assign(3, new Val(ip6->ip6_nxt, TYPE_COUNT));
 		rv->Assign(4, new Val(ip6->ip6_hlim, TYPE_COUNT));
-		rv->Assign(5, new AddrVal(ip6->ip6_src));
+		rv->Assign(5, new AddrVal(IPAddr(ip6->ip6_src)));
-		rv->Assign(6, new AddrVal(ip6->ip6_dst));
+		rv->Assign(6, new AddrVal(IPAddr(ip6->ip6_dst)));
 		if ( ! chain )
 			chain = new VectorVal(new VectorType(
 			        hdrType(ip6_ext_hdr_type, "ip6_ext_hdr")->Ref()));
@@ -305,6 +305,24 @@ void IPv6_Hdr_Chain::Init(const struct ip6_hdr* ip6, bool set_next, uint16 next)
 
 		chain.push_back(p);
 
+		// RFC 5095 deprecates routing type 0 headers, so raise weirds for that.
+		if ( current_type == IPPROTO_ROUTING &&
+		     ((const struct ip6_rthdr*)hdrs)->ip6r_type == 0 )
+			{
+			IPAddr src(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_src);
+
+			if ( ((const struct ip6_rthdr*)hdrs)->ip6r_segleft > 0 )
+				{
+				const in6_addr* a = (const in6_addr*)(hdrs+len-16);
+				reporter->Weird(src, IPAddr(*a), "routing0_segleft");
+				}
+			else
+				{
+				IPAddr dst(((const struct ip6_hdr*)(chain[0]->Data()))->ip6_dst);
+				reporter->Weird(src, dst, "routing0_header");
+				}
+			}
+
 		hdrs += len;
 		length += len;
 		} while ( current_type != IPPROTO_FRAGMENT &&

src/IP.h

@@ -171,6 +171,20 @@ public:
 		{ return IsFragment() ?
 				(ntohs(GetFragHdr()->ip6f_offlg) & 0x0001) != 0 : 0; }
 
+	/**
+	 * Returns whether the chain contains a routing type 0 extension header
+	 * with nonzero segments left.
+	 */
+	bool RH0SegLeft() const
+		{
+		for ( size_t i = 0; i < chain.size(); ++i )
+			if ( chain[i]->Type() == IPPROTO_ROUTING  &&
+			     ((const struct ip6_rthdr*)chain[i]->Data())->ip6r_type == 0 &&
+			     ((const struct ip6_rthdr*)chain[i]->Data())->ip6r_segleft > 0 )
+				return true;
+		return false;
+		}
+
 	/**
 	 * Returns a vector of ip6_ext_hdr RecordVals that includes script-layer
 	 * representation of all extension headers in the chain.
@@ -343,6 +357,13 @@ public:
 	size_t NumHeaders() const
 		{ return ip4 ? 1 : ip6_hdrs->Size(); }
 
+	/**
+	 * Returns true if this is an IPv6 header containing a routing type 0
+	 * extension with nonzero segments left, else returns false.
+	 */
+	bool RH0SegLeft() const
+		{ return ip4 ? false : ip6_hdrs->RH0SegLeft(); }
+
 	/**
 	 * Returns an ip_hdr or ip6_hdr_chain RecordVal.
 	 */

src/IPAddr.cc

@@ -251,7 +251,7 @@ IPPrefix::IPPrefix(const in6_addr& in6, uint8_t length)
 IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length)
 	: prefix(addr)
 	{
-	if ( prefix.GetFamily() == IPAddr::IPv4 )
+	if ( prefix.GetFamily() == IPv4 )
 		{
 		if ( length > 32 )
 			reporter->InternalError("Bad IPAddr(v4) IPPrefix length : %d",
@@ -276,7 +276,7 @@ string IPPrefix::AsString() const
 	{
 	char l[16];
 
-	if ( prefix.GetFamily() == IPAddr::IPv4 )
+	if ( prefix.GetFamily() == IPv4 )
 		modp_uitoa10(length - 96, l);
 	else
 		modp_uitoa10(length, l);

src/IPAddr.h

@@ -10,6 +10,8 @@
 #include "BroString.h"
 #include "Hash.h"
 #include "util.h"
+#include "Type.h"
+#include "threading/SerialTypes.h"
 
 struct ConnID;
 class ExpectedConn;
@@ -25,7 +27,7 @@ public:
 	/**
 	 * Address family.
 	 */
-	enum Family { IPv4, IPv6 };
+	typedef IPFamily Family;
 
 	/**
 	 * Byte order.
@@ -45,7 +47,7 @@ public:
 	 *
 	 * @param in6 The IPv6 address.
 	 */
-	IPAddr(const in4_addr& in4)
+	explicit IPAddr(const in4_addr& in4)
 		{
 		memcpy(in6.s6_addr, v4_mapped_prefix, sizeof(v4_mapped_prefix));
 		memcpy(&in6.s6_addr[12], &in4.s_addr, sizeof(in4.s_addr));
@@ -56,7 +58,7 @@ public:
 	 *
 	 * @param in6 The IPv6 address.
 	 */
-	IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { }
+	explicit IPAddr(const in6_addr& arg_in6) : in6(arg_in6) { }
 
 	/**
 	 * Constructs an address instance from a string representation.
@@ -318,14 +320,19 @@ public:
 		return memcmp(&addr1.in6, &addr2.in6, sizeof(in6_addr)) < 0;
 		}
 
+	/** Converts the address into the type used internally by the
+	  * inter-thread communication.
+	  */
+	void ConvertToThreadingValue(threading::Value::addr_t* v) const;
+
 	friend HashKey* BuildConnIDHashKey(const ConnID& id);
 	friend HashKey* BuildExpectedConnHashKey(const ExpectedConn& c);
 
-	friend class IPPrefix;
-
 	unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
 
 private:
+	friend class IPPrefix;
+
 	/**
 	 * Initializes an address instance from a string representation.
 	 *
@@ -384,6 +391,25 @@ inline bool IPAddr::IsLoopback() const
 			&& (in6.s6_addr[14] == 0) && (in6.s6_addr[15] == 1));
 	}
 
+inline void IPAddr::ConvertToThreadingValue(threading::Value::addr_t* v) const
+	{
+	v->family = GetFamily();
+
+	switch ( v->family ) {
+
+	case IPv4:
+		CopyIPv4(&v->in.in4);
+		return;
+
+	case IPv6:
+		CopyIPv6(&v->in.in6);
+		return;
+
+	// Can't be reached.
+	abort();
+	}
+	}
+
 /**
   * Returns a hash key for a given ConnID. Passes ownership to caller.
   */
@@ -459,7 +485,7 @@ public:
 	 */
 	uint8_t Length() const
 		{
-		return prefix.GetFamily() == IPAddr::IPv4 ? length - 96 : length;
+		return prefix.GetFamily() == IPv4 ? length - 96 : length;
 		}
 
 	/**
@@ -516,6 +542,15 @@ public:
 		return new HashKey(&key, sizeof(key));
 		}
 
+	/** Converts the prefix into the type used internally by the
+	  * inter-thread communication.
+	  */
+	void ConvertToThreadingValue(threading::Value::subnet_t* v) const
+		{
+		v->length = length;
+		prefix.ConvertToThreadingValue(&v->prefix);
+		}
+
 	unsigned int MemoryAllocation() const { return padded_sizeof(*this); }
 
 	/**

src/LogMgr.h

@@ -1,143 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// A class managing log writers and filters.
-
-#ifndef LOGMGR_H
-#define LOGMGR_H
-
-#include "Val.h"
-#include "EventHandler.h"
-#include "RemoteSerializer.h"
-#include "IPAddr.h"
-
-class SerializationFormat;
-
-// Description of a log field.
-struct LogField {
-	string name;
-	TypeTag type;
-	// inner type of sets
-	TypeTag subtype;
-
-	LogField() 	{ subtype = TYPE_VOID; }
-	LogField(const LogField& other)
-		: name(other.name), type(other.type), subtype(other.subtype) {  }
-
-	// (Un-)serialize.
-	bool Read(SerializationFormat* fmt);
-	bool Write(SerializationFormat* fmt) const;
-};
-
-// Values as logged by a writer.
-struct LogVal {
-	TypeTag type;
-	bool present; // False for unset fields.
-
-	// The following union is a subset of BroValUnion, including only the
-	// types we can log directly.
-	struct set_t { bro_int_t size; LogVal** vals; };
-	typedef set_t vec_t;
-
-	union _val {
-		bro_int_t int_val;
-		bro_uint_t uint_val;
-		IPAddr* addr_val;
-		IPPrefix* subnet_val;
-		double double_val;
-		string* string_val;
-		set_t set_val;
-		vec_t vector_val;
-	} val;
-
-	LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true)
-		: type(arg_type), present(arg_present)	{}
-	~LogVal();
-
-	// (Un-)serialize.
-	bool Read(SerializationFormat* fmt);
-	bool Write(SerializationFormat* fmt) const;
-
-	// Returns true if the type can be logged the framework. If
-	// `atomic_only` is true, will not permit composite types.
-	static bool IsCompatibleType(BroType* t, bool atomic_only=false);
-
-private:
-	LogVal(const LogVal& other)	{ }
-};
-
-class LogWriter;
-class RemoteSerializer;
-class RotationTimer;
-
-class LogMgr {
-public:
-	LogMgr();
-	~LogMgr();
-
-	// These correspond to the BiFs visible on the scripting layer. The
-	// actual BiFs just forward here.
-	bool CreateStream(EnumVal* id, RecordVal* stream);
-	bool EnableStream(EnumVal* id);
-	bool DisableStream(EnumVal* id);
-	bool AddFilter(EnumVal* id, RecordVal* filter);
-	bool RemoveFilter(EnumVal* id, StringVal* name);
-	bool RemoveFilter(EnumVal* id, string name);
-	bool Write(EnumVal* id, RecordVal* columns);
-	bool SetBuf(EnumVal* id, bool enabled);	// Adjusts all writers.
-	bool Flush(EnumVal* id);		// Flushes all writers..
-
-protected:
-	friend class LogWriter;
-	friend class RemoteSerializer;
-	friend class RotationTimer;
-
-	//// Function also used by the RemoteSerializer.
-
-	// Takes ownership of fields.
-	LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path,
-				int num_fields, LogField** fields);
-
-	// Takes ownership of values..
-	bool Write(EnumVal* id, EnumVal* writer, string path,
-		   int num_fields, LogVal** vals);
-
-	// Announces all instantiated writers to peer.
-	void SendAllWritersTo(RemoteSerializer::PeerID peer);
-
-	//// Functions safe to use by writers.
-
-	// Signals that a file has been rotated.
-	bool FinishedRotation(LogWriter* writer, string new_name, string old_name,
-			      double open, double close, bool terminating);
-
-	// Reports an error for the given writer.
-	void Error(LogWriter* writer, const char* msg);
-
-	// Deletes the values as passed into Write().
-	void DeleteVals(int num_fields, LogVal** vals);
-
-private:
-	struct Filter;
-	struct Stream;
-	struct WriterInfo;
-
-	bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
-			    TableVal* include, TableVal* exclude, string path, list<int> indices);
-
-	LogVal** RecordToFilterVals(Stream* stream, Filter* filter,
-				    RecordVal* columns);
-
-	LogVal* ValToLogVal(Val* val, BroType* ty = 0);
-	Stream* FindStream(EnumVal* id);
-	void RemoveDisabledWriters(Stream* stream);
-	void InstallRotationTimer(WriterInfo* winfo);
-	void Rotate(WriterInfo* info);
-	Filter* FindFilter(EnumVal* id, StringVal* filter);
-	WriterInfo* FindWriter(LogWriter* writer);
-
-	vector<Stream *> streams;	// Indexed by stream enum.
-};
-
-extern LogMgr* log_mgr;
-
-#endif

src/LogWriter.cc

@@ -1,158 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-
-#include "util.h"
-#include "LogWriter.h"
-
-LogWriter::LogWriter()
-	{
-	buf = 0;
-	buf_len = 1024;
-	buffering = true;
-	disabled = false;
-	}
-
-LogWriter::~LogWriter()
-	{
-	if ( buf )
-		free(buf);
-
-	for(int i = 0; i < num_fields; ++i)
-		delete fields[i];
-
-	delete [] fields;
-	}
-
-bool LogWriter::Init(string arg_path, int arg_num_fields,
-		     const LogField* const * arg_fields)
-	{
-	path = arg_path;
-	num_fields = arg_num_fields;
-	fields = arg_fields;
-
-	if ( ! DoInit(arg_path, arg_num_fields, arg_fields) )
-		{
-		disabled = true;
-		return false;
-		}
-
-	return true;
-	}
-
-bool LogWriter::Write(int arg_num_fields, LogVal** vals)
-	{
-	// Double-check that the arguments match. If we get this from remote,
-	// something might be mixed up.
-	if ( num_fields != arg_num_fields )
-		{
-		DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)",
-			arg_num_fields, num_fields);
-
-		DeleteVals(vals);
-		return false;
-		}
-
-	for ( int i = 0; i < num_fields; ++i )
-		{
-		if ( vals[i]->type != fields[i]->type )
-			{
-			DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)",
-				vals[i]->type, fields[i]->type);
-			DeleteVals(vals);
-			return false;
-			}
-		}
-
-	bool result = DoWrite(num_fields, fields, vals);
-
-	DeleteVals(vals);
-
-	if ( ! result )
-		disabled = true;
-
-	return result;
-	}
-
-bool LogWriter::SetBuf(bool enabled)
-	{
-	if ( enabled == buffering )
-		// No change.
-		return true;
-
-	buffering = enabled;
-
-	if ( ! DoSetBuf(enabled) )
-		{
-		disabled = true;
-		return false;
-		}
-
-	return true;
-	}
-
-bool LogWriter::Rotate(string rotated_path, double open,
-		       double close, bool terminating)
-	{
-	if ( ! DoRotate(rotated_path, open, close, terminating) )
-		{
-		disabled = true;
-		return false;
-		}
-
-	return true;
-	}
-
-bool LogWriter::Flush()
-	{
-	if ( ! DoFlush() )
-		{
-		disabled = true;
-		return false;
-		}
-
-	return true;
-	}
-
-void LogWriter::Finish()
-	{
-	DoFinish();
-	}
-
-const char* LogWriter::Fmt(const char* format, ...)
-	{
-	if ( ! buf )
-		buf = (char*) malloc(buf_len);
-
-	va_list al;
-	va_start(al, format);
-	int n = safe_vsnprintf(buf, buf_len, format, al);
-	va_end(al);
-
-	if ( (unsigned int) n >= buf_len )
-		{ // Not enough room, grow the buffer.
-		buf_len = n + 32;
-		buf = (char*) realloc(buf, buf_len);
-
-		// Is it portable to restart?
-		va_start(al, format);
-		n = safe_vsnprintf(buf, buf_len, format, al);
-		va_end(al);
-		}
-
-	return buf;
-	}
-
-void LogWriter::Error(const char *msg)
-	{
-	log_mgr->Error(this, msg);
-	}
-
-void LogWriter::DeleteVals(LogVal** vals)
-	{
-	log_mgr->DeleteVals(num_fields, vals);
-	}
-
-bool LogWriter::FinishedRotation(string new_name, string old_name, double open,
-				 double close, bool terminating)
-	{
-	return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating);
-	}

src/LogWriter.h

@@ -1,192 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// Interface API for a log writer backend. The LogMgr creates a separate
-// writer instance of pair of (writer type, output path).
-//
-// Note thay classes derived from LogWriter must be fully thread-safe and not
-// use any non-thread-safe Bro functionality (which includes almost
-// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!.
-//
-// The one exception to this rule is the constructor: it is guaranteed to be
-// executed inside the main thread and can thus in particular access global
-// script variables.
-
-#ifndef LOGWRITER_H
-#define LOGWRITER_H
-
-#include "LogMgr.h"
-#include "BroString.h"
-
-class LogWriter {
-public:
-	LogWriter();
-	virtual ~LogWriter();
-
-	//// Interface methods to interact with the writer. Note that these
-	//// methods are not necessarily thread-safe and must be called only
-	//// from the main thread (which will typically mean only from the
-	//// LogMgr). In particular, they must not be called from the
-	//// writer's derived implementation.
-
-	// One-time initialization of the writer to define the logged fields.
-	// Interpretation of "path" is left to the writer, and will be
-	// corresponding the value configured on the script-level.
-	// 
-	// Returns false if an error occured, in which case the writer must
-	// not be used further.
-	//
-	// The new instance takes ownership of "fields", and will delete them
-	// when done.
-	bool Init(string path, int num_fields, const LogField* const * fields);
-
-	// Writes one log entry. The method takes ownership of "vals" and
-	// will return immediately after queueing the write request, which is
-	// potentially before output has actually been written out.
-	//
-	// num_fields and the types of the LogVals must match what was passed
-	// to Init().
-	//
-	// Returns false if an error occured, in which case the writer must
-	// not be used any further.
-	bool Write(int num_fields, LogVal** vals);
-
-	// Sets the buffering status for the writer, if the writer supports
-	// that. (If not, it will be ignored).
-	bool SetBuf(bool enabled);
-
-	// Flushes any currently buffered output, if the writer supports 
-	// that. (If not, it will be ignored).
-	bool Flush();
-
-	// Triggers rotation, if the writer supports that. (If not, it will
-	// be ignored).
-	bool Rotate(string rotated_path, double open, double close, bool terminating);
-
-	// Finishes writing to this logger regularly. Must not be called if
-	// an error has been indicated earlier. After calling this, no
-	// further writing must be performed.
-	void Finish();
-
-	//// Thread-safe methods that may be called from the writer
-	//// implementation.
-
-	// The following methods return the information as passed to Init().
-	const string Path() const	{ return path; }
-	int NumFields() const	{ return num_fields; }
-	const LogField* const * Fields() const	{ return fields; }
-
-protected:
-	// Methods for writers to override. If any of these returs false, it
-	// will be assumed that a fatal error has occured that prevents the
-	// writer from further operation. It will then be disabled and
-	// deleted. When return false, the writer should also report the
-	// error via Error(). Note that even if a writer does not support the
-	// functionality for one these methods (like rotation), it must still
-	// return true if that is not to be considered a fatal error.
-	//
-	// Called once for initialization of the writer.
-	virtual bool DoInit(string path, int num_fields,
-			    const LogField* const * fields) = 0;
-
-	// Called once per log entry to record.
-	virtual bool DoWrite(int num_fields, const LogField* const * fields,
-			     LogVal** vals) = 0;
-
-	// Called when the buffering status for this writer is changed. If
-	// buffering is disabled, the writer should attempt to write out
-	// information as quickly as possible even if doing so may have a
-	// performance impact. If enabled (which is the default), it may
-	// buffer data as helpful and write it out later in a way optimized
-	// for performance. The current buffering state can be queried via
-	// IsBuf().
-	//
-	// A writer may ignore buffering changes if it doesn't fit with its
-	// semantics (but must still return true in that case).
-	virtual bool DoSetBuf(bool enabled) = 0;
-
-	// Called to flush any currently buffered output.
-	//
-	// A writer may ignore flush requests if it doesn't fit with its
-	// semantics (but must still return true in that case).
-	virtual bool DoFlush() = 0;
-
-	// Called when a log output is to be rotated. Most directly this only
-	// applies to writers writing into files, which should then close the
-	// current file and open a new one.  However, a writer may also
-	// trigger other apppropiate actions if semantics are similar.
-	// 
-	// Once rotation has finished, the implementation should call
-	// RotationDone() to signal the log manager that potential
-	// postprocessors can now run.
-	//
-	// "rotate_path" reflects the path to where the rotated output is to
-	// be moved, with specifics depending on the writer. It should
-	// generally be interpreted in a way consistent with that of "path"
-	// as passed into DoInit(). As an example, for file-based output, 
-	// "rotate_path" could be the original filename extended with a
-	// timestamp indicating the time of the rotation.
-	// 
-	// "open" and "close" are the network time's when the *current* file
-	// was opened and closed, respectively.
-	//
-	// "terminating" indicated whether the rotation request occurs due
-	// the main Bro prcoess terminating (and not because we've reach a
-	// regularly scheduled time for rotation).
-	//
-	// A writer may ignore rotation requests if it doesn't fit with its
-	// semantics (but must still return true in that case).
-	virtual bool DoRotate(string rotated_path, double open, double close,
-			      bool terminating) = 0;
-
-	// Called once on termination. Not called when any of the other
-	// methods has previously signaled an error, i.e., executing this
-	// method signals a regular shutdown of the writer.
-	virtual void DoFinish() = 0;
-
-	//// Methods for writers to use. These are thread-safe.
-
-	// A thread-safe version of fmt().
-	const char* Fmt(const char* format, ...);
-
-	// Returns the current buffering state.
-	bool IsBuf()	{ return buffering; }
-
-	// Reports an error to the user.
-	void Error(const char *msg);
-
-	// Signals to the log manager that a file has been rotated.
-	//
-	// new_name: The filename of the rotated file. old_name: The filename
-	// of the origina file.
-	//
-	// open/close: The timestamps when the original file was opened and
-	// closed, respectively.
-	//
-	// terminating: True if rotation request occured due to the main Bro
-	// process shutting down.
-	bool FinishedRotation(string new_name, string old_name, double open,
-			      double close, bool terminating);
-
-private:
-	friend class LogMgr;
-
-	// When an error occurs, we call this method to set a flag marking
-	// the writer as disabled. The LogMgr will check the flag later and
-	// remove the writer.
-	bool Disabled()	{ return disabled; }
-
-	// Deletes the values as passed into Write().
-	void DeleteVals(LogVal** vals);
-
-	string path;
-	int num_fields;
-	const LogField* const * fields;
-	bool buffering;
-	bool disabled;
-
-	// For implementing Fmt().
-	char* buf;
-	unsigned int buf_len;
-};
-
-#endif

src/LogWriterNone.h

@@ -1,30 +0,0 @@
-// See the file "COPYING" in the main distribution directory for copyright.
-//
-// Dummy log writer that just discards everything (but still pretends to rotate).
-
-#ifndef LOGWRITERNONE_H
-#define LOGWRITERNONE_H
-
-#include "LogWriter.h"
-
-class LogWriterNone : public LogWriter {
-public:
-	LogWriterNone()	{}
-	~LogWriterNone()	{};
-
-	static LogWriter* Instantiate()	{ return new LogWriterNone; }
-
-protected:
-	virtual bool DoInit(string path, int num_fields,
-			    const LogField* const * fields)	{ return true; }
-
-	virtual bool DoWrite(int num_fields, const LogField* const * fields,
-			     LogVal** vals)	{ return true; }
-	virtual bool DoSetBuf(bool enabled)	{ return true; }
-	virtual bool DoRotate(string rotated_path, double open, double close,
-			      bool terminating);
-	virtual bool DoFlush()	{ return true; }
-	virtual void DoFinish()	{}
-};
-
-#endif

src/Login.cc

@@ -38,7 +38,7 @@ Login_Analyzer::Login_Analyzer(AnalyzerTag::Tag tag, Connection* conn)
 
 	if ( ! re_skip_authentication )
 		{
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 		HeapLeakChecker::Disabler disabler;
 #endif
 		re_skip_authentication = init_RE(skip_authentication);

src/Net.cc

@@ -455,6 +455,7 @@ void net_run()
 				// date on timers and events.
 				network_time = ct;
 				expire_timers();
+				usleep(1); // Just yield.
 				}
 			}
 
@@ -484,6 +485,8 @@ void net_run()
 			// since Bro timers are not high-precision anyway.)
 			if ( ! using_communication )
 				usleep(100000);
+			else
+				usleep(1000);
 
 			// Flawfinder says about usleep:
 			//

src/PersistenceSerializer.cc

@@ -137,7 +137,7 @@ bool PersistenceSerializer::CheckForFile(UnserialInfo* info, const char* file,
 
 bool PersistenceSerializer::ReadAll(bool is_init, bool delete_files)
 	{
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	HeapLeakChecker::Disabler disabler;
 #endif
 

src/RemoteSerializer.cc

@@ -183,8 +183,9 @@
 #include "Sessions.h"
 #include "File.h"
 #include "Conn.h"
-#include "LogMgr.h"
 #include "Reporter.h"
+#include "threading/SerialTypes.h"
+#include "logging/Manager.h"
 #include "IPAddr.h"
 #include "bro_inet_ntop.h"
 
@@ -532,6 +533,7 @@ RemoteSerializer::RemoteSerializer()
 	terminating = false;
 	in_sync = 0;
 	last_flush = 0;
+	received_logs = 0;
 	}
 
 RemoteSerializer::~RemoteSerializer()
@@ -681,7 +683,7 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip,
 	if ( ! initialized )
 		reporter->InternalError("remote serializer not initialized");
 
-	if ( ip.GetFamily() == IPAddr::IPv6 )
+	if ( ip.GetFamily() == IPv6 )
 		Error("inter-Bro communication not supported over IPv6");
 
 	const uint32* bytes;
@@ -1238,7 +1240,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl)
 	if ( ! initialized )
 		reporter->InternalError("remote serializer not initialized");
 
-	if ( ip.GetFamily() == IPAddr::IPv6 )
+	if ( ip.GetFamily() == IPv6 )
 		Error("inter-Bro communication not supported over IPv6");
 
 	const uint32* bytes;
@@ -1353,6 +1355,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time)
 	{
 	Poll(false);
 
+	if ( received_logs > 0 )
+		{
+		// If we processed logs last time, assume there's more.
+		idle = false;
+		received_logs = 0;
+		return timer_mgr->Time();
+		}
+
 	double et = events.length() ? events[0]->time : -1;
 	double pt = packets.length() ? packets[0]->time : -1;
 
@@ -2470,7 +2480,7 @@ bool RemoteSerializer::ProcessRemotePrint()
 	return true;
 	}
 
-bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields)
+bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields)
 	{
 	loop_over_list(peers, i)
 		{
@@ -2480,7 +2490,7 @@ bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string
 	return true;
 	}
 
-bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields)
+bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields)
 	{
 	SetErrorDescr("logging");
 
@@ -2493,6 +2503,9 @@ bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal*
 	if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING )
 		return false;
 
+	if ( ! peer->logs_requested )
+		return false;
+
 	BinarySerializationFormat fmt;
 
 	fmt.StartWrite();
@@ -2534,7 +2547,7 @@ error:
 	return false;
 	}
 
-bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals)
+bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals)
 	{
 	loop_over_list(peers, i)
 		{
@@ -2544,7 +2557,7 @@ bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, i
 	return true;
 	}
 
-bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals)
+bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals)
 	{
 	if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING )
 		return false;
@@ -2552,7 +2565,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st
 	if ( ! peer->logs_requested )
 		return false;
 
-	assert(peer->log_buffer);
+	if ( ! peer->log_buffer )
+		// Peer shutting down.
+		return false;
 
 	// Serialize the log record entry.
 
@@ -2587,8 +2602,11 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st
 	if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) )
 		{
 		if ( ! FlushLogBuffer(peer) )
+			{
+			delete [] data;
 			return false;
 			}
+		}
 
 	// If the data is actually larger than our complete buffer, just send it out.
 	if ( len > LOG_BUFFER_SIZE )
@@ -2610,6 +2628,9 @@ error:
 
 bool RemoteSerializer::FlushLogBuffer(Peer* p)
 	{
+	if ( ! p->logs_requested )
+		return false;
+
 	last_flush = network_time;
 
 	if ( p->state == Peer::CLOSING )
@@ -2631,11 +2652,17 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 	if ( current_peer->state == Peer::CLOSING )
 		return false;
 
+#ifdef USE_PERFTOOLS_DEBUG
+	// Don't track allocations here, they'll be released only after the
+	// main loop exists. And it's just a tiny amount anyway.
+	HeapLeakChecker::Disabler disabler;
+#endif
+
 	assert(current_args);
 
 	EnumVal* id_val = 0;
 	EnumVal* writer_val = 0;
-	LogField** fields = 0;
+	threading::Field** fields = 0;
 
 	BinarySerializationFormat fmt;
 	fmt.StartRead(current_args->data, current_args->len);
@@ -2652,11 +2679,11 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 	if ( ! success )
 		goto error;
 
-	fields = new LogField* [num_fields];
+	fields = new threading::Field* [num_fields];
 
 	for ( int i = 0; i < num_fields; i++ )
 		{
-		fields[i] = new LogField;
+		fields[i] = new threading::Field;
 		if ( ! fields[i]->Read(&fmt) )
 			goto error;
 		}
@@ -2666,7 +2693,7 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 	id_val = new EnumVal(id, BifType::Enum::Log::ID);
 	writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
 
-	if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) )
+	if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) )
 		goto error;
 
 	Unref(id_val);
@@ -2697,7 +2724,7 @@ bool RemoteSerializer::ProcessLogWrite()
 		// Unserialize one entry.
 		EnumVal* id_val = 0;
 		EnumVal* writer_val = 0;
-		LogVal** vals = 0;
+		threading::Value** vals = 0;
 
 		int id, writer;
 		string path;
@@ -2711,11 +2738,11 @@ bool RemoteSerializer::ProcessLogWrite()
 		if ( ! success )
 			goto error;
 
-		vals = new LogVal* [num_fields];
+		vals = new threading::Value* [num_fields];
 
 		for ( int i = 0; i < num_fields; i++ )
 			{
-			vals[i] = new LogVal;
+			vals[i] = new threading::Value;
 			if ( ! vals[i]->Read(&fmt) )
 				goto error;
 			}
@@ -2735,6 +2762,8 @@ bool RemoteSerializer::ProcessLogWrite()
 
 	fmt.EndRead();
 
+	++received_logs;
+
 	return true;
 
 error:
@@ -2844,7 +2873,7 @@ void RemoteSerializer::GotID(ID* id, Val* val)
 					(desc && *desc) ? desc : "not set"),
 			current_peer);
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 		// May still be cached, but we don't care.
 		heap_checker->IgnoreObject(id);
 #endif
@@ -3376,6 +3405,11 @@ void SocketComm::Run()
 		small_timeout.tv_usec =
 			io->CanWrite() || io->CanRead() ? 1 : 10;
 
+#if 0
+		if ( ! io->CanWrite() )
+			usleep(10);
+#endif
+
 		int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except,
 				&small_timeout);
 

src/RemoteSerializer.h

@@ -14,8 +14,11 @@
 // FIXME: Change this to network byte order
 
 class IncrementalSendTimer;
-class LogField;
+
-class LogVal;
+namespace threading {
+	class Field;
+	class Value;
+}
 
 // This class handles the communication done in Bro's main loop.
 class RemoteSerializer : public Serializer, public IOSource {
@@ -99,13 +102,13 @@ public:
 	bool SendPrintHookEvent(BroFile* f, const char* txt, size_t len);
 
 	// Send a request to create a writer on a remote side.
-	bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields);
+	bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields);
 
 	// Broadcasts a request to create a writer.
-	bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields);
+	bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Field* const * fields);
 
 	// Broadcast a log entry to everybody interested.
-	bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals);
+	bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals);
 
 	// Synchronzizes time with all connected peers. Returns number of
 	// current sync-point, or -1 on error.
@@ -300,7 +303,7 @@ protected:
 	bool SendID(SerialInfo* info, Peer* peer, const ID& id);
 	bool SendCapabilities(Peer* peer);
 	bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p);
-	bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals);
+	bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const threading::Value* const * vals);
 
 	void UnregisterHandlers(Peer* peer);
 	void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0);
@@ -335,6 +338,7 @@ private:
 	int propagate_accesses;
 	bool ignore_accesses;
 	bool terminating;
+	int received_logs;
 	Peer* source_peer;
 	PeerID id_counter;	// Keeps track of assigned IDs.
 	uint32 current_sync_point;

src/RuleMatcher.cc

@@ -191,7 +191,7 @@ void RuleMatcher::Delete(RuleHdrTest* node)
 
 bool RuleMatcher::ReadFiles(const name_list& files)
 	{
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	HeapLeakChecker::Disabler disabler;
 #endif
 
@@ -1082,7 +1082,7 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to)
 			bool is_v4_mask = m[0] == 0xffffffff &&
 						m[1] == m[0] && m[2] == m[0];
 
-			if ( v->AsSubNet().Prefix().GetFamily() == IPAddr::IPv4 &&
+			if ( v->AsSubNet().Prefix().GetFamily() == IPv4 &&
 			     is_v4_mask )
 				{
 				mval->val = ntohl(*n);

src/SerializationFormat.cc

@@ -250,9 +250,9 @@ bool BinarySerializationFormat::Read(IPAddr* addr, const char* tag)
 		}
 
 	if ( n == 1 )
-		*addr = IPAddr(IPAddr::IPv4, raw, IPAddr::Network);
+		*addr = IPAddr(IPv4, raw, IPAddr::Network);
 	else
-		*addr = IPAddr(IPAddr::IPv6, raw, IPAddr::Network);
+		*addr = IPAddr(IPv6, raw, IPAddr::Network);
 
 	return true;
 	}
@@ -269,6 +269,32 @@ bool BinarySerializationFormat::Read(IPPrefix* prefix, const char* tag)
 	return true;
 	}
 
+bool BinarySerializationFormat::Read(struct in_addr* addr, const char* tag)
+	{
+	uint32_t* bytes = (uint32_t*) &addr->s_addr;
+
+	if ( ! Read(&bytes[0], "addr4") )
+		return false;
+
+	bytes[0] = htonl(bytes[0]);
+	return true;
+	}
+
+bool BinarySerializationFormat::Read(struct in6_addr* addr, const char* tag)
+	{
+	uint32_t* bytes = (uint32_t*) &addr->s6_addr;
+
+	for ( int i = 0; i < 4; ++i )
+		{
+		if ( ! Read(&bytes[i], "addr6-part") )
+			return false;
+
+		bytes[i] = htonl(bytes[i]);
+		}
+
+	return true;
+	}
+
 bool BinarySerializationFormat::Write(char v, const char* tag)
 	{
 	DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag);
@@ -362,6 +388,29 @@ bool BinarySerializationFormat::Write(const IPPrefix& prefix, const char* tag)
 	return Write(prefix.Prefix(), "prefix") && Write(prefix.Length(), "width");
 	}
 
+bool BinarySerializationFormat::Write(const struct in_addr& addr, const char* tag)
+	{
+	const uint32_t* bytes = (uint32_t*) &addr.s_addr;
+
+	if ( ! Write(ntohl(bytes[0]), "addr4") )
+		return false;
+
+	return true;
+	}
+
+bool BinarySerializationFormat::Write(const struct in6_addr& addr, const char* tag)
+	{
+	const uint32_t* bytes = (uint32_t*) &addr.s6_addr;
+
+	for ( int i = 0; i < 4; ++i )
+		{
+		if ( ! Write(ntohl(bytes[i]), "addr6-part") )
+			return false;
+		}
+
+	return true;
+	}
+
 bool BinarySerializationFormat::WriteOpenTag(const char* tag)
 	{
 	return true;
@@ -464,6 +513,18 @@ bool XMLSerializationFormat::Read(IPPrefix* prefix, const char* tag)
 	return false;
 	}
 
+bool XMLSerializationFormat::Read(struct in_addr* addr, const char* tag)
+	{
+	reporter->InternalError("no reading of xml");
+	return false;
+	}
+
+bool XMLSerializationFormat::Read(struct in6_addr* addr, const char* tag)
+	{
+	reporter->InternalError("no reading of xml");
+	return false;
+	}
+
 bool XMLSerializationFormat::Write(char v, const char* tag)
 	{
 	return WriteElem(tag, "char", &v, 1);
@@ -556,6 +617,18 @@ bool XMLSerializationFormat::Write(const IPPrefix& prefix, const char* tag)
 	return false;
 	}
 
+bool XMLSerializationFormat::Write(const struct in_addr& addr, const char* tag)
+	{
+	reporter->InternalError("XML output of in_addr not implemented");
+	return false;
+	}
+
+bool XMLSerializationFormat::Write(const struct in6_addr& addr, const char* tag)
+	{
+	reporter->InternalError("XML output of in6_addr not implemented");
+	return false;
+	}
+
 bool XMLSerializationFormat::WriteEncodedString(const char* s, int len)
 	{
 	while ( len-- )

src/SerializationFormat.h

@@ -9,6 +9,9 @@ using namespace std;
 
 #include "util.h"
 
+class IPAddr;
+class IPPrefix;
+
 // Abstract base class.
 class SerializationFormat {
 public:
@@ -30,6 +33,8 @@ public:
 	virtual bool Read(string* s, const char* tag) = 0;
 	virtual bool Read(IPAddr* addr, const char* tag) = 0;
 	virtual bool Read(IPPrefix* prefix, const char* tag) = 0;
+	virtual bool Read(struct in_addr* addr, const char* tag) = 0;
+	virtual bool Read(struct in6_addr* addr, const char* tag) = 0;
 
 	// Returns number of raw bytes read since last call to StartRead().
 	int BytesRead() const	{ return bytes_read; }
@@ -54,6 +59,8 @@ public:
 	virtual bool Write(const string& s, const char* tag) = 0;
 	virtual bool Write(const IPAddr& addr, const char* tag) = 0;
 	virtual bool Write(const IPPrefix& prefix, const char* tag) = 0;
+	virtual bool Write(const struct in_addr& addr, const char* tag) = 0;
+	virtual bool Write(const struct in6_addr& addr, const char* tag) = 0;
 
 	virtual bool WriteOpenTag(const char* tag) = 0;
 	virtual bool WriteCloseTag(const char* tag) = 0;
@@ -96,6 +103,8 @@ public:
 	virtual bool Read(string* s, const char* tag);
 	virtual bool Read(IPAddr* addr, const char* tag);
 	virtual bool Read(IPPrefix* prefix, const char* tag);
+	virtual bool Read(struct in_addr* addr, const char* tag);
+	virtual bool Read(struct in6_addr* addr, const char* tag);
 	virtual bool Write(int v, const char* tag);
 	virtual bool Write(uint16 v, const char* tag);
 	virtual bool Write(uint32 v, const char* tag);
@@ -109,6 +118,8 @@ public:
 	virtual bool Write(const string& s, const char* tag);
 	virtual bool Write(const IPAddr& addr, const char* tag);
 	virtual bool Write(const IPPrefix& prefix, const char* tag);
+	virtual bool Write(const struct in_addr& addr, const char* tag);
+	virtual bool Write(const struct in6_addr& addr, const char* tag);
 	virtual bool WriteOpenTag(const char* tag);
 	virtual bool WriteCloseTag(const char* tag);
 	virtual bool WriteSeparator();
@@ -133,6 +144,8 @@ public:
 	virtual bool Write(const string& s, const char* tag);
 	virtual bool Write(const IPAddr& addr, const char* tag);
 	virtual bool Write(const IPPrefix& prefix, const char* tag);
+	virtual bool Write(const struct in_addr& addr, const char* tag);
+	virtual bool Write(const struct in6_addr& addr, const char* tag);
 	virtual bool WriteOpenTag(const char* tag);
 	virtual bool WriteCloseTag(const char* tag);
 	virtual bool WriteSeparator();
@@ -150,6 +163,8 @@ public:
 	virtual bool Read(string* s, const char* tag);
 	virtual bool Read(IPAddr* addr, const char* tag);
 	virtual bool Read(IPPrefix* prefix, const char* tag);
+	virtual bool Read(struct in_addr* addr, const char* tag);
+	virtual bool Read(struct in6_addr* addr, const char* tag);
 
 private:
 	// Encodes non-printable characters.

src/Sessions.cc

@@ -482,6 +482,22 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		return;
 		}
 
+	// Stop analyzing IPv6 packets that use routing type 0 headers with segments
+	// left since RH0 headers are deprecated by RFC 5095 and we'd have to make
+	// extra effort to get the destination in the connection/flow endpoint right.
+	if ( ip_hdr->RH0SegLeft() )
+		{
+		dump_this_packet = 1;
+		if ( rh0_segleft )
+			{
+			val_list* vl = new val_list();
+			vl->append(ip_hdr->BuildPktHdrVal());
+			mgr.QueueEvent(rh0_segleft, vl);
+			}
+		Remove(f);
+		return;
+		}
+
 	int proto = ip_hdr->NextProto();
 
 	if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt) )

src/StateAccess.cc

@@ -678,7 +678,7 @@ bool StateAccess::DoUnserialize(UnserialInfo* info)
 			target.id = new ID(name, SCOPE_GLOBAL, true);
 			Ref(target.id);
 			global_scope()->Insert(name, target.id);
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 			heap_checker->IgnoreObject(target.id);
 #endif
 			}

src/Stats.cc

@@ -8,7 +8,7 @@
 #include "cq.h"
 #include "DNS_Mgr.h"
 #include "Trigger.h"
-
+#include "threading/Manager.h"
 
 int killed_by_inactivity = 0;
 
@@ -203,6 +203,25 @@ void ProfileLogger::Log()
 					current_timers[i]));
 		}
 
+	file->Write(fmt("%0.6f Threads: current=%d\n", network_time, thread_mgr->NumThreads()));
+
+	const threading::Manager::msg_stats_list& thread_stats = thread_mgr->GetMsgThreadStats();
+	for ( threading::Manager::msg_stats_list::const_iterator i = thread_stats.begin();
+	      i != thread_stats.end(); ++i ) 
+		{
+		threading::MsgThread::Stats s = i->second;
+		file->Write(fmt("%0.6f   %-25s in=%" PRIu64 " out=%" PRIu64 " pending=%" PRIu64 "/%" PRIu64
+				" (#queue r/w: in=%" PRIu64 "/%" PRIu64 " out=%" PRIu64 "/%" PRIu64 ")"
+			        "\n",
+			    network_time,
+			    i->first.c_str(),
+			    s.sent_in, s.sent_out,
+			    s.pending_in, s.pending_out,
+			    s.queue_in_stats.num_reads, s.queue_in_stats.num_writes,
+			    s.queue_out_stats.num_reads, s.queue_out_stats.num_writes
+			    ));
+		}
+
 	// Script-level state.
 	unsigned int size, mem = 0;
 	PDict(ID)* globals = global_scope()->Vars();

src/Val.cc

@@ -606,7 +606,7 @@ ID* MutableVal::Bind() const
 			ip = htonl(0x7f000001);	// 127.0.0.1
 
 		safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#",
-			      IPAddr(IPAddr::IPv4, &ip, IPAddr::Network)->AsString().c_str(),
+			      IPAddr(IPv4, &ip, IPAddr::Network)->AsString().c_str(),
 			      getpid());
 #else
 		safe_snprintf(name, MAX_NAME_SIZE, "#%s#%d#", host, getpid());
@@ -864,12 +864,12 @@ AddrVal::AddrVal(const char* text) : Val(TYPE_ADDR)
 AddrVal::AddrVal(uint32 addr) : Val(TYPE_ADDR)
 	{
 	// ### perhaps do gethostbyaddr here?
-	val.addr_val = new IPAddr(IPAddr::IPv4, &addr, IPAddr::Network);
+	val.addr_val = new IPAddr(IPv4, &addr, IPAddr::Network);
 	}
 
 AddrVal::AddrVal(const uint32 addr[4]) : Val(TYPE_ADDR)
 	{
-	val.addr_val = new IPAddr(IPAddr::IPv6, addr, IPAddr::Network);
+	val.addr_val = new IPAddr(IPv6, addr, IPAddr::Network);
 	}
 
 AddrVal::AddrVal(const IPAddr& addr) : Val(TYPE_ADDR)
@@ -889,7 +889,7 @@ unsigned int AddrVal::MemoryAllocation() const
 
 Val* AddrVal::SizeVal() const
 	{
-	if ( val.addr_val->GetFamily() == IPAddr::IPv4 )
+	if ( val.addr_val->GetFamily() == IPv4 )
 		return new Val(32, TYPE_COUNT);
 	else
 		return new Val(128, TYPE_COUNT);
@@ -933,13 +933,13 @@ SubNetVal::SubNetVal(const char* text, int width) : Val(TYPE_SUBNET)
 
 SubNetVal::SubNetVal(uint32 addr, int width) : Val(TYPE_SUBNET)
 	{
-	IPAddr a(IPAddr::IPv4, &addr, IPAddr::Network);
+	IPAddr a(IPv4, &addr, IPAddr::Network);
 	val.subnet_val = new IPPrefix(a, width);
 	}
 
 SubNetVal::SubNetVal(const uint32* addr, int width) : Val(TYPE_SUBNET)
 	{
-	IPAddr a(IPAddr::IPv6, addr, IPAddr::Network);
+	IPAddr a(IPv6, addr, IPAddr::Network);
 	val.subnet_val = new IPPrefix(a, width);
 	}
 
@@ -953,6 +953,16 @@ SubNetVal::~SubNetVal()
 	delete val.subnet_val;
 	}
 
+const IPAddr& SubNetVal::Prefix() const
+	{
+	return val.subnet_val->Prefix();
+	}
+
+int SubNetVal::Width() const
+	{
+	return val.subnet_val->Length();
+	}
+
 unsigned int SubNetVal::MemoryAllocation() const
 	{
 	return padded_sizeof(*this) + val.subnet_val->MemoryAllocation();
@@ -978,7 +988,7 @@ IPAddr SubNetVal::Mask() const
 		uint32 m[4];
 		for ( unsigned int i = 0; i < 4; ++i )
 			m[i] = 0;
-		IPAddr rval(IPAddr::IPv6, m, IPAddr::Host);
+		IPAddr rval(IPv6, m, IPAddr::Host);
 		return rval;
 		}
 
@@ -994,7 +1004,7 @@ IPAddr SubNetVal::Mask() const
 	while ( ++mp < m + 4 )
 		   *mp = 0;
 
-	IPAddr rval(IPAddr::IPv6, m, IPAddr::Host);
+	IPAddr rval(IPv6, m, IPAddr::Host);
 	return rval;
 	}
 

src/Val.h

@@ -514,10 +514,6 @@ protected:
 #define ICMP_PORT_MASK	0x30000
 
 
-typedef enum {
-	TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP,
-} TransportProto;
-
 class PortVal : public Val {
 public:
 	// Constructors - both take the port number in host order.
@@ -589,8 +585,8 @@ public:
 
 	Val* SizeVal() const;
 
-	const IPAddr& Prefix() const { return val.subnet_val->Prefix(); }
+	const IPAddr& Prefix() const;
-	int Width() const	{ return val.subnet_val->Length(); }
+	int Width() const;
 	IPAddr Mask() const;
 
 	bool Contains(const IPAddr& addr) const;

src/bro.bif

@@ -1949,7 +1949,7 @@ function is_local_interface%(ip: addr%) : bool
 	if ( ent )
 		{
 		for ( unsigned int len = 0; ent->h_addr_list[len]; ++len )
-			addrs.push_back(IPAddr(IPAddr::IPv4, (uint32*)ent->h_addr_list[len],
+			addrs.push_back(IPAddr(IPv4, (uint32*)ent->h_addr_list[len],
 			                       IPAddr::Network));
 		}
 
@@ -1958,7 +1958,7 @@ function is_local_interface%(ip: addr%) : bool
 	if ( ent )
 		{
 		for ( unsigned int len = 0; ent->h_addr_list[len]; ++len )
-			addrs.push_back(IPAddr(IPAddr::IPv6, (uint32*)ent->h_addr_list[len],
+			addrs.push_back(IPAddr(IPv6, (uint32*)ent->h_addr_list[len],
 			                       IPAddr::Network));
 		}
 
@@ -2024,7 +2024,7 @@ function gethostname%(%) : string
 ## Returns: true if *a* is an IPv4 address, else false.
 function is_v4_addr%(a: addr%): bool
 	%{
-	if ( a->AsAddr().GetFamily() == IPAddr::IPv4 )
+	if ( a->AsAddr().GetFamily() == IPv4 )
 		return new Val(1, TYPE_BOOL);
 	else
 		return new Val(0, TYPE_BOOL);
@@ -2037,7 +2037,7 @@ function is_v4_addr%(a: addr%): bool
 ## Returns: true if *a* is an IPv6 address, else false.
 function is_v6_addr%(a: addr%): bool
 	%{
-	if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
+	if ( a->AsAddr().GetFamily() == IPv6 )
 		return new Val(1, TYPE_BOOL);
 	else
 		return new Val(0, TYPE_BOOL);
@@ -2050,18 +2050,15 @@ function is_v6_addr%(a: addr%): bool
 # ===========================================================================
 
 ## Converts the *data* field of :bro:type:`ip6_routing` records that have
-## *rtype* of 0 into a set of addresses.
+## *rtype* of 0 into a vector of addresses.
 ##
 ## s: The *data* field of an :bro:type:`ip6_routing` record that has
 ##    an *rtype* of 0.
 ##
-## Returns: The set of addresses contained in the routing header data.
+## Returns: The vector of addresses contained in the routing header data.
-function routing0_data_to_addrs%(s: string%): addr_set
+function routing0_data_to_addrs%(s: string%): addr_vec
 	%{
-	BroType* index_type = base_type(TYPE_ADDR);
+	VectorVal* rval = new VectorVal(new VectorType(base_type(TYPE_ADDR)));
-	TypeList* set_index = new TypeList(index_type);
-	set_index->Append(index_type);
-	TableVal* tv = new TableVal(new SetType(set_index, 0));
 
 	int len = s->Len();
 	const u_char* bytes = s->Bytes();
@@ -2073,13 +2070,13 @@ function routing0_data_to_addrs%(s: string%): addr_set
 
 	while ( len > 0 )
 		{
-		IPAddr a(IPAddr::IPv6, (const uint32*) bytes, IPAddr::Network);
+		IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network);
-		tv->Assign(new AddrVal(a), 0);
+		rval->Assign(rval->Size(), new AddrVal(a), 0);
 		bytes += 16;
 		len -= 16;
 		}
 
-	return tv;
+	return rval;
 	%}
 
 ## Converts a :bro:type:`addr` to a :bro:type:`index_vec`.
@@ -3555,7 +3552,7 @@ function lookup_location%(a: addr%) : geo_location
 		}
 
 #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
-	if ( geoip_v6 && a->AsAddr().GetFamily() == IPAddr::IPv6 )
+	if ( geoip_v6 && a->AsAddr().GetFamily() == IPv6 )
 		{
 		geoipv6_t ga;
 		a->AsAddr().CopyIPv6(&ga);
@@ -3567,7 +3564,7 @@ function lookup_location%(a: addr%) : geo_location
 	else
 #endif
 
-	if ( geoip && a->AsAddr().GetFamily() == IPAddr::IPv4 )
+	if ( geoip && a->AsAddr().GetFamily() == IPv4 )
 		{
 		const uint32* bytes;
 		a->AsAddr().GetBytes(&bytes);
@@ -3650,7 +3647,7 @@ function lookup_asn%(a: addr%) : count
 		{
 // IPv6 support showed up in 1.4.5.
 #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
-		if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
+		if ( a->AsAddr().GetFamily() == IPv6 )
 			{
 			geoipv6_t ga;
 			a->AsAddr().CopyIPv6(&ga);
@@ -3659,7 +3656,7 @@ function lookup_asn%(a: addr%) : count
 		else
 #endif
 
-		if ( a->AsAddr().GetFamily() == IPAddr::IPv4 )
+		if ( a->AsAddr().GetFamily() == IPv4 )
 			{
 			const uint32* bytes;
 			a->AsAddr().GetBytes(&bytes);
@@ -5386,7 +5383,7 @@ function preserve_prefix%(a: addr, width: count%): any
 	AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50];
 	if ( ip_anon )
 		{
-		if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
+		if ( a->AsAddr().GetFamily() == IPv6 )
 			builtin_error("preserve_prefix() not supported for IPv6 addresses");
 		else
 			{
@@ -5415,7 +5412,7 @@ function preserve_subnet%(a: subnet%): any
 	AnonymizeIPAddr* ip_anon = ip_anonymizer[PREFIX_PRESERVING_A50];
 	if ( ip_anon )
 		{
-		if ( a->AsSubNet().Prefix().GetFamily() == IPAddr::IPv6 )
+		if ( a->AsSubNet().Prefix().GetFamily() == IPv6 )
 			builtin_error("preserve_subnet() not supported for IPv6 addresses");
 		else
 			{
@@ -5451,7 +5448,7 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
 	if ( anon_class < 0 || anon_class >= NUM_ADDR_ANONYMIZATION_CLASSES )
 		builtin_error("anonymize_addr(): invalid ip addr anonymization class");
 
-	if ( a->AsAddr().GetFamily() == IPAddr::IPv6 )
+	if ( a->AsAddr().GetFamily() == IPv6 )
 		{
 		builtin_error("anonymize_addr() not supported for IPv6 addresses");
 		return 0;

src/event.bif

@@ -478,6 +478,14 @@ event ipv6_ext_headers%(c: connection, p: pkt_hdr%);
 ## .. bro:see:: new_packet tcp_packet ipv6_ext_headers
 event esp_packet%(p: pkt_hdr%);
 
+## Generated for any packets using an IPv6 Routing Type 0 extension header
+## with non-zero segments left.
+##
+## p: Information from the header of the packet that triggered the event.
+##
+## .. bro:see:: new_packet tcp_packet ipv6_ext_headers
+event rh0_segleft%(p: pkt_hdr%);
+
 ## Generated for every packet that has non-empty transport-layer payload. This is a
 ## very low-level and expensive event that should be avoided when at all possible.
 ## It's usually  infeasible to handle when processing even medium volumes of

src/logging.bif

@@ -3,8 +3,9 @@
 module Log;
 
 %%{
-#include "LogMgr.h"
 #include "NetVar.h"
+
+#include "logging/Manager.h"
 %%}
 
 type Filter: record;

src/logging/Manager.h

@@ -0,0 +1,205 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// A class managing log writers and filters.
+
+#ifndef LOGGING_MANAGER_H
+#define LOGGING_MANAGER_H
+
+#include "../Val.h"
+#include "../EventHandler.h"
+#include "../RemoteSerializer.h"
+
+class SerializationFormat;
+class RemoteSerializer;
+class RotationTimer;
+
+namespace logging {
+
+
+class WriterBackend;
+class WriterFrontend;
+class RotationFinishedMessage;
+
+/**
+ * Singleton class for managing log streams.
+ */
+class Manager {
+public:
+	/**
+	 * Constructor.
+	 */
+	Manager();
+
+	/**
+	 * Destructor.
+	 */
+	~Manager();
+
+	/**
+	 * Creates a new log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * @param stream A record of script type \c Log::Stream.
+	 *
+	 * This method corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool CreateStream(EnumVal* id, RecordVal* stream);
+
+	/**
+	 * Enables a log log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * This method corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool EnableStream(EnumVal* id);
+	
+	/**
+	 * Disables a log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool DisableStream(EnumVal* id);
+
+	/**
+	 * Adds a filter to a log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * @param filter A record of script type \c Log::Filter.
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool AddFilter(EnumVal* id, RecordVal* filter);
+
+	/**
+	 * Removes a filter from a log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * @param name The name of the filter to remove.
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool RemoveFilter(EnumVal* id, StringVal* name);
+
+	/**
+	 * Removes a filter from a log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * @param name The name of the filter to remove.
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool RemoveFilter(EnumVal* id, string name);
+
+	/**
+	 * Write a record to a log stream.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * @param colums A record of the type defined for the stream's
+	 * columns.
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool Write(EnumVal* id, RecordVal* columns);
+
+	/**
+	 * Sets log streams buffering state. This adjusts all associated
+	 * writers to the new state.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * @param enabled False to disable buffering (default is enabled).
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool SetBuf(EnumVal* id, bool enabled);
+
+	/**
+	 * Flushes a log stream. This flushed all associated writers.
+	 *
+	 * @param id  The enum value corresponding the log stream.
+	 *
+	 * This methods corresponds directly to the internal BiF defined in
+	 * logging.bif, which just forwards here.
+	 */
+	bool Flush(EnumVal* id);
+
+	/**
+	 * Prepares the log manager to terminate. This will flush all log
+	 * stream.
+	 */
+	void Terminate();
+
+protected:
+	friend class WriterFrontend;
+	friend class RotationFinishedMessage;
+	friend class ::RemoteSerializer;
+	friend class ::RotationTimer;
+
+	// Instantiates a new WriterBackend of the given type (note that
+	// doing so creates a new thread!).
+	WriterBackend* CreateBackend(WriterFrontend* frontend, bro_int_t type);
+
+	//// Function also used by the RemoteSerializer.
+
+	// Takes ownership of fields.
+	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path,
+				int num_fields, const threading::Field* const* fields,
+				bool local, bool remote);
+
+	// Takes ownership of values..
+	bool Write(EnumVal* id, EnumVal* writer, string path,
+		   int num_fields, threading::Value** vals);
+
+	// Announces all instantiated writers to peer.
+	void SendAllWritersTo(RemoteSerializer::PeerID peer);
+
+	// Signals that a file has been rotated.
+	bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
+			      double open, double close, bool terminating);
+
+	// Deletes the values as passed into Write().
+	void DeleteVals(int num_fields, threading::Value** vals);
+
+private:
+	struct Filter;
+	struct Stream;
+	struct WriterInfo;
+
+	bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
+			    TableVal* include, TableVal* exclude, string path, list<int> indices);
+
+	threading::Value** RecordToFilterVals(Stream* stream, Filter* filter,
+				    RecordVal* columns);
+
+	threading::Value* ValToLogVal(Val* val, BroType* ty = 0);
+	Stream* FindStream(EnumVal* id);
+	void RemoveDisabledWriters(Stream* stream);
+	void InstallRotationTimer(WriterInfo* winfo);
+	void Rotate(WriterInfo* info);
+	Filter* FindFilter(EnumVal* id, StringVal* filter);
+	WriterInfo* FindWriter(WriterFrontend* writer);
+
+	vector<Stream *> streams;	// Indexed by stream enum.
+};
+
+}
+
+extern logging::Manager* log_mgr;
+
+#endif

src/logging/WriterBackend.cc

@@ -0,0 +1,281 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "util.h"
+#include "threading/SerialTypes.h"
+
+#include "WriterBackend.h"
+#include "WriterFrontend.h"
+
+// Messages sent from backend to frontend (i.e., "OutputMessages").
+
+using threading::Value;
+using threading::Field;
+
+namespace logging  {
+
+class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
+{
+public:
+        RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
+				double open, double close, bool terminating)
+		: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
+		new_name(new_name), old_name(old_name), open(open),
+		close(close), terminating(terminating)	{ }
+
+	virtual bool Process()
+		{
+		return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating);
+		}
+
+private:
+        string new_name;
+        string old_name;
+        double open;
+        double close;
+        bool terminating;
+};
+
+class FlushWriteBufferMessage : public threading::OutputMessage<WriterFrontend>
+{
+public:
+        FlushWriteBufferMessage(WriterFrontend* writer)
+		: threading::OutputMessage<WriterFrontend>("FlushWriteBuffer", writer)	{}
+
+	virtual bool Process()	{ Object()->FlushWriteBuffer(); return true; }
+};
+
+class DisableMessage : public threading::OutputMessage<WriterFrontend>
+{
+public:
+        DisableMessage(WriterFrontend* writer)
+		: threading::OutputMessage<WriterFrontend>("Disable", writer)	{}
+
+	virtual bool Process()	{ Object()->SetDisable(); return true; }
+};
+
+}
+
+// Backend methods.
+
+using namespace logging;
+
+WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
+	{
+	path = "<path not yet set>";
+	num_fields = 0;
+	fields = 0;
+	buffering = true;
+	frontend = arg_frontend;
+
+	SetName(frontend->Name());
+	}
+
+WriterBackend::~WriterBackend()
+	{
+	if ( fields )
+		{
+		for(int i = 0; i < num_fields; ++i)
+			delete fields[i];
+
+		delete [] fields;
+		}
+	}
+
+void WriterBackend::DeleteVals(int num_writes, Value*** vals)
+	{
+	for ( int j = 0; j < num_writes; ++j )
+		{
+		// Note this code is duplicated in Manager::DeleteVals().
+		for ( int i = 0; i < num_fields; i++ )
+			delete vals[j][i];
+
+		delete [] vals[j];
+		}
+
+	delete [] vals;
+	}
+
+bool WriterBackend::FinishedRotation(string new_name, string old_name,
+				     double open, double close, bool terminating)
+	{
+	SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating));
+	return true;
+	}
+
+void WriterBackend::DisableFrontend()
+	{
+	SendOut(new DisableMessage(frontend));
+	}
+
+bool WriterBackend::Init(string arg_path, int arg_num_fields, const Field* const*  arg_fields)
+	{
+	path = arg_path;
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	string name = Fmt("%s/%s", path.c_str(), frontend->Name().c_str());
+
+	SetName(name);
+
+	if ( ! DoInit(arg_path, arg_num_fields, arg_fields) )
+		{
+		DisableFrontend();
+		return false;
+		}
+
+	return true;
+	}
+
+bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
+	{
+	// Double-check that the arguments match. If we get this from remote,
+	// something might be mixed up.
+	if ( num_fields != arg_num_fields )
+		{
+
+#ifdef DEBUG
+		const char* msg = Fmt("Number of fields don't match in WriterBackend::Write() (%d vs. %d)",
+				      arg_num_fields, num_fields);
+		Debug(DBG_LOGGING, msg);
+#endif
+
+		DeleteVals(num_writes, vals);
+		DisableFrontend();
+		return false;
+		}
+
+#ifdef DEBUG
+	// Double-check all the types match.
+	for ( int j = 0; j < num_writes; j++ )
+		{
+		for ( int i = 0; i < num_fields; ++i )
+			{
+			if ( vals[j][i]->type != fields[i]->type )
+				{
+				const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)",
+						      vals[j][i]->type, fields[i]->type);
+				Debug(DBG_LOGGING, msg);
+
+				DisableFrontend();
+				DeleteVals(num_writes, vals);
+				return false;
+				}
+			}
+		}
+#endif
+
+	bool success = true;
+
+	for ( int j = 0; j < num_writes; j++ )
+		{
+		success = DoWrite(num_fields, fields, vals[j]);
+
+		if ( ! success )
+			break;
+		}
+
+	DeleteVals(num_writes, vals);
+
+	if ( ! success )
+		DisableFrontend();
+
+	return success;
+	}
+
+bool WriterBackend::SetBuf(bool enabled)
+	{
+	if ( enabled == buffering )
+		// No change.
+		return true;
+
+	buffering = enabled;
+
+	if ( ! DoSetBuf(enabled) )
+		{
+		DisableFrontend();
+		return false;
+		}
+
+	return true;
+	}
+
+bool WriterBackend::Rotate(string rotated_path, double open,
+			   double close, bool terminating)
+	{
+	if ( ! DoRotate(rotated_path, open, close, terminating) )
+		{
+		DisableFrontend();
+		return false;
+		}
+
+	return true;
+	}
+
+bool WriterBackend::Flush()
+	{
+	if ( ! DoFlush() )
+		{
+		DisableFrontend();
+		return false;
+		}
+
+	return true;
+	}
+
+bool WriterBackend::Finish()
+	{
+	if ( ! DoFlush() )
+		{
+		DisableFrontend();
+		return false;
+		}
+
+	return true;
+	}
+
+bool WriterBackend::DoHeartbeat(double network_time, double current_time)
+	{
+	MsgThread::DoHeartbeat(network_time, current_time);
+
+	SendOut(new FlushWriteBufferMessage(frontend));
+
+	return true;
+	}
+
+string WriterBackend::Render(const threading::Value::addr_t& addr) const
+	{
+	if ( addr.family == IPv4 )
+		{
+		char s[INET_ADDRSTRLEN];
+
+		if ( inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) == NULL )
+			return "<bad IPv4 address conversion>";
+		else
+			return s;
+		}
+	else
+		{
+		char s[INET6_ADDRSTRLEN];
+
+		if ( inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) == NULL )
+			return "<bad IPv6 address conversion>";
+		else
+			return s;
+		}
+	}
+
+string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
+	{
+	char l[16];
+
+	if ( subnet.prefix.family == IPv4 )
+		modp_uitoa10(subnet.length - 96, l);
+	else
+		modp_uitoa10(subnet.length, l);
+
+	string s = Render(subnet.prefix) + "/" + l;
+
+	return s;
+	}
+
+

src/logging/WriterBackend.h

@@ -0,0 +1,319 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Bridge class between main process and writer threads.
+
+#ifndef LOGGING_WRITERBACKEND_H
+#define LOGGING_WRITERBACKEND_H
+
+#include "Manager.h"
+
+#include "threading/MsgThread.h"
+
+namespace logging  {
+
+/**
+ * Base class for writer implementation. When the logging::Manager creates a
+ * new logging filter, it instantiates a WriterFrontend. That then in turn
+ * creates a WriterBackend of the right type. The frontend then forwards
+ * messages over the backend as its methods are called.
+ *
+ * All of this methods must be called only from the corresponding child
+ * thread (the constructor and destructor are the exceptions.)
+ */
+class WriterBackend : public threading::MsgThread
+{
+public:
+	/**
+	 * Constructor.
+	 *
+	 * @param frontend The frontend writer that created this backend. The
+	 * *only* purpose of this value is to be passed back via messages as
+	 * a argument to callbacks. One must not otherwise access the
+	 * frontend, it's running in a different thread.
+	 *
+	 * @param name A descriptive name for writer's type (e.g., \c Ascii).
+	 *
+	 */
+	WriterBackend(WriterFrontend* frontend);
+
+	/**
+	 * Destructor.
+	 */
+	virtual ~WriterBackend();
+
+	/**
+	 * One-time initialization of the writer to define the logged fields.
+	 *
+	 * @param path A string left to the interpretation of the writer
+	 * implementation; it corresponds to the value configured on the
+	 * script-level for the logging filter.
+	 *
+	 * @param num_fields The number of log fields for the stream.
+	 *
+	 * @param fields An array of size \a num_fields with the log fields.
+	 * The methods takes ownership of the array.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Init(string path, int num_fields, const threading::Field* const*  fields);
+
+	/**
+	 * Writes one log entry.
+	 *
+	 * @param num_fields: The number of log fields for this stream. The
+	 * value must match what was passed to Init().
+	 *
+	 * @param An array of size \a num_fields with the log values. Their
+	 * types musst match with the field passed to Init(). The method
+	 * takes ownership of \a vals..
+	 *
+	 * Returns false if an error occured, in which case the writer must
+	 * not be used any further.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Write(int num_fields, int num_writes, threading::Value*** vals);
+
+	/**
+	 * Sets the buffering status for the writer, assuming the writer
+	 * supports that. (If not, it will be ignored).
+	 *
+	 * @param enabled False if buffering is to be disabled (by default
+	 * it's on).
+	 *
+	 * @return False if an error occured.
+	 */
+	bool SetBuf(bool enabled);
+
+	/**
+	 * Flushes any currently buffered output, assuming the writer
+	 * supports that. (If not, it will be ignored).
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Flush();
+
+	/**
+	 * Triggers rotation, if the writer supports that. (If not, it will
+	 * be ignored).
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Rotate(string rotated_path, double open, double close, bool terminating);
+
+	/**
+	 * Finishes writing to this logger in a regularl fashion. Must not be
+	 * called if an error has been indicated earlier. After calling this,
+	 * no further writing must be performed.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Finish();
+
+	/**
+	 * Disables the frontend that has instantiated this backend. Once
+	 * disabled,the frontend will not send any further message over.
+	 */
+	void DisableFrontend();
+
+	/**
+	 * Returns the log path as passed into the constructor.
+	 */
+	const string Path() const	{ return path; }
+
+	/**
+	 * Returns the number of log fields as passed into the constructor.
+	 */
+	int NumFields() const	{ return num_fields; }
+
+	/**
+	 * Returns the log fields as passed into the constructor.
+	 */
+	const threading::Field* const * Fields() const	{ return fields; }
+
+	/**
+	 * Returns the current buffering state.
+	 *
+	 * @return True if buffering is enabled.
+	 */
+	bool IsBuf()	{ return buffering; }
+
+	/**
+	 * Signals that a file has been rotated. This must be called by a
+	 * writer's implementation of DoRotate() once rotation has finished.
+	 *
+	 * Most of the parameters should be passed through from DoRotate().
+	 *
+	 * @param new_name The filename of the rotated file.
+	 *
+	 * @param old_name The filename of the original file.
+	 *
+	 * @param open: The timestamp when the original file was opened.
+	 *
+	 * @param close: The timestamp when the origina file was closed.
+	 *
+	 * @param terminating: True if the original rotation request occured
+	 * due to the main Bro process shutting down.
+	 */
+	bool FinishedRotation(string new_name, string old_name,
+			      double open, double close, bool terminating);
+
+	/** Helper method to render an IP address as a string.
+	  *
+	  * @param addr The address.
+	  *
+	  * @return An ASCII representation of the address.
+	  */
+	string Render(const threading::Value::addr_t& addr) const;
+
+	/** Helper method to render an subnet value as a string.
+	  *
+	  * @param addr The address.
+	  *
+	  * @return An ASCII representation of the address.
+	  */
+	string Render(const threading::Value::subnet_t& subnet) const;
+
+protected:
+	/**
+	 * Writer-specific intialization method.
+	 *
+	 * A writer implementation must override this method. If it returns
+	 * false, it will be assumed that a fatal error has occured that
+	 * prevents the writer from further operation; it will then be
+	 * disabled and eventually deleted. When returning false, an
+	 * implementation should also call Error() to indicate what happened.
+	 */
+	virtual bool DoInit(string path, int num_fields,
+			    const threading::Field* const*  fields) = 0;
+
+	/**
+	 * Writer-specific output method implementing recording of fone log
+	 * entry.
+	 *
+	 * A writer implementation must override this method. If it returns
+	 * false, it will be assumed that a fatal error has occured that
+	 * prevents the writer from further operation; it will then be
+	 * disabled and eventually deleted. When returning false, an
+	 * implementation should also call Error() to indicate what happened.
+	 */
+	virtual bool DoWrite(int num_fields, const threading::Field* const*  fields,
+			     threading::Value** vals) = 0;
+
+	/**
+	 * Writer-specific method implementing a change of fthe buffering
+	 * state.  If buffering is disabled, the writer should attempt to
+	 * write out information as quickly as possible even if doing so may
+	 * have a performance impact. If enabled (which is the default), it
+	 * may buffer data as helpful and write it out later in a way
+	 * optimized for performance. The current buffering state can be
+	 * queried via IsBuf().
+	 *
+	 * A writer implementation must override this method but it can just
+	 * ignore calls if buffering doesn't align with its semantics.
+	 *
+	 * If the method returns false, it will be assumed that a fatal error
+	 * has occured that prevents the writer from further operation; it
+	 * will then be disabled and eventually deleted. When returning
+	 * false, an implementation should also call Error() to indicate what
+	 * happened.
+	 */
+	virtual bool DoSetBuf(bool enabled) = 0;
+
+	/**
+	 * Writer-specific method implementing flushing of its output.
+	 *
+	 * A writer implementation must override this method but it can just
+	 * ignore calls if flushing doesn't align with its semantics.
+	 *
+	 * If the method returns false, it will be assumed that a fatal error
+	 * has occured that prevents the writer from further operation; it
+	 * will then be disabled and eventually deleted. When returning
+	 * false, an implementation should also call Error() to indicate what
+	 * happened.
+	 */
+	virtual bool DoFlush() = 0;
+
+	/**
+	 * Writer-specific method implementing log rotation.  Most directly
+	 * this only applies to writers writing into files, which should then
+	 * close the current file and open a new one.  However, a writer may
+	 * also trigger other apppropiate actions if semantics are similar. *
+	 * Once rotation has finished, the implementation must call
+	 * FinishedRotation() to signal the log manager that potential
+	 * postprocessors can now run.
+	 *
+	 * A writer implementation must override this method but it can just
+	 * ignore calls if flushing doesn't align with its semantics. It
+	 * still needs to call FinishedRotation() though.
+	 *
+	 * If the method returns false, it will be assumed that a fatal error
+	 * has occured that prevents the writer from further operation; it
+	 * will then be disabled and eventually deleted. When returning
+	 * false, an implementation should also call Error() to indicate what
+	 * happened.
+	 *
+	 * @param rotate_path Reflects the path to where the rotated output
+	 * is to be moved, with specifics depending on the writer. It should
+	 * generally be interpreted in a way consistent with that of \c path
+	 * as passed into DoInit(). As an example, for file-based output, \c
+	 * rotate_path could be the original filename extended with a
+	 * timestamp indicating the time of the rotation.
+	 *
+	 * @param open The network time when the *current* file was opened.
+	 *
+	 * @param close The network time when the *current* file was closed.
+	 *
+	 * @param terminating Indicates whether the rotation request occurs
+	 * due the main Bro prcoess terminating (and not because we've
+	 * reached a regularly scheduled time for rotation).
+	 */
+	virtual bool DoRotate(string rotated_path, double open, double close,
+			      bool terminating) = 0;
+
+	/**
+	 * Writer-specific method implementing log output finalization at
+	 * termination. Not called when any of the other methods has
+	 * previously signaled an error, i.e., executing this method signals
+	 * a regular shutdown of the writer.
+	 *
+	 * A writer implementation must override this method but it can just
+	 * ignore calls if flushing doesn't align with its semantics.
+	 *
+	 * If the method returns false, it will be assumed that a fatal error
+	 * has occured that prevents the writer from further operation; it
+	 * will then be disabled and eventually deleted. When returning
+	 * false, an implementation should also call Error() to indicate what
+	 * happened.
+	 */
+	virtual bool DoFinish() = 0;
+
+	/**
+	 * Triggered by regular heartbeat messages from the main thread.
+	 *
+	 * This method can be overridden but once must call
+	 * WriterBackend::DoHeartbeat().
+	 */
+	virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+	/**
+	 * Deletes the values as passed into Write().
+	 */
+	void DeleteVals(int num_writes, threading::Value*** vals);
+
+	// Frontend that instantiated us. This object must not be access from
+	// this class, it's running in a different thread!
+	WriterFrontend* frontend;
+
+	string path;	// Log path.
+	int num_fields;	// Number of log fields.
+	const threading::Field* const*  fields;	// Log fields.
+	bool buffering;	// True if buffering is enabled.
+};
+
+
+}
+
+#endif
+

src/logging/WriterFrontend.cc

@@ -0,0 +1,279 @@
+
+#include "Net.h"
+#include "threading/SerialTypes.h"
+
+#include "WriterFrontend.h"
+#include "WriterBackend.h"
+
+using threading::Value;
+using threading::Field;
+
+namespace logging  {
+
+// Messages sent from frontend to backend (i.e., "InputMessages").
+
+class InitMessage : public threading::InputMessage<WriterBackend>
+{
+public:
+	InitMessage(WriterBackend* backend, const string path, const int num_fields, const Field* const* fields)
+		: threading::InputMessage<WriterBackend>("Init", backend),
+		path(path), num_fields(num_fields), fields(fields) { }
+
+	virtual bool Process() { return Object()->Init(path, num_fields, fields); }
+
+private:
+	const string path;
+	const int num_fields;
+	const Field * const* fields;
+};
+
+class RotateMessage : public threading::InputMessage<WriterBackend>
+{
+public:
+	RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open,
+		      const double close, const bool terminating)
+		: threading::InputMessage<WriterBackend>("Rotate", backend),
+		frontend(frontend),
+		rotated_path(rotated_path), open(open),
+		close(close), terminating(terminating) { }
+
+	virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); }
+
+private:
+	WriterFrontend* frontend;
+	const string rotated_path;
+	const double open;
+	const double close;
+	const bool terminating;
+};
+
+class WriteMessage : public threading::InputMessage<WriterBackend>
+{
+public:
+	WriteMessage(WriterBackend* backend, int num_fields, int num_writes, Value*** vals)
+		: threading::InputMessage<WriterBackend>("Write", backend),
+		num_fields(num_fields), num_writes(num_writes), vals(vals)	{}
+
+	virtual bool Process() { return Object()->Write(num_fields, num_writes, vals); }
+
+private:
+	int num_fields;
+	int num_writes;
+	Value ***vals;
+};
+
+class SetBufMessage : public threading::InputMessage<WriterBackend>
+{
+public:
+	SetBufMessage(WriterBackend* backend, const bool enabled)
+		: threading::InputMessage<WriterBackend>("SetBuf", backend),
+		enabled(enabled) { }
+
+	virtual bool Process() { return Object()->SetBuf(enabled); }
+
+private:
+	const bool enabled;
+};
+
+class FlushMessage : public threading::InputMessage<WriterBackend>
+{
+public:
+	FlushMessage(WriterBackend* backend)
+		: threading::InputMessage<WriterBackend>("Flush", backend)	{}
+
+	virtual bool Process() { return Object()->Flush(); }
+};
+
+class FinishMessage : public threading::InputMessage<WriterBackend>
+{
+public:
+	FinishMessage(WriterBackend* backend)
+		: threading::InputMessage<WriterBackend>("Finish", backend)	{}
+
+	virtual bool Process() { return Object()->Finish(); }
+};
+
+}
+
+// Frontend methods.
+
+using namespace logging;
+
+WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
+	{
+	stream = arg_stream;
+	writer = arg_writer;
+	Ref(stream);
+	Ref(writer);
+
+	disabled = initialized = false;
+	buf = true;
+	local = arg_local;
+	remote = arg_remote;
+	write_buffer = 0;
+	write_buffer_pos = 0;
+	ty_name = "<not set>";
+
+	if ( local )
+		{
+		backend = log_mgr->CreateBackend(this, writer->AsEnum());
+		assert(backend);
+		backend->Start();
+		}
+
+	else
+		backend = 0;
+	}
+
+WriterFrontend::~WriterFrontend()
+	{
+	Unref(stream);
+	Unref(writer);
+	}
+
+string WriterFrontend::Name() const
+	{
+	if ( path.size() )
+		return ty_name;
+
+	return ty_name + "/" + path;
+	}
+
+void WriterFrontend::Stop()
+	{
+	FlushWriteBuffer();
+	SetDisable();
+
+	if ( backend )
+		backend->Stop();
+	}
+
+void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields)
+	{
+	if ( disabled )
+		return;
+
+	if ( initialized )
+		reporter->InternalError("writer initialize twice");
+
+	path = arg_path;
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	initialized = true;
+
+	if ( backend )
+		backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields));
+
+	if ( remote )
+		remote_serializer->SendLogCreateWriter(stream,
+						       writer,
+						       arg_path,
+						       arg_num_fields,
+						       arg_fields);
+
+	}
+
+void WriterFrontend::Write(int num_fields, Value** vals)
+	{
+	if ( disabled )
+		return;
+
+	if ( remote )
+		remote_serializer->SendLogWrite(stream,
+						writer,
+						path,
+						num_fields,
+						vals);
+
+	if ( ! backend )
+		{
+		DeleteVals(vals);
+		return;
+		}
+
+	if ( ! write_buffer )
+		{
+		// Need new buffer.
+		write_buffer = new Value**[WRITER_BUFFER_SIZE];
+		write_buffer_pos = 0;
+		}
+
+	write_buffer[write_buffer_pos++] = vals;
+
+	if ( write_buffer_pos >= WRITER_BUFFER_SIZE || ! buf || terminating )
+		// Buffer full (or no bufferin desired or termiating).
+		FlushWriteBuffer();
+
+	}
+
+void WriterFrontend::FlushWriteBuffer()
+	{
+	if ( ! write_buffer_pos )
+		// Nothing to do.
+		return;
+
+	if ( backend )
+		backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer));
+
+	// Clear buffer (no delete, we pass ownership to child thread.)
+	write_buffer = 0;
+	write_buffer_pos = 0;
+	}
+
+void WriterFrontend::SetBuf(bool enabled)
+	{
+	if ( disabled )
+		return;
+
+	buf = enabled;
+
+	if ( backend )
+		backend->SendIn(new SetBufMessage(backend, enabled));
+
+	if ( ! buf )
+		// Make sure no longer buffer any still queued data.
+		FlushWriteBuffer();
+	}
+
+void WriterFrontend::Flush()
+	{
+	if ( disabled )
+		return;
+
+	FlushWriteBuffer();
+
+	if ( backend )
+		backend->SendIn(new FlushMessage(backend));
+	}
+
+void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
+	{
+	if ( disabled )
+		return;
+
+	FlushWriteBuffer();
+
+	if ( backend )
+		backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
+	}
+
+void WriterFrontend::Finish()
+	{
+	if ( disabled )
+		return;
+
+	FlushWriteBuffer();
+
+	if ( backend )
+		backend->SendIn(new FinishMessage(backend));
+	}
+
+void WriterFrontend::DeleteVals(Value** vals)
+	{
+	// Note this code is duplicated in Manager::DeleteVals().
+	for ( int i = 0; i < num_fields; i++ )
+		delete vals[i];
+
+	delete [] vals;
+	}

src/logging/WriterFrontend.h

@@ -0,0 +1,222 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef LOGGING_WRITERFRONTEND_H
+#define LOGGING_WRITERFRONTEND_H
+
+#include "Manager.h"
+
+#include "threading/MsgThread.h"
+
+namespace logging  {
+
+class WriterBackend;
+
+/**
+ * Bridge class between the logging::Manager and backend writer threads. The
+ * Manager instantiates one \a WriterFrontend for each open logging filter.
+ * Each frontend in turns instantiates a WriterBackend-derived class
+ * internally that's specific to the particular output format. That backend
+ * runs in a new thread, and it receives messages from the frontend that
+ * correspond to method called by the manager.
+ *
+ */
+class WriterFrontend {
+public:
+	/**
+	 * Constructor.
+	 *
+	 * stream: The logging stream.
+	 *
+	 * writer: The backend writer type, with the value corresponding to the
+	 * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The
+	 * frontend will internally instantiate a WriterBackend of the
+	 * corresponding type.
+	 * 
+	 * local: If true, the writer will instantiate a local backend.
+	 *
+	 * remote: If true, the writer will forward all data to remote
+	 * clients.
+	 *
+	 * Frontends must only be instantiated by the main thread.
+	 */
+	WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote);
+
+	/**
+	 * Destructor.
+	 *
+	 * Frontends must only be destroyed by the main thread.
+	 */
+	virtual ~WriterFrontend();
+
+	/**
+	 * Stops all output to this writer. Calling this methods disables all
+	 * message forwarding to the backend and stops the backend thread.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void Stop();
+
+	/**
+	 * Initializes the writer.
+	 *
+	 * This method generates a message to the backend writer and triggers
+	 * the corresponding message there. If the backend method fails, it
+	 * sends a message back that will asynchronously call Disable().
+	 *
+	 * See WriterBackend::Init() for arguments. The method takes
+	 * ownership of \a fields.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void Init(string path, int num_fields, const threading::Field* const*  fields);
+
+	/**
+	 * Write out a record.
+	 *
+	 * This method generates a message to the backend writer and triggers
+	 * the corresponding message there. If the backend method fails, it
+	 * sends a message back that will asynchronously call Disable().
+	 *
+	 * As an optimization, if buffering is enabled (which is the default)
+	 * this method may buffer several writes and send them over to the
+	 * backend in bulk with a single message. An explicit bulk write of
+	 * all currently buffered data can be triggered with
+	 * FlushWriteBuffer(). The backend writer triggers this with a
+	 * message at every heartbeat.
+	 *
+	 * See WriterBackend::Writer() for arguments (except that this method
+	 * takes only a single record, not an array). The method takes
+	 * ownership of \a vals.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void Write(int num_fields, threading::Value** vals);
+
+	/**
+	 * Sets the buffering state.
+	 *
+	 * This method generates a message to the backend writer and triggers
+	 * the corresponding message there. If the backend method fails, it
+	 * sends a message back that will asynchronously call Disable().
+	 *
+	 * See WriterBackend::SetBuf() for arguments.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void SetBuf(bool enabled);
+
+	/**
+	 * Flushes the output.
+	 *
+	 * This method generates a message to the backend writer and triggers
+	 * the corresponding message there. In addition, it also triggers
+	 * FlushWriteBuffer(). If the backend method fails, it sends a
+	 * message back that will asynchronously call Disable().
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void Flush();
+
+	/**
+	 * Triggers log rotation.
+	 *
+	 * This method generates a message to the backend writer and triggers
+	 * the corresponding message there. If the backend method fails, it
+	 * sends a message back that will asynchronously call Disable().
+	 *
+	 * See WriterBackend::Rotate() for arguments.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void Rotate(string rotated_path, double open, double close, bool terminating);
+
+	/**
+	 * Finalizes writing to this tream.
+	 *
+	 * This method generates a message to the backend writer and triggers
+	 * the corresponding message there. If the backend method fails, it
+	 * sends a message back that will asynchronously call Disable().
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void Finish();
+
+	/**
+	 * Explicitly triggers a transfer of all potentially buffered Write()
+	 * operations over to the backend.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void FlushWriteBuffer();
+
+	/**
+	 * Disables the writer frontend. From now on, all method calls that
+	 * would normally send message over to the backend, turn into no-ops.
+	 * Note though that it does not stop the backend itself, use Stop()
+	 * to do thast as well (this method is primarily for use as callback
+	 * when the backend wants to disable the frontend).
+	 *
+	 * Disabled frontend will eventually be discarded by the
+	 * logging::Manager.
+	 *
+	 * This method must only be called from the main thread.
+	 */
+	void SetDisable()	{ disabled = true; }
+
+	/**
+	 * Returns true if the writer frontend has been disabled with SetDisable().
+	 */
+	bool Disabled()	{ return disabled; }
+
+	/**
+	 * Returns the log path as passed into the constructor.
+	 */
+	const string Path() const	{ return path; }
+
+	/**
+	 * Returns the number of log fields as passed into the constructor.
+	 */
+	int NumFields() const	{ return num_fields; }
+
+	/**
+	 * Returns a descriptive name for the writer, including the type of
+	 * the backend and the path used.
+	 *
+	 * This method is safe to call from any thread.
+	 */
+	string Name() const;
+
+	/**
+	 * Returns the log fields as passed into the constructor.
+	 */
+	const threading::Field* const * Fields() const	{ return fields; }
+
+protected:
+	friend class Manager;
+
+	void DeleteVals(threading::Value** vals);
+
+	EnumVal* stream;
+	EnumVal* writer;
+
+	WriterBackend* backend;	// The backend we have instanatiated.
+	bool disabled;	// True if disabled.
+	bool initialized;	// True if initialized.
+	bool buf;	// True if buffering is enabled (default).
+	bool local;	// True if logging locally.
+	bool remote;	// True if loggin remotely.
+
+	string ty_name;	// Name of the backend type. Set by the manager.
+	string path;	// The log path.
+	int num_fields;	// The number of log fields.
+	const threading::Field* const*  fields;	// The log fields.
+
+	// Buffer for bulk writes.
+	static const int WRITER_BUFFER_SIZE = 1000;
+	int write_buffer_pos;	// Position of next write in buffer.
+	threading::Value*** write_buffer;	// Buffer of size WRITER_BUFFER_SIZE.
+};
+
+}
+
+#endif

src/logging/writers/Ascii.cc

@@ -3,10 +3,17 @@
 #include <string>
 #include <errno.h>
 
-#include "LogWriterAscii.h"
 #include "NetVar.h"
+#include "threading/SerialTypes.h"
 
-LogWriterAscii::LogWriterAscii()
+#include "Ascii.h"
+
+using namespace logging;
+using namespace writer;
+using threading::Value;
+using threading::Field;
+
+Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	{
 	file = 0;
 
@@ -42,7 +49,7 @@ LogWriterAscii::LogWriterAscii()
 	desc.AddEscapeSequence(separator, separator_len);
 	}
 
-LogWriterAscii::~LogWriterAscii()
+Ascii::~Ascii()
 	{
 	if ( file )
 		fclose(file);
@@ -54,7 +61,7 @@ LogWriterAscii::~LogWriterAscii()
 	delete [] header_prefix;
 	}
 
-bool LogWriterAscii::WriteHeaderField(const string& key, const string& val)
+bool Ascii::WriteHeaderField(const string& key, const string& val)
 	{
 	string str = string(header_prefix, header_prefix_len) +
 		key + string(separator, separator_len) + val + "\n";
@@ -62,8 +69,8 @@ bool LogWriterAscii::WriteHeaderField(const string& key, const string& val)
 	return (fwrite(str.c_str(), str.length(), 1, file) == 1);
 	}
 
-bool LogWriterAscii::DoInit(string path, int num_fields,
+bool Ascii::DoInit(string path, int num_fields,
-			    const LogField* const * fields)
+			    const Field* const * fields)
 	{
 	if ( output_to_stdout )
 		path = "/dev/stdout";
@@ -108,7 +115,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields,
 				types += string(separator, separator_len);
 				}
 
-			const LogField* field = fields[i];
+			const Field* field = fields[i];
 			names += field->name;
 			types += type_name(field->type);
 			if ( (field->type == TYPE_TABLE) || (field->type == TYPE_VECTOR) )
@@ -131,17 +138,18 @@ write_error:
 	return false;
 	}
 
-bool LogWriterAscii::DoFlush()
+bool Ascii::DoFlush()
 	{
 	fflush(file);
 	return true;
 	}
 
-void LogWriterAscii::DoFinish()
+bool Ascii::DoFinish()
 	{
+	return true;
 	}
 
-bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
+bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 	{
 	if ( ! val->present )
 		{
@@ -161,16 +169,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
 
 	case TYPE_COUNT:
 	case TYPE_COUNTER:
-	case TYPE_PORT:
 		desc->Add(val->val.uint_val);
 		break;
 
+	case TYPE_PORT:
+		desc->Add(val->val.port_val.port);
+		break;
+
 	case TYPE_SUBNET:
-		desc->Add(*val->val.subnet_val);
+		desc->Add(Render(val->val.subnet_val));
 		break;
 
 	case TYPE_ADDR:
-		desc->Add(*val->val.addr_val);
+		desc->Add(Render(val->val.addr_val));
 		break;
 
 	case TYPE_TIME:
@@ -279,8 +290,8 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
 	return true;
 	}
 
-bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
+bool Ascii::DoWrite(int num_fields, const Field* const * fields,
-			     LogVal** vals)
+			     Value** vals)
 	{
 	if ( ! file )
 		DoInit(Path(), NumFields(), Fields());
@@ -310,8 +321,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
 	return true;
 	}
 
-bool LogWriterAscii::DoRotate(string rotated_path, double open,
+bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
-			      double close, bool terminating)
 	{
 	// Don't rotate special files or if there's not one currently open.
 	if ( ! file || IsSpecial(Path()) )
@@ -332,13 +342,13 @@ bool LogWriterAscii::DoRotate(string rotated_path, double open,
 	return true;
 	}
 
-bool LogWriterAscii::DoSetBuf(bool enabled)
+bool Ascii::DoSetBuf(bool enabled)
 	{
 	// Nothing to do.
 	return true;
 	}
 
-string LogWriterAscii::LogExt()
+string Ascii::LogExt()
 	{
 	const char* ext = getenv("BRO_LOG_SUFFIX");
 	if ( ! ext ) ext = "log";

src/logging/writers/Ascii.h

@@ -2,33 +2,36 @@
 //
 // Log writer for delimiter-separated ASCII logs.
 
-#ifndef LOGWRITERASCII_H
+#ifndef LOGGING_WRITER_ASCII_H
-#define LOGWRITERASCII_H
+#define LOGGING_WRITER_ASCII_H
 
-#include "LogWriter.h"
+#include "../WriterBackend.h"
 
-class LogWriterAscii : public LogWriter {
+namespace logging { namespace writer {
+
+class Ascii : public WriterBackend {
 public:
-	LogWriterAscii();
+	Ascii(WriterFrontend* frontend);
-	~LogWriterAscii();
+	~Ascii();
 
-	static LogWriter* Instantiate()	{ return new LogWriterAscii; }
+	static WriterBackend* Instantiate(WriterFrontend* frontend)
+		{ return new Ascii(frontend); }
 	static string LogExt();
 
 protected:
 	virtual bool DoInit(string path, int num_fields,
-			    const LogField* const * fields);
+			    const threading::Field* const* fields);
-	virtual bool DoWrite(int num_fields, const LogField* const * fields,
+	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
-			     LogVal** vals);
+			     threading::Value** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, double open, double close,
+	virtual bool DoRotate(string rotated_path, double open,
-			      bool terminating);
+			      double close, bool terminating);
 	virtual bool DoFlush();
-	virtual void DoFinish();
+	virtual bool DoFinish();
 
 private:
 	bool IsSpecial(string path) 	{ return path.find("/dev/") == 0; }
-	bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field);
+	bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
 	bool WriteHeaderField(const string& key, const string& value);
 
 	FILE* file;
@@ -55,4 +58,8 @@ private:
 	int header_prefix_len;
 };
 
+}
+}
+
+
 #endif

src/logging/writers/None.cc

@@ -1,8 +1,10 @@
 
-#include "LogWriterNone.h"
+#include "None.h"
 
-bool LogWriterNone::DoRotate(string rotated_path, double open,
+using namespace logging;
-			      double close, bool terminating)
+using namespace writer;
+
+bool None::DoRotate(string rotated_path, double open, double close, bool terminating)
 	{
 	if ( ! FinishedRotation(string("/dev/null"), Path(), open, close, terminating))
 		{

src/logging/writers/None.h

@@ -0,0 +1,36 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Dummy log writer that just discards everything (but still pretends to rotate).
+
+#ifndef LOGGING_WRITER_NONE_H
+#define LOGGING_WRITER_NONE_H
+
+#include "../WriterBackend.h"
+
+namespace logging { namespace writer {
+
+class None : public WriterBackend {
+public:
+	None(WriterFrontend* frontend) : WriterBackend(frontend)	{}
+	~None()	{};
+
+	static WriterBackend* Instantiate(WriterFrontend* frontend)
+		{ return new None(frontend); }
+
+protected:
+	virtual bool DoInit(string path, int num_fields,
+			    const threading::Field* const * fields)	{ return true; }
+
+	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
+			     threading::Value** vals)	{ return true; }
+	virtual bool DoSetBuf(bool enabled)	{ return true; }
+	virtual bool DoRotate(string rotated_path, double open,
+			      double close, bool terminating);
+	virtual bool DoFlush()	{ return true; }
+	virtual bool DoFinish()	{ return true; }
+};
+
+}
+}
+
+#endif

src/main.cc

@@ -29,7 +29,6 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "Event.h"
 #include "File.h"
 #include "Reporter.h"
-#include "LogMgr.h"
 #include "Net.h"
 #include "NetVar.h"
 #include "Var.h"
@@ -47,7 +46,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "DPM.h"
 #include "BroDoc.h"
 #include "Brofiler.h"
-#include "LogWriterAscii.h"
+
+#include "threading/Manager.h"
+#include "logging/Manager.h"
+#include "logging/writers/Ascii.h"
 
 #include "binpac_bro.h"
 
@@ -63,7 +65,7 @@ extern "C" {
 #include "setsignal.h"
 };
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 HeapLeakChecker* heap_checker = 0;
 int perftools_leaks = 0;
 int perftools_profile = 0;
@@ -74,7 +76,8 @@ char* writefile = 0;
 name_list prefixes;
 DNS_Mgr* dns_mgr;
 TimerMgr* timer_mgr;
-LogMgr* log_mgr;
+logging::Manager* log_mgr = 0;
+threading::Manager* thread_mgr = 0;
 Stmt* stmts;
 EventHandlerPtr net_done = 0;
 RuleMatcher* rule_matcher = 0;
@@ -174,7 +177,7 @@ void usage()
 	fprintf(stderr, "    -W|--watchdog                  | activate watchdog timer\n");
 	fprintf(stderr, "    -Z|--doc-scripts               | generate documentation for all loaded scripts\n");
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	fprintf(stderr, "    -m|--mem-leaks                 | show leaks  [perftools]\n");
 	fprintf(stderr, "    -M|--mem-profile               | record heap [perftools]\n");
 #endif
@@ -195,7 +198,7 @@ void usage()
 	fprintf(stderr, "    $BRO_PREFIXES                  | prefix list (%s)\n", bro_prefixes());
 	fprintf(stderr, "    $BRO_DNS_FAKE                  | disable DNS lookups (%s)\n", bro_dns_fake());
 	fprintf(stderr, "    $BRO_SEED_FILE                 | file to load seeds from (not set)\n");
-	fprintf(stderr, "    $BRO_LOG_SUFFIX                | ASCII log file extension (.%s)\n", LogWriterAscii::LogExt().c_str());
+	fprintf(stderr, "    $BRO_LOG_SUFFIX                | ASCII log file extension (.%s)\n", logging::writer::Ascii::LogExt().c_str());
 	fprintf(stderr, "    $BRO_PROFILER_FILE             | Output file for script execution statistics (not set)\n");
 
 	exit(1);
@@ -241,7 +244,7 @@ void done_with_network()
 
 	net_finish(1);
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 
 		if ( perftools_profile )
 			{
@@ -285,6 +288,9 @@ void terminate_bro()
 	if ( remote_serializer )
 		remote_serializer->LogStats();
 
+	log_mgr->Terminate();
+	thread_mgr->Terminate();
+
 	delete timer_mgr;
 	delete dns_mgr;
 	delete persistence_serializer;
@@ -296,6 +302,7 @@ void terminate_bro()
 	delete remote_serializer;
 	delete dpm;
 	delete log_mgr;
+	delete thread_mgr;
 	delete reporter;
 	}
 
@@ -324,6 +331,13 @@ RETSIGTYPE sig_handler(int signo)
 	{
 	set_processing_status("TERMINATING", "sig_handler");
 	signal_val = signo;
+
+	if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) )
+		// If the thread manager is already terminating (i.e.,
+		// waiting for child threads to exit), another term signal
+		// will send the threads a kill.
+		thread_mgr->KillThreads();
+
 	return RETSIGVAL;
 	}
 
@@ -410,7 +424,7 @@ int main(int argc, char** argv)
 #ifdef	USE_IDMEF
 		{"idmef-dtd",		required_argument,	0,	'n'},
 #endif
-#ifdef	USE_PERFTOOLS
+#ifdef	USE_PERFTOOLS_DEBUG
 		{"mem-leaks",	no_argument,		0,	'm'},
 		{"mem-profile",	no_argument,		0,	'M'},
 #endif
@@ -452,7 +466,7 @@ int main(int argc, char** argv)
 	safe_strncpy(opts, "B:D:e:f:I:i:K:l:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLOPSWbdghvZ",
 		     sizeof(opts));
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	strncat(opts, "mM", 2);
 #endif
 
@@ -608,7 +622,7 @@ int main(int argc, char** argv)
 			exit(0);
 			break;
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 		case 'm':
 			perftools_leaks = 1;
 			break;
@@ -658,7 +672,9 @@ int main(int argc, char** argv)
 	set_processing_status("INITIALIZING", "main");
 
 	bro_start_time = current_time(true);
+
 	reporter = new Reporter();
+	thread_mgr = new threading::Manager();
 
 #ifdef DEBUG
 	if ( debug_streams )
@@ -724,7 +740,7 @@ int main(int argc, char** argv)
 	persistence_serializer = new PersistenceSerializer();
 	remote_serializer = new RemoteSerializer();
 	event_registry = new EventRegistry();
-	log_mgr = new LogMgr();
+	log_mgr = new logging::Manager();
 
 	if ( events_file )
 		event_player = new EventPlayer(events_file);
@@ -742,14 +758,14 @@ int main(int argc, char** argv)
 	// nevertheless reported; see perftools docs), thus
 	// we suppress some messages here.
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	{
 	HeapLeakChecker::Disabler disabler;
 #endif
 
 	yyparse();
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 	}
 #endif
 
@@ -996,12 +1012,14 @@ int main(int argc, char** argv)
 
 	have_pending_timers = ! reading_traces && timer_mgr->Size() > 0;
 
+	io_sources.Register(thread_mgr, true);
+
 	if ( io_sources.Size() > 0 || have_pending_timers )
 		{
 		if ( profiling_logger )
 			profiling_logger->Log();
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 		if ( perftools_leaks )
 			heap_checker = new HeapLeakChecker("net_run");
 

src/net_util.cc

@@ -38,33 +38,6 @@ int ones_complement_checksum(const IPAddr& a, uint32 sum)
 	return ones_complement_checksum(bytes, len*4, sum);
 	}
 
-int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len)
-	{
-	// ### Note, this is only correct for IPv4.  This routine is only
-	// used by the connection compressor (which we turn off for IPv6
-	// traffic).
-
-	int tcp_len = tp->th_off * 4 + len;
-	uint32 sum;
-
-	if ( len % 2 == 1 )
-		// Add in pad byte.
-		sum = htons(((const u_char*) tp)[tcp_len - 1] << 8);
-	else
-		sum = 0;
-
-	sum = ones_complement_checksum((void*) &ip->ip_src.s_addr, 4, sum);
-	sum = ones_complement_checksum((void*) &ip->ip_dst.s_addr, 4, sum);
-
-	uint32 addl_pseudo =
-		(htons(IPPROTO_TCP) << 16) | htons((unsigned short) tcp_len);
-
-	sum = ones_complement_checksum((void*) &addl_pseudo, 4, sum);
-	sum = ones_complement_checksum((void*) tp, tcp_len, sum);
-
-	return sum;
-	}
-
 int udp_checksum(const struct ip* ip, const struct udphdr* up, int len)
 	{
 	uint32 sum;
@@ -191,12 +164,11 @@ const char* fmt_conn_id(const IPAddr& src_addr, uint32 src_port,
 const char* fmt_conn_id(const uint32* src_addr, uint32 src_port,
 			const uint32* dst_addr, uint32 dst_port)
 	{
-	IPAddr src(IPAddr::IPv6, src_addr, IPAddr::Network);
+	IPAddr src(IPv6, src_addr, IPAddr::Network);
-	IPAddr dst(IPAddr::IPv6, dst_addr, IPAddr::Network);
+	IPAddr dst(IPv6, dst_addr, IPAddr::Network);
 	return fmt_conn_id(src, src_port, dst, dst_port);
 	}
 
-
 uint32 extract_uint32(const u_char* data)
 	{
 	uint32 val;

src/net_util.h

@@ -5,6 +5,13 @@
 
 #include "config.h"
 
+// Define first.
+typedef enum {
+	TRANSPORT_UNKNOWN, TRANSPORT_TCP, TRANSPORT_UDP, TRANSPORT_ICMP,
+} TransportProto;
+
+typedef enum { IPv4, IPv6 } IPFamily;
+
 #include <assert.h>
 #include <sys/types.h>
 
@@ -21,7 +28,6 @@
 #include <netinet/ip_icmp.h>
 
 #include "util.h"
-#include "IPAddr.h"
 
 #ifdef HAVE_NETINET_IP6_H
 #include <netinet/ip6.h>
@@ -58,11 +64,12 @@ inline int seq_delta(uint32 a, uint32 b)
 	return int(a-b);
 	}
 
+class IPAddr;
+
 // Returns the ones-complement checksum of a chunk of b short-aligned bytes.
 extern int ones_complement_checksum(const void* p, int b, uint32 sum);
 extern int ones_complement_checksum(const IPAddr& a, uint32 sum);
 
-extern int tcp_checksum(const struct ip* ip, const struct tcphdr* tp, int len);
 extern int udp_checksum(const struct ip* ip, const struct udphdr* up, int len);
 extern int udp6_checksum(const struct ip6_hdr* ip, const struct udphdr* up,
 				int len);

src/threading/BasicThread.cc

@@ -0,0 +1,174 @@
+
+#include <sys/signal.h>
+#include <signal.h>
+
+#include "config.h"
+#include "BasicThread.h"
+#include "Manager.h"
+
+#ifdef HAVE_LINUX
+#include <sys/prctl.h>
+#endif
+
+using namespace threading;
+
+uint64_t BasicThread::thread_counter = 0;
+
+BasicThread::BasicThread()
+	{
+	started = false;
+	terminating = false;
+	pthread = 0;
+
+	buf_len = 2048;
+	buf = (char*) malloc(buf_len);
+
+	name = Fmt("thread-%d", ++thread_counter);
+
+	thread_mgr->AddThread(this);
+	}
+
+BasicThread::~BasicThread()
+	{
+        if ( buf )
+		free(buf);
+	}
+
+void BasicThread::SetName(const string& arg_name)
+	{
+	// Slight race condition here with reader threads, but shouldn't matter.
+	name = arg_name;
+	}
+
+void BasicThread::SetOSName(const string& name)
+	{
+#ifdef HAVE_LINUX
+	prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
+#endif
+
+#ifdef __APPLE__
+	pthread_setname_np(name.c_str());
+#endif
+
+#ifdef FREEBSD
+	pthread_set_name_np(pthread_self(), name, name.c_str());
+#endif
+	}
+
+const char* BasicThread::Fmt(const char* format, ...)
+	{
+	va_list al;
+	va_start(al, format);
+	int n = safe_vsnprintf(buf, buf_len, format, al);
+	va_end(al);
+
+	if ( (unsigned int) n >= buf_len )
+		{ // Not enough room, grow the buffer.
+		int tmp_len = n + 32;
+		char* tmp = (char*) malloc(tmp_len);
+
+		// Is it portable to restart?
+		va_start(al, format);
+		n = safe_vsnprintf(tmp, tmp_len, format, al);
+		va_end(al);
+
+		free(tmp);
+		}
+
+	return buf;
+	}
+
+void BasicThread::Start()
+	{
+	if ( started )
+		return;
+
+	if ( pthread_mutex_init(&terminate, 0) != 0  )
+		reporter->FatalError("Cannot create terminate mutex for thread %s", name.c_str());
+
+	// We use this like a binary semaphore and acquire it immediately.
+	if ( pthread_mutex_lock(&terminate) != 0 )
+		reporter->FatalError("Cannot aquire terminate mutex for thread %s", name.c_str());
+
+	if ( pthread_create(&pthread, 0, BasicThread::launcher, this) != 0 )
+		reporter->FatalError("Cannot create thread %s", name.c_str());
+
+	DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
+
+	started = true;
+
+	OnStart();
+	}
+
+void BasicThread::Stop()
+	{
+	if ( ! started )
+		return;
+
+	if ( terminating )
+		return;
+
+	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
+
+	// Signal that it's ok for the thread to exit now by unlocking the
+	// mutex.
+	if ( pthread_mutex_unlock(&terminate) != 0 )
+		reporter->FatalError("Failure flagging terminate condition for thread %s", name.c_str());
+
+	terminating = true;
+
+	OnStop();
+	}
+
+void BasicThread::Join()
+	{
+	if ( ! started )
+		return;
+
+	if ( ! terminating )
+		Stop();
+
+	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
+
+	if ( pthread_join(pthread, 0) != 0  )
+		reporter->FatalError("Failure joining thread %s", name.c_str());
+
+	pthread_mutex_destroy(&terminate);
+
+	DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
+
+	pthread = 0;
+	}
+
+void BasicThread::Kill()
+	{
+	if ( ! (started && pthread) )
+		return;
+
+	// I believe this is safe to call from a signal handler ... Not error
+	// checking so that killing doesn't bail out if we have already
+	// terminated.
+	pthread_kill(pthread, SIGKILL);
+	}
+
+void* BasicThread::launcher(void *arg)
+	{
+	BasicThread* thread = (BasicThread *)arg;
+
+	// Block signals in thread. We handle signals only in the main
+	// process.
+	sigset_t mask_set;
+	sigfillset(&mask_set);
+	int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
+	assert(res == 0);  //
+
+	// Run thread's main function.
+	thread->Run();
+
+	// Wait until somebody actually wants us to terminate.
+	if ( pthread_mutex_lock(&thread->terminate) != 0 )
+		reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
+
+	return 0;
+	}
+

src/threading/BasicThread.h

@@ -0,0 +1,170 @@
+
+#ifndef THREADING_BASICTHREAD_H
+#define THREADING_BASICTHREAD_H
+
+#include <pthread.h>
+#include <semaphore.h>
+
+#include "Queue.h"
+#include "util.h"
+
+using namespace std;
+
+namespace threading {
+
+class Manager;
+
+/**
+ * Base class for all threads.
+ *
+ * This class encapsulates all the OS-level thread handling. All thread
+ * instances are automatically added to the threading::Manager for management. The
+ * manager also takes care of deleting them (which must not be done
+ * manually).
+ */
+class BasicThread
+{
+public:
+	/**
+	 * Creates a new thread object. Instantiating the object does however
+	 * not yet start the actual OS thread, that requires calling Start().
+	 *
+	 * Only Bro's main thread may create new thread instances.
+	 *
+	 * @param name A descriptive name for thread the thread. This may
+	 * show up in messages to the user.
+	 */
+	BasicThread();
+
+	/**
+	 * Returns a descriptive name for the thread. If not set via
+	 * SetName(). If not set, a default name is choosen automatically.
+	 *
+	 * This method is safe to call from any thread.
+	 */
+	const string& Name() const { return name; }
+
+	/**
+	* Sets a descriptive name for the thread. This should be a string
+	* that's useful in output presented to the user and uniquely
+	* identifies the thread.
+	*
+	* This method must be called only from the thread itself.
+	*/
+	void SetName(const string& name);
+
+	/**
+	 * Set the name shown by the OS as the thread's description. Not
+	 * supported on all OSs.
+	 */
+	void SetOSName(const string& name);
+
+	/**
+	 * Starts the thread. Calling this methods will spawn a new OS thread
+	 * executing Run(). Note that one can't restart a thread after a
+	 * Stop(), doing so will be ignored.
+	 *
+	 * Only Bro's main thread must call this method.
+	 */
+	void Start();
+
+	/**
+	 * Signals the thread to stop. The method lets Terminating() now
+	 * return true. It does however not force the thread to terminate.
+	 * It's up to the Run() method to to query Terminating() and exit
+	 * eventually.
+	 *
+	 * Calling this method has no effect if Start() hasn't been executed
+	 * yet.
+	 *
+	 * Only Bro's main thread must call this method.
+	 */
+	void Stop();
+
+	/**
+	 * Returns true if Stop() has been called.
+	 *
+	 * This method is safe to call from any thread.
+	 */
+	bool Terminating()  const { return terminating; }
+
+	/**
+	 * A version of fmt() that the thread can safely use.
+	 *
+	 * This is safe to call from Run() but must not be used from any
+	 * other thread than the current one.
+	 */
+	const char* Fmt(const char* format, ...);
+
+protected:
+	friend class Manager;
+
+	/**
+	 * Entry point for the thread. This must be overridden by derived
+	 * classes and will execute in a separate thread once Start() is
+	 * called. The thread will not terminate before this method finishes.
+	 * An implementation should regularly check Terminating() to see if
+	 * exiting has been requested.
+	 */
+	virtual void Run() = 0;
+
+	/**
+	 * Executed with Start(). This is a hook into starting the thread. It
+	 * will be called from Bro's main thread after the OS thread has been
+	 * started.
+	 */
+	virtual void OnStart()	{}
+
+	/**
+	 * Executed with Stop(). This is a hook into stopping the thread. It
+	 * will be called from Bro's main thread after the thread has been
+	 * signaled to stop.
+	 */
+	virtual void OnStop()	{}
+
+	/**
+	 * Destructor. This will be called by the manager.
+	 *
+	 * Only Bro's main thread may delete thread instances.
+	 *
+	 */
+	virtual ~BasicThread();
+
+	/**
+	 * Waits until the thread's Run() method has finished and then joins
+	 * it. This is called from the threading::Manager.
+	 */
+	void Join();
+
+	/**
+	 * Kills the thread immediately. One still needs to call Join()
+	 * afterwards.
+	 *
+	 * This is called from the threading::Manager and safe to execute
+	 * during a signal handler.
+	 */
+	void Kill();
+
+private:
+	// pthread entry function.
+	static void* launcher(void *arg);
+
+	string name;
+	pthread_t pthread;
+	bool started; 		// Set to to true once running.
+	bool terminating;	// Set to to true to signal termination.
+
+	// Used as a semaphore to tell the pthread thread when it may
+	// terminate.
+	pthread_mutex_t terminate;
+
+	// For implementing Fmt().
+	char* buf;
+	unsigned int buf_len;
+
+	static uint64_t thread_counter;
+};
+
+}
+
+#endif

src/threading/Manager.cc

@@ -0,0 +1,147 @@
+
+#include "Manager.h"
+
+using namespace threading;
+
+Manager::Manager()
+	{
+	DBG_LOG(DBG_THREADING, "Creating thread manager ...");
+
+	did_process = true;
+	next_beat = 0;
+	terminating = false;
+	idle = true;
+	}
+
+Manager::~Manager()
+	{
+	if ( all_threads.size() )
+		Terminate();
+	}
+
+void Manager::Terminate()
+	{
+	DBG_LOG(DBG_THREADING, "Terminating thread manager ...");
+
+	terminating = true;
+
+	// First process remaining thread output for the message threads.
+	do Process(); while ( did_process );
+
+	// Signal all to stop.
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		(*i)->Stop();
+
+	// Then join them all.
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		{
+		(*i)->Join();
+		delete *i;
+		}
+
+	all_threads.clear();
+	msg_threads.clear();
+
+	idle = true;
+	closed = true;
+	terminating = false;
+	}
+
+void Manager::KillThreads()
+	{
+	DBG_LOG(DBG_THREADING, "Killing threads ...");
+
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		(*i)->Kill();
+	}
+
+void Manager::AddThread(BasicThread* thread)
+	{
+	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
+	all_threads.push_back(thread);
+	idle = false;
+	}
+
+void Manager::AddMsgThread(MsgThread* thread)
+	{
+	DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str());
+	msg_threads.push_back(thread);
+	}
+
+void Manager::GetFds(int* read, int* write, int* except)
+	{
+	}
+
+double Manager::NextTimestamp(double* network_time)
+	{
+//	fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat);
+
+	if ( ::network_time && (did_process || ::network_time > next_beat || ! next_beat) )
+		// If we had something to process last time (or out heartbeat
+		// is due or not set yet), we want to check for more asap.
+		return timer_mgr->Time();
+
+	return -1.0;
+	}
+
+void Manager::Process()
+	{
+	bool do_beat = false;
+
+	if ( network_time && (network_time > next_beat || ! next_beat) )
+		{
+		do_beat = true;
+		next_beat = ::network_time + HEART_BEAT_INTERVAL;
+		}
+
+	did_process = false;
+
+	for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ )
+		{
+		MsgThread* t = *i;
+
+		if ( do_beat )
+			t->Heartbeat();
+
+		while ( t->HasOut() )
+			{
+			Message* msg = t->RetrieveOut();
+
+			if ( msg->Process() )
+				{
+				if ( network_time )
+					did_process = true;
+				}
+
+			else
+				{
+				string s = msg->Name() + " failed, terminating thread";
+				reporter->Error("%s", s.c_str());
+				t->Stop();
+			}
+
+			delete msg;
+			}
+		}
+
+//	fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat);
+	}
+
+const threading::Manager::msg_stats_list& threading::Manager::GetMsgThreadStats()
+	{
+	stats.clear();
+
+	for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ )
+		{
+		MsgThread* t = *i;
+
+		MsgThread::Stats s;
+		t->GetStats(&s);
+
+		stats.push_back(std::make_pair(t->Name(),s));
+		}
+
+	return stats;
+	}
+
+

src/threading/Manager.h

@@ -0,0 +1,146 @@
+
+#ifndef THREADING_MANAGER_H
+#define THREADING_MANAGER_H
+
+#include <list>
+
+#include "IOSource.h"
+
+#include "BasicThread.h"
+#include "MsgThread.h"
+
+namespace threading {
+
+/**
+ * The thread manager coordinates all child threads. Once a BasicThread is
+ * instantitated, it gets addedd to the manager, which will delete it later
+ * once it has terminated.
+ *
+ * In addition to basic threads, the manager also provides additional
+ * functionality specific to MsgThread instances. In particular, it polls
+ * their outgoing message queue on a regular basis and feeds data sent into
+ * the rest of Bro. It also triggers the regular heartbeats.
+ */
+class Manager : public IOSource
+{
+public:
+	/**
+	 * Constructor. Only a single instance of the manager must be
+	 * created.
+	 */
+	Manager();
+
+	/**
+	 * Destructir.
+	 */
+	~Manager();
+
+	/**
+	 * Terminates the manager's processor. The method signals all threads
+	 * to terminates and wait for them to do so. It then joins them and
+	 * returns to the caller. Afterwards, no more thread instances may be
+	 * created.
+	 */
+	void Terminate();
+
+	/**
+	 * Returns True if we are currently in Terminate() waiting for 
+	 * threads to exit.
+	 */
+	bool Terminating() const	{ return terminating; }
+
+	/**
+	 * Immediately kills all child threads. It does however not yet join
+	 * them, one still needs to call Terminate() for that.
+	 *
+	 * This method is safe to call from a signal handler, and can in fact
+	 * be called while Terminate() is already in progress.
+	 */
+	void KillThreads();
+
+	typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
+
+	/**
+	 * Returns statistics from all current MsgThread instances.
+	 *
+	 * @return A list of statistics, with one entry for each MsgThread.
+	 * Each entry is a tuple of thread name and statistics. The list
+	 * reference remains valid until the next call to this method (or
+	 * termination of the manager).
+	 */
+	const msg_stats_list& GetMsgThreadStats();
+
+	/**
+	 * Returns the number of currently active threads. This counts all
+	 * threads that are not yet joined, includingt any potentially in
+	 * Terminating() state.
+	 */
+	int NumThreads() const { return all_threads.size(); }
+
+protected:
+	friend class BasicThread;
+	friend class MsgThread;
+
+	/**
+	 * Registers a new basic thread with the manager. This is
+	 * automatically called by the thread's constructor.
+	 *
+	 * @param thread The thread.
+	 */
+	void AddThread(BasicThread* thread);
+
+	/**
+	 * Registers a new message thread with the manager. This is
+	 * automatically called by the thread's constructor. This must be
+	 * called \a in \a addition to AddThread(BasicThread* thread). The
+	 * MsgThread constructor makes sure to do so. 
+	 *
+	 * @param thread The thread.
+	 */
+	void AddMsgThread(MsgThread* thread);
+
+	/**
+	 * Part of the IOSource interface.
+	 */
+	virtual void GetFds(int* read, int* write, int* except);
+
+	/**
+	 * Part of the IOSource interface.
+	 */
+	virtual double NextTimestamp(double* network_time);
+
+	/**
+	 * Part of the IOSource interface.
+	 */
+	virtual void Process();
+
+	/**
+	 * Part of the IOSource interface.
+	 */
+	virtual const char* Tag()	{ return "threading::Manager"; }
+
+private:
+	static const int HEART_BEAT_INTERVAL = 1;
+
+	typedef std::list<BasicThread*> all_thread_list;
+	all_thread_list all_threads;
+
+	typedef std::list<MsgThread*> msg_thread_list;
+	msg_thread_list msg_threads;
+
+	bool did_process;	// True if the last Process() found some work to do.
+	double next_beat;	// Timestamp when the next heartbeat will be sent.
+	bool terminating;	// True if we are in Terminate().
+
+	msg_stats_list stats;
+};
+
+}
+
+/**
+ * A singleton instance of the thread manager. All methods must only be
+ * called from Bro's main thread.
+ */
+extern threading::Manager* thread_mgr;
+
+#endif

src/threading/MsgThread.cc

@@ -0,0 +1,289 @@
+
+#include "DebugLogger.h"
+
+#include "MsgThread.h"
+#include "Manager.h"
+
+using namespace threading;
+
+namespace threading  {
+
+////// Messages.
+
+// Signals child thread to terminate. This is actually a no-op; its only
+// purpose is unblock the current read operation so that the child's Run()
+// methods can check the termination status.
+class TerminateMessage : public InputMessage<MsgThread>
+{
+public:
+	TerminateMessage(MsgThread* thread) : InputMessage<MsgThread>("Terminate", thread)	{ }
+
+	virtual bool Process()	{ return true; }
+};
+
+/// Sends a heartbeat to the child thread.
+class HeartbeatMessage : public InputMessage<MsgThread>
+{
+public:
+	HeartbeatMessage(MsgThread* thread, double arg_network_time, double arg_current_time)
+		: InputMessage<MsgThread>("Heartbeat", thread)
+		{ network_time = arg_network_time; current_time = arg_current_time; }
+
+	virtual bool Process()	{ return Object()->DoHeartbeat(network_time, current_time); }
+
+private:
+	double network_time;
+	double current_time;
+};
+
+// A message from the child to be passed on to the Reporter.
+class ReporterMessage : public OutputMessage<MsgThread>
+{
+public:
+	enum Type {
+		INFO, WARNING, ERROR, FATAL_ERROR, FATAL_ERROR_WITH_CORE,
+		INTERNAL_WARNING, INTERNAL_ERROR
+	};
+
+	ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
+		: OutputMessage<MsgThread>("ReporterMessage", thread)
+		{ type = arg_type; msg = arg_msg; }
+
+	virtual bool Process();
+
+private:
+	string msg;
+	Type type;
+};
+
+#ifdef DEBUG
+// A debug message from the child to be passed on to the DebugLogger.
+class DebugMessage : public OutputMessage<MsgThread>
+{
+public:
+	DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
+		: OutputMessage<MsgThread>("DebugMessage", thread)
+		{ stream = arg_stream; msg = arg_msg; }
+
+	virtual bool Process()
+		{
+		string s = Object()->Name() + ": " + msg;
+		debug_logger.Log(stream, "%s", s.c_str());
+		return true;
+		}
+private:
+	string msg;
+	DebugStream stream;
+};
+#endif
+
+}
+
+////// Methods.
+
+Message::~Message()
+	{
+	}
+
+bool ReporterMessage::Process()
+	{
+	string s = Object()->Name() + ": " + msg;
+	const char* cmsg = s.c_str();
+
+	switch ( type ) {
+
+	case INFO:
+		reporter->Info("%s", cmsg);
+		break;
+
+	case WARNING:
+		reporter->Warning("%s", cmsg);
+		break;
+
+	case ERROR:
+		reporter->Error("%s", cmsg);
+		break;
+
+	case FATAL_ERROR:
+		reporter->FatalError("%s", cmsg);
+		break;
+
+	case FATAL_ERROR_WITH_CORE:
+		reporter->FatalErrorWithCore("%s", cmsg);
+		break;
+
+	case INTERNAL_WARNING:
+		reporter->InternalWarning("%s", cmsg);
+		break;
+
+	case INTERNAL_ERROR :
+		reporter->InternalError("%s", cmsg);
+		break;
+
+	default:
+		reporter->InternalError("unknown ReporterMessage type %d", type);
+	}
+
+	return true;
+	}
+
+MsgThread::MsgThread() : BasicThread()
+	{
+	cnt_sent_in = cnt_sent_out = 0;
+	thread_mgr->AddMsgThread(this);
+	}
+
+void MsgThread::OnStop()
+	{
+	// This is to unblock the current queue read operation.
+	SendIn(new TerminateMessage(this), true);
+	}
+
+void MsgThread::Heartbeat()
+	{
+	SendIn(new HeartbeatMessage(this, network_time, current_time()));
+	}
+
+bool MsgThread::DoHeartbeat(double network_time, double current_time)
+	{
+	string n = Name();
+
+	n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
+		cnt_sent_in - queue_in.Size(),
+		cnt_sent_out - queue_out.Size());
+
+	SetOSName(n.c_str());
+
+	return true;
+	}
+
+void MsgThread::Info(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::INFO, this, msg));
+	}
+
+void MsgThread::Warning(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::WARNING, this, msg));
+	}
+
+void MsgThread::Error(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::ERROR, this, msg));
+	}
+
+void MsgThread::FatalError(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR, this, msg));
+	}
+
+void MsgThread::FatalErrorWithCore(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::FATAL_ERROR_WITH_CORE, this, msg));
+	}
+
+void MsgThread::InternalWarning(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::INTERNAL_WARNING, this, msg));
+	}
+
+void MsgThread::InternalError(const char* msg)
+	{
+	SendOut(new ReporterMessage(ReporterMessage::INTERNAL_ERROR, this, msg));
+	}
+
+#ifdef DEBUG
+
+void MsgThread::Debug(DebugStream stream, const char* msg)
+	{
+	SendOut(new DebugMessage(stream, this, msg));
+	}
+
+#endif
+
+void MsgThread::SendIn(BasicInputMessage* msg, bool force)
+	{
+	if ( Terminating() && ! force )
+		{
+		delete msg;
+		return;
+		}
+
+	DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
+
+	queue_in.Put(msg);
+	++cnt_sent_in;
+	}
+
+
+void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
+	{
+	if ( Terminating() && ! force )
+		{
+		delete msg;
+		return;
+		}
+
+	queue_out.Put(msg);
+
+	++cnt_sent_out;
+	}
+
+BasicOutputMessage* MsgThread::RetrieveOut()
+	{
+	BasicOutputMessage* msg = queue_out.Get();
+	assert(msg);
+
+	DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s",  msg->Name().c_str(), Name().c_str());
+
+	return msg;
+	}
+
+BasicInputMessage* MsgThread::RetrieveIn()
+	{
+	BasicInputMessage* msg = queue_in.Get();
+	assert(msg);
+
+#ifdef DEBUG
+	string s = Fmt("Retrieved '%s' in %s",  msg->Name().c_str(), Name().c_str());
+	Debug(DBG_THREADING, s.c_str());
+#endif
+
+	return msg;
+	}
+
+void MsgThread::Run()
+	{
+	while ( true )
+		{
+		// When requested to terminate, we only do so when
+		// all input has been processed.
+		if ( Terminating() && ! queue_in.Ready() )
+			break;
+
+		BasicInputMessage* msg = RetrieveIn();
+
+		bool result = msg->Process();
+
+		if ( ! result )
+			{
+			string s = msg->Name() + " failed, terminating thread";
+			Error(s.c_str());
+			Stop();
+			break;
+			}
+
+		delete msg;
+		}
+	}
+
+void MsgThread::GetStats(Stats* stats)
+	{
+	stats->sent_in = cnt_sent_in;
+	stats->sent_out = cnt_sent_out;
+	stats->pending_in = queue_in.Size();
+	stats->pending_out = queue_out.Size();
+	queue_in.GetStats(&stats->queue_in_stats);
+	queue_out.GetStats(&stats->queue_out_stats);
+	}
+

src/threading/MsgThread.h

@@ -0,0 +1,402 @@
+
+#ifndef THREADING_MSGTHREAD_H
+#define THREADING_MSGTHREAD_H
+
+#include <pthread.h>
+
+#include "DebugLogger.h"
+
+#include "BasicThread.h"
+#include "Queue.h"
+
+namespace threading {
+
+class BasicInputMessage;
+class BasicOutputMessage;
+class HeartbeatMessage;
+
+/**
+ * A specialized thread that provides bi-directional message passing between
+ * Bro's main thread and the child thread. Messages are instances of
+ * BasicInputMessage and BasicOutputMessage for message sent \a to the child
+ * thread and received \a from the child thread, respectively.
+ *
+ * The thread's Run() method implements main loop that processes incoming
+ * messages until Terminating() indicates that execution should stop. Once
+ * that happens, the thread stops accepting any new messages, finishes
+ * processes all remaining ones still in the queue, and then exits.
+ */
+class MsgThread : public BasicThread
+{
+public:
+	/**
+	 * Constructor. It automatically registers the thread with the
+	 * threading::Manager.
+	 *
+	 * Only Bro's main thread may instantiate a new thread.
+	 */
+	MsgThread();
+
+	/**
+	 * Sends a message to the child thread. The message will be proceesed
+	 * once the thread has retrieved it from its incoming queue.
+	 *
+	 * Only the main thread may call this method.
+	 *
+	 * @param msg The message.
+	 */
+	void SendIn(BasicInputMessage* msg)	{ return SendIn(msg, false); }
+
+	/**
+	 * Sends a message from the child thread to the main thread.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg The mesasge.
+	 */
+	void SendOut(BasicOutputMessage* msg)	{ return SendOut(msg, false); }
+
+	/**
+	 * Reports an informational message from the child thread. The main
+	 * thread will pass this to the Reporter once received.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void Info(const char* msg);
+
+	/**
+	 * Reports a warning from the child thread that may indicate a
+	 * problem. The main thread will pass this to the Reporter once
+	 * received.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void Warning(const char* msg);
+
+	/**
+	 * Reports a non-fatal error from the child thread. The main thread
+	 * will pass this to the Reporter once received. Processing proceeds
+	 * normally after the error has been reported.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void Error(const char* msg);
+
+	/**
+	 * Reports a fatal error from the child thread. The main thread will
+	 * pass this to the Reporter once received. Bro will terminate after
+	 * the message has been reported.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void FatalError(const char* msg);
+
+	/**
+	 * Reports a fatal error from the child thread. The main thread will
+	 * pass this to the Reporter once received. Bro will terminate with a
+	 * core dump after the message has been reported.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void FatalErrorWithCore(const char* msg);
+
+	/**
+	 * Reports a potential internal problem from the child thread. The
+	 * main thread will pass this to the Reporter once received. Bro will
+	 * continue normally.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void InternalWarning(const char* msg);
+
+	/**
+	 * Reports an internal program error from the child thread. The main
+	 * thread will pass this to the Reporter once received. Bro will
+	 * terminate with a core dump after the message has been reported.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void InternalError(const char* msg);
+
+#ifdef DEBUG
+	/**
+	 * Records a debug message for the given stream from the child
+	 * thread. The main thread will pass this to the DebugLogger once
+	 * received.
+	 *
+	 * Only the child thread may call this method.
+	 *
+	 * @param msg  The message. It will be prefixed with the thread's name.
+	 */
+	void Debug(DebugStream stream, const char* msg);
+#endif
+
+	/**
+	 * Statistics about inter-thread communication.
+	 */
+	struct Stats
+		{
+		uint64_t sent_in;	//! Number of messages sent to the child thread.
+		uint64_t sent_out;	//! Number of messages sent from the child thread to the main thread
+		uint64_t pending_in;	//! Number of messages sent to the child but not yet processed.
+		uint64_t pending_out;	//! Number of messages sent from the child but not yet processed by the main thread.
+
+		/// Statistics from our queues.
+		Queue<BasicInputMessage *>::Stats  queue_in_stats;
+		Queue<BasicOutputMessage *>::Stats queue_out_stats;
+		};
+
+	/**
+	 * Returns statistics about the inter-thread communication.
+	 *
+	 * @param stats A pointer to a structure that will be filled with
+	 * current numbers.
+	 */
+	void GetStats(Stats* stats);
+
+protected:
+	friend class Manager;
+	friend class HeartbeatMessage;
+
+	/**
+	 * Pops a message sent by the child from the child-to-main queue.
+	 *
+	 * This is method is called regularly by the threading::Manager.
+	 *
+	 * @return The message, wth ownership passed to caller. Returns null
+	 * if the queue is empty.
+	 */
+	BasicOutputMessage* RetrieveOut();
+
+	/**
+	 * Triggers a heartbeat message being sent to the client thread.
+	 *
+	 * This is method is called regularly by the threading::Manager.
+	 *
+	 * Can be overriden in derived classed to hook into the heart beat,
+	 * but must call the parent implementation. Note that this method is
+	 * always called by the main thread and must not access data of the
+	 * child thread directly. See DoHeartbeat() if you want to do
+	 * something on the child-side.
+	 */
+	virtual void Heartbeat();
+
+	/**
+	 * Overriden from BasicThread.
+	 *
+	 */
+	virtual void Run();
+	virtual void OnStop();
+
+	/**
+	 * Regulatly triggered for execution in the child thread.
+	 *
+	 * When overriding, one must call the parent class' implementation.
+	 *
+	 * network_time: The network_time when the heartbeat was trigger by
+	 * the main thread.
+	 *
+	 * current_time: Wall clock when the heartbeat was trigger by the
+	 * main thread.
+	 */
+	virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+	/**
+	 * Pops a message sent by the main thread from the main-to-chold
+	 * queue.
+	 *
+	 * Must only be called by the child thread.
+	 *
+	 * @return The message, wth ownership passed to caller. Returns null
+	 * if the queue is empty.
+	 */
+	BasicInputMessage* RetrieveIn();
+
+	/**
+	 * Queues a message for the child.
+	 *
+	 * Must only be called by the main thread.
+	 *
+	 * @param msg  The message.
+	 *
+	 * @param force: If true, the message will be queued even when we're already
+	 * Terminating(). Normally, the message would be discarded in that
+	 * case.
+	 */
+	void SendIn(BasicInputMessage* msg, bool force);
+
+	/**
+	 * Queues a message for the main thread.
+	 *
+	 * Must only be called by the child thread.
+	 *
+	 * @param msg  The message.
+	 *
+	 * @param force: If true, the message will be queued even when we're already
+	 * Terminating(). Normally, the message would be discarded in that
+	 * case.
+	 */
+	void SendOut(BasicOutputMessage* msg, bool force);
+
+	/**
+	 * Returns true if there's at least one message pending for the child
+	 * thread.
+	 */
+	bool HasIn()	{ return queue_in.Ready(); }
+
+	/**
+	 * Returns true if there's at least one message pending for the main
+	 * thread.
+	 */
+	bool HasOut()	{ return queue_out.Ready(); }
+
+	Queue<BasicInputMessage *> queue_in;
+	Queue<BasicOutputMessage *> queue_out;
+
+	uint64_t cnt_sent_in;	// Counts message sent to child.
+	uint64_t cnt_sent_out;	// Counts message sent by child.
+};
+
+/**
+ * Base class for all message between Bro's main process and a MsgThread.
+ */
+class Message
+{
+public:
+	/**
+	 * Destructor.
+	 */
+	virtual ~Message();
+
+	/**
+	 * Returns a descriptive name for the message's general type. This is
+	 * what's passed into the constructor and used mainly for debugging
+	 * purposes.
+	 */
+	const string& Name() const { return name; }
+
+	/**
+	 * Callback that must be overriden for processing a message.
+	 */
+	virtual bool Process() = 0; // Thread will be terminated if returngin false.
+
+protected:
+	/**
+	 * Constructor.
+	 *
+	 * @param arg_name A descriptive name for the type of message. Used
+	 * mainly for debugging purposes.
+	 */
+	Message(const string& arg_name)	{ name = arg_name; }
+
+private:
+	string name;
+};
+
+/**
+ * Base class for messages sent from Bro's main thread to a child MsgThread.
+ */
+class BasicInputMessage : public Message
+{
+protected:
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the type of message. Used
+	 * mainly for debugging purposes.
+	 */
+	BasicInputMessage(const string& name) : Message(name)	{}
+};
+
+/**
+ * Base class for messages sent from a child MsgThread to Bro's main thread.
+ */
+class BasicOutputMessage : public Message
+{
+protected:
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the type of message. Used
+	 * mainly for debugging purposes.
+	 */
+	BasicOutputMessage(const string& name) : Message(name)	{}
+};
+
+/**
+ * A paremeterized InputMessage that stores a pointer to an argument object.
+ * Normally, the objects will be used from the Process() callback.
+ */
+template<typename O>
+class InputMessage : public BasicInputMessage
+{
+public:
+	/**
+	 * Returns the objects passed to the constructor.
+	 */
+	O* Object() const { return object; }
+
+protected:
+	/**
+	 * Constructor.
+	 *
+	 * @param name: A descriptive name for the type of message. Used
+	 * mainly for debugging purposes.
+	 *
+	 * @param arg_object: An object to store with the message. 
+	 */
+	InputMessage(const string& name, O* arg_object) : BasicInputMessage(name)
+		{ object = arg_object; }
+
+private:
+	O* object;
+};
+
+/**
+ * A paremeterized OututMessage that stores a pointer to an argument object.
+ * Normally, the objects will be used from the Process() callback.
+ */
+template<typename O>
+class OutputMessage : public BasicOutputMessage
+{
+public:
+	/**
+	 * Returns the objects passed to the constructor.
+	 */
+	O* Object() const { return object; }
+
+protected:
+	/**
+	 * Constructor.
+	 *
+	 * @param name A descriptive name for the type of message. Used
+	 * mainly for debugging purposes.
+	 *
+	 * @param arg_object An object to store with the message. 
+	 */
+	OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name)
+		{ object = arg_object; }
+
+private:
+	O* object;
+};
+
+}
+
+
+#endif

src/threading/Queue.h

@@ -0,0 +1,222 @@
+
+#ifndef THREADING_QUEUE_H
+#define THREADING_QUEUE_H
+
+#include <pthread.h>
+#include <queue>
+#include <deque>
+#include <stdint.h>
+
+#include "Reporter.h"
+
+#undef Queue // Defined elsewhere unfortunately.
+
+namespace threading {
+
+/**
+ * A thread-safe single-reader single-writer queue.
+ *
+ * The implementation uses multiple queues and reads/writes in rotary fashion
+ * in an attempt to limit contention.
+ *
+ * All Queue instances must be instantiated by Bro's main thread.
+ *
+ * TODO: Unclear how critical performance is for this qeueue. We could like;y
+ * optimize it further if helpful.
+ */
+template<typename T>
+class Queue
+{
+public:
+	/**
+	 * Constructor.
+	 */
+	Queue();
+
+	/**
+	 * Destructor.
+	 */
+	~Queue();
+
+	/**
+	 * Retrieves one elment.
+	 */
+	T Get();
+
+	/**
+	 * Queues one element.
+	 */
+	void Put(T data);
+
+	/**
+	 * Returns true if the next Get() operation will succeed.
+	 */
+	bool Ready();
+
+	/**
+	 * Returns the number of queued items not yet retrieved.
+	 */
+	uint64_t Size();
+
+	/**
+	 * Statistics about inter-thread communication.
+	 */
+	struct Stats
+		{
+		uint64_t num_reads;	//! Number of messages read from the queue.
+		uint64_t num_writes;	//! Number of messages written to the queue.
+		};
+
+	/**
+	 * Returns statistics about the queue's usage.
+	 *
+	 * @param stats A pointer to a structure that will be filled with
+	 * current numbers. */
+	void GetStats(Stats* stats);
+
+private:
+	static const int NUM_QUEUES = 8;
+
+	pthread_mutex_t mutex[NUM_QUEUES];	// Mutex protected shared accesses.
+	pthread_cond_t has_data[NUM_QUEUES];	// Signals when data becomes available
+	std::queue<T> messages[NUM_QUEUES];	// Actually holds the queued messages
+
+	int read_ptr;	// Where the next operation will read from
+	int write_ptr;	// Where the next operation will write to
+
+	// Statistics.
+	uint64_t num_reads;
+	uint64_t num_writes;
+};
+
+inline static void safe_lock(pthread_mutex_t* mutex)
+	{
+	if ( pthread_mutex_lock(mutex) != 0 )
+		reporter->FatalErrorWithCore("cannot lock mutex");
+	}
+
+inline static void safe_unlock(pthread_mutex_t* mutex)
+	{
+	if ( pthread_mutex_unlock(mutex) != 0 )
+		reporter->FatalErrorWithCore("cannot unlock mutex");
+	}
+
+template<typename T>
+inline Queue<T>::Queue()
+	{
+	read_ptr = 0;
+	write_ptr = 0;
+	num_reads = num_writes = 0;
+
+	for( int i = 0; i < NUM_QUEUES; ++i )
+		{
+		if ( pthread_cond_init(&has_data[i], NULL) != 0 )
+			reporter->FatalError("cannot init queue condition variable");
+
+		if ( pthread_mutex_init(&mutex[i], NULL) != 0 )
+			reporter->FatalError("cannot init queue mutex");
+		}
+	}
+
+template<typename T>
+inline Queue<T>::~Queue()
+	{
+	for( int i = 0; i < NUM_QUEUES; ++i )
+		{
+		pthread_cond_destroy(&has_data[i]);
+		pthread_mutex_destroy(&mutex[i]);
+		}
+	}
+
+template<typename T>
+inline T Queue<T>::Get()
+	{
+	safe_lock(&mutex[read_ptr]);
+
+	int old_read_ptr = read_ptr;
+
+	if ( messages[read_ptr].empty() )
+		pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]);
+
+	T data = messages[read_ptr].front();
+	messages[read_ptr].pop();
+
+	read_ptr = (read_ptr + 1) % NUM_QUEUES;
+	++num_reads;
+
+	safe_unlock(&mutex[old_read_ptr]);
+
+	return data;
+	}
+
+template<typename T>
+inline void Queue<T>::Put(T data)
+	{
+	safe_lock(&mutex[write_ptr]);
+
+	int old_write_ptr = write_ptr;
+
+	bool need_signal = messages[write_ptr].empty();
+
+	messages[write_ptr].push(data);
+
+	if ( need_signal )
+		pthread_cond_signal(&has_data[write_ptr]);
+
+	write_ptr = (write_ptr + 1) % NUM_QUEUES;
+	++num_writes;
+
+	safe_unlock(&mutex[old_write_ptr]);
+	}
+
+
+template<typename T>
+inline bool Queue<T>::Ready()
+	{
+	safe_lock(&mutex[read_ptr]);
+
+	bool ret = (messages[read_ptr].size());
+
+	safe_unlock(&mutex[read_ptr]);
+
+	return ret;
+	}
+
+template<typename T>
+inline uint64_t Queue<T>::Size()
+	{
+	// Need to lock all queues.
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		safe_lock(&mutex[i]);
+
+	uint64_t size = 0;
+
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		size += messages[i].size();
+
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		safe_unlock(&mutex[i]);
+
+	return size;
+	}
+
+template<typename T>
+inline void Queue<T>::GetStats(Stats* stats)
+	{
+	// To be safe, we look all queues. That's probably unneccessary, but
+	// doesn't really hurt.
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		safe_lock(&mutex[i]);
+
+	stats->num_reads = num_reads;
+	stats->num_writes = num_writes;
+
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		safe_unlock(&mutex[i]);
+	}
+
+}
+
+
+#endif
+

src/threading/SerialTypes.cc

@@ -0,0 +1,355 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+
+#include "SerialTypes.h"
+#include "../RemoteSerializer.h"
+
+
+using namespace threading;
+
+bool Field::Read(SerializationFormat* fmt)
+	{
+	int t;
+	int st;
+
+	bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") );
+	type = (TypeTag) t;
+	subtype = (TypeTag) st;
+
+	return success;
+	}
+
+bool Field::Write(SerializationFormat* fmt) const
+	{
+	return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype"));
+	}
+
+Value::~Value()
+	{
+	if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
+	     && present )
+		delete val.string_val;
+
+	if ( type == TYPE_TABLE && present )
+		{
+		for ( int i = 0; i < val.set_val.size; i++ )
+			delete val.set_val.vals[i];
+
+		delete [] val.set_val.vals;
+		}
+
+	if ( type == TYPE_VECTOR && present )
+		{
+		for ( int i = 0; i < val.vector_val.size; i++ )
+			delete val.vector_val.vals[i];
+
+		delete [] val.vector_val.vals;
+		}
+	}
+
+bool Value::IsCompatibleType(BroType* t, bool atomic_only)
+	{
+	if ( ! t )
+		return false;
+
+	switch ( t->Tag() )	{
+	case TYPE_BOOL:
+	case TYPE_INT:
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+	case TYPE_PORT:
+	case TYPE_SUBNET:
+	case TYPE_ADDR:
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		return true;
+
+	case TYPE_RECORD:
+		return ! atomic_only;
+
+	case TYPE_TABLE:
+		{
+		if ( atomic_only )
+			return false;
+
+		if ( ! t->IsSet() )
+			return false;
+
+		return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true);
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( atomic_only )
+			return false;
+
+		return IsCompatibleType(t->AsVectorType()->YieldType(), true);
+		}
+
+	default:
+		return false;
+	}
+
+	return false;
+	}
+
+bool Value::Read(SerializationFormat* fmt)
+	{
+	int ty;
+
+	if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) )
+		return false;
+
+	type = (TypeTag)(ty);
+
+	if ( ! present )
+		return true;
+
+	switch ( type ) {
+	case TYPE_BOOL:
+	case TYPE_INT:
+		return fmt->Read(&val.int_val, "int");
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		return fmt->Read(&val.uint_val, "uint");
+
+	case TYPE_PORT: {
+		int proto;
+		if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) {
+			return false;
+		}
+
+		switch ( proto ) {
+			case 0:
+				val.port_val.proto = TRANSPORT_UNKNOWN;
+				break;
+			case 1:
+				val.port_val.proto = TRANSPORT_TCP;
+				break;
+			case 2:
+				val.port_val.proto = TRANSPORT_UDP;
+				break;
+			case 3:
+				val.port_val.proto = TRANSPORT_ICMP;
+				break;
+			default:
+				return false;
+		}
+
+		return true;
+		}
+
+	case TYPE_ADDR:
+		{
+		char family;
+
+		if ( ! fmt->Read(&family, "addr-family") )
+			return false;
+
+		switch ( family ) {
+		case 4:
+			val.addr_val.family = IPv4;
+			return fmt->Read(&val.addr_val.in.in4, "addr-in4");
+
+		case 6:
+			val.addr_val.family = IPv6;
+			return fmt->Read(&val.addr_val.in.in6, "addr-in6");
+
+		}
+
+		// Can't be reached.
+		abort();
+		}
+
+	case TYPE_SUBNET:
+		{
+		char length;
+		char family;
+
+		if ( ! (fmt->Read(&length, "subnet-len") && fmt->Read(&family, "subnet-family")) ) 
+			return false;
+
+		switch ( family ) {
+		case 4:
+			val.subnet_val.length = (uint8_t)length;
+			val.subnet_val.prefix.family = IPv4;
+			return fmt->Read(&val.subnet_val.prefix.in.in4, "subnet-in4");
+
+		case 6:
+			val.subnet_val.length = (uint8_t)length;
+			val.subnet_val.prefix.family = IPv6;
+			return fmt->Read(&val.subnet_val.prefix.in.in6, "subnet-in6");
+
+		}
+
+		// Can't be reached.
+		abort();
+		}
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		return fmt->Read(&val.double_val, "double");
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		{
+		val.string_val = new string;
+		return fmt->Read(val.string_val, "string");
+		}
+
+	case TYPE_TABLE:
+		{
+		if ( ! fmt->Read(&val.set_val.size, "set_size") )
+			return false;
+
+		val.set_val.vals = new Value* [val.set_val.size];
+
+		for ( int i = 0; i < val.set_val.size; ++i )
+			{
+			val.set_val.vals[i] = new Value;
+
+			if ( ! val.set_val.vals[i]->Read(fmt) )
+				return false;
+			}
+
+		return true;
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( ! fmt->Read(&val.vector_val.size, "vector_size") )
+			return false;
+
+		val.vector_val.vals = new Value* [val.vector_val.size];
+
+		for ( int i = 0; i < val.vector_val.size; ++i )
+			{
+			val.vector_val.vals[i] = new Value;
+
+			if ( ! val.vector_val.vals[i]->Read(fmt) )
+				return false;
+			}
+
+		return true;
+		}
+
+	default:
+		reporter->InternalError("unsupported type %s in Value::Write", type_name(type));
+	}
+
+	return false;
+	}
+
+bool Value::Write(SerializationFormat* fmt) const
+	{
+	if ( ! (fmt->Write((int)type, "type") &&
+		fmt->Write(present, "present")) )
+		return false;
+
+	if ( ! present )
+		return true;
+
+	switch ( type ) {
+	case TYPE_BOOL:
+	case TYPE_INT:
+		return fmt->Write(val.int_val, "int");
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		return fmt->Write(val.uint_val, "uint");
+
+	case TYPE_PORT:
+		return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto");
+
+	case TYPE_ADDR:
+		{
+		switch ( val.addr_val.family ) {
+		case IPv4:
+			return fmt->Write((char)4, "addr-family")
+				&& fmt->Write(val.addr_val.in.in4, "addr-in4");
+
+		case IPv6:
+			return fmt->Write((char)6, "addr-family")
+				&& fmt->Write(val.addr_val.in.in6, "addr-in6");
+			break;
+		}
+
+		// Can't be reached.
+		abort();
+		}
+
+	case TYPE_SUBNET:
+		{
+		if ( ! fmt->Write((char)val.subnet_val.length, "subnet-length") )
+			return false;
+
+		switch ( val.subnet_val.prefix.family ) {
+		case IPv4:
+			return fmt->Write((char)4, "subnet-family")
+				&& fmt->Write(val.subnet_val.prefix.in.in4, "subnet-in4");
+
+		case IPv6:
+			return fmt->Write((char)6, "subnet-family")
+				&& fmt->Write(val.subnet_val.prefix.in.in6, "subnet-in6");
+			break;
+		}
+
+		// Can't be reached.
+		abort();
+		}
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		return fmt->Write(val.double_val, "double");
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		return fmt->Write(*val.string_val, "string");
+
+	case TYPE_TABLE:
+		{
+		if ( ! fmt->Write(val.set_val.size, "set_size") )
+			return false;
+
+		for ( int i = 0; i < val.set_val.size; ++i )
+			{
+			if ( ! val.set_val.vals[i]->Write(fmt) )
+				return false;
+			}
+
+		return true;
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( ! fmt->Write(val.vector_val.size, "vector_size") )
+			return false;
+
+		for ( int i = 0; i < val.vector_val.size; ++i )
+			{
+			if ( ! val.vector_val.vals[i]->Write(fmt) )
+				return false;
+			}
+
+		return true;
+		}
+
+	default:
+		reporter->InternalError("unsupported type %s in Value::REad", type_name(type));
+	}
+
+	return false;
+	}
+

src/threading/SerialTypes.h

@@ -0,0 +1,146 @@
+
+#ifndef THREADING_SERIALIZATIONTYPES_H
+#define THREADING_SERIALIZATIONTYPES_H
+
+using namespace std;
+
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include "Type.h"
+#include "net_util.h"
+
+class SerializationFormat;
+
+namespace threading {
+
+/**
+ * Definition of a log file, i.e., one column of a log stream.
+ */
+struct Field {
+	string name;	//! Name of the field.
+	TypeTag type;	//! Type of the field.
+	TypeTag subtype;	//! Inner type for sets.
+
+	/**
+	 * Constructor.
+	 */
+	Field() 	{ subtype = TYPE_VOID; }
+
+	/**
+	 * Copy constructor.
+	 */
+	Field(const Field& other)
+		: name(other.name), type(other.type), subtype(other.subtype) {  }
+
+	/**
+	 * Unserializes a field.
+	 *
+	 * @param fmt The serialization format to use. The format handles
+	 * low-level I/O.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Read(SerializationFormat* fmt);
+
+	/**
+	 * Serializes a field.
+	 *
+	 * @param fmt The serialization format to use. The format handles
+	 * low-level I/O.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Write(SerializationFormat* fmt) const;
+};
+
+/**
+ * Definition of a log value, i.e., a entry logged by a stream.
+ *
+ * This struct essentialy represents a serialization of a Val instance (for
+ * those Vals supported).
+ */
+struct Value {
+	TypeTag type;	//! The type of the value.
+	bool present;	//! False for optional record fields that are not set.
+
+	struct set_t { bro_int_t size; Value** vals; };
+	typedef set_t vec_t;
+	struct port_t { bro_uint_t port; TransportProto proto; };
+
+        struct addr_t {
+		IPFamily family;
+		union {
+			struct in_addr in4;
+			struct in6_addr in6;
+		} in;
+	};
+
+	struct subnet_t { addr_t prefix; uint8_t length; };
+
+	/**
+	 * This union is a subset of BroValUnion, including only the types we
+	 * can log directly. See IsCompatibleType().
+	 */
+	union _val {
+		bro_int_t int_val;
+		bro_uint_t uint_val;
+		port_t port_val;
+		double double_val;
+		set_t set_val;
+		vec_t vector_val;
+		addr_t addr_val;
+		subnet_t subnet_val;
+		string* string_val;
+	} val;
+
+	/**
+	* Constructor.
+	*
+	* arg_type: The type of the value.
+	*
+	* arg_present: False if the value represents an optional record field
+	* that is not set.
+	 */
+	Value(TypeTag arg_type = TYPE_ERROR, bool arg_present = true)
+		: type(arg_type), present(arg_present)	{}
+
+	/**
+	 * Destructor.
+	 */
+	~Value();
+
+	/**
+	 * Unserializes a value.
+	 *
+	 * @param fmt The serialization format to use. The format handles low-level I/O.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Read(SerializationFormat* fmt);
+
+	/**
+	 * Serializes a value.
+	 *
+	 * @param fmt The serialization format to use. The format handles
+	 * low-level I/O.
+	 *
+	 * @return False if an error occured.
+	 */
+	bool Write(SerializationFormat* fmt) const;
+
+	/**
+	 * Returns true if the type can be represented by a Value. If
+	 * `atomic_only` is true, will not permit composite types.
+	 */
+	static bool IsCompatibleType(BroType* t, bool atomic_only=false);
+
+private:
+friend class ::IPAddr;
+	Value(const Value& other)	{ } // Disabled.
+};
+
+}
+
+#endif /* THREADING_SERIALIZATIONTZPES_H */

src/util.h

@@ -37,7 +37,7 @@
 
 #endif
 
-#ifdef USE_PERFTOOLS
+#ifdef USE_PERFTOOLS_DEBUG
 #include <google/heap-checker.h>
 #include <google/heap-profiler.h>
 extern HeapLeakChecker* heap_checker;

testing/btest/Baseline/bifs.routing0_data_to_addrs/output

@@ -1,4 +1 @@
-{
+[2001:78:1:32::1, 2001:78:1:32::2]
-2001:78:1:32::1,
-2001:78:1:32::2
-}

testing/btest/Baseline/core.ipv6-atomic-frag/output

@@ -0,0 +1,4 @@
+[orig_h=2001:db8:1::2, orig_p=36951/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp]
+[orig_h=2001:db8:1::2, orig_p=59694/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp]
+[orig_h=2001:db8:1::2, orig_p=27393/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp]
+[orig_h=2001:db8:1::2, orig_p=45805/tcp, resp_h=2001:db8:1::1, resp_p=80/tcp]

testing/btest/Baseline/core.ipv6_ext_headers/output

@@ -1 +1 @@
-[ip=<uninitialized>, ip6=[class=0, flow=0, len=59, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>], [id=43, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=[nxt=17, len=4, rtype=0, segleft=2, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>]]], tcp=<uninitialized>, udp=[sport=53/udp, dport=53/udp, ulen=11], icmp=<uninitialized>]
+[ip=<uninitialized>, ip6=[class=0, flow=0, len=68, nxt=0, hlim=64, src=2001:4f8:4:7:2e0:81ff:fe52:ffff, dst=2001:4f8:4:7:2e0:81ff:fe52:9a6b, exts=[[id=0, hopopts=[nxt=43, len=0, options=[[otype=1, len=4, data=\0\0\0\0]]], dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>], [id=43, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=[nxt=6, len=4, rtype=0, segleft=0, data=\0\0\0\0 ^A\0x\0^A\02\0\0\0\0\0\0\0^A ^A\0x\0^A\02\0\0\0\0\0\0\0^B], fragment=<uninitialized>, ah=<uninitialized>, esp=<uninitialized>]]], tcp=[sport=30000/tcp, dport=80/tcp, seq=0, ack=0, hl=20, dl=0, flags=2, win=8192], udp=<uninitialized>, icmp=<uninitialized>]

testing/btest/Baseline/core.ipv6_rh0/segleft.out

@@ -0,0 +1,2 @@
+flow_weird routing0_segleft from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:78:1:32::2
+rh0 w/ segments left from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:4f8:4:7:2e0:81ff:fe52:9a6b

testing/btest/Baseline/core.ipv6_rh0/segleft0.out

@@ -0,0 +1,2 @@
+flow_weird routing0_header from 2001:4f8:4:7:2e0:81ff:fe52:ffff to 2001:4f8:4:7:2e0:81ff:fe52:9a6b
+new_connection: [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/tcp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=80/tcp]

testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	metrics
+#fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
+#types	time	enum	string	addr	string	subnet	count
+1331256494.591966	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1331256494.591966	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1331256494.591966	TEST_METRIC	foo-bar	1.2.3.4	-	-	6

testing/btest/Baseline/core.leaks.remote/sender.test.failure.log

@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test.failure
+#fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+#types	time	addr	port	addr	port	string	string
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/core.leaks.remote/sender.test.log

@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+#types	time	addr	port	addr	port	string	string
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/core.leaks.remote/sender.test.success.log

@@ -0,0 +1,9 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test.success
+#fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+#types	time	addr	port	addr	port	string	string
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR

testing/btest/bifs/routing0_data_to_addrs.test

@@ -1,7 +1,7 @@
-# @TEST-EXEC: bro -C -b -r $TRACES/ext_hdr_hbh_routing.trace %INPUT >output
+# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft.trace %INPUT >output
 # @TEST-EXEC: btest-diff output
 
-event ipv6_ext_headers(c: connection, p: pkt_hdr)
+event rh0_segleft(p: pkt_hdr)
 	{
 	for ( h in p$ip6$exts )
 		if ( p$ip6$exts[h]$id == IPPROTO_ROUTING )

testing/btest/core/ipv6-atomic-frag.test

@@ -0,0 +1,7 @@
+# @TEST-EXEC: bro -r $TRACES/ipv6-http-atomic-frag.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+event new_connection(c: connection)
+	{
+	print c$id;
+	}

testing/btest/core/ipv6_ext_headers.test

@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro -C -b -r $TRACES/ext_hdr_hbh_routing.trace %INPUT >output
+# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft0.trace %INPUT >output
 # @TEST-EXEC: btest-diff output
 
 # Just check that the event is raised correctly for a packet containing

testing/btest/core/ipv6_rh0.test

@@ -0,0 +1,22 @@
+# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft0.trace %INPUT >segleft0.out
+# @TEST-EXEC: btest-diff segleft0.out
+# @TEST-EXEC: bro -b -r $TRACES/ipv6-hbh-rh0-segleft.trace %INPUT >segleft.out
+# @TEST-EXEC: btest-diff segleft.out
+
+# This will be raised only by the packet with RH0 and segments left.
+event rh0_segleft(p: pkt_hdr)
+	{
+	print fmt("rh0 w/ segments left from %s to %s", p$ip6$src, p$ip6$dst);
+	}
+
+# This will be raised only by the packet with RH0 and no segments left.
+event new_connection(c: connection)
+	{
+	print fmt("new_connection: %s", c$id);
+	}
+
+# This will be raised by any packet with RH0 regardless of segments left.
+event flow_weird(name: string, src: addr, dst: addr)
+	{
+	print fmt("flow_weird %s from %s to %s", name, src, dst);
+	}

testing/btest/core/leaks/basic-cluster.bro

@@ -0,0 +1,39 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
+# @TEST-EXEC: btest-bg-wait -k 30
+# @TEST-EXEC: btest-diff manager-1/metrics.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
+};
+@TEST-END-FILE
+
+redef Log::default_rotation_interval = 0secs;
+
+redef enum Metrics::ID += {
+	TEST_METRIC,
+};
+
+event bro_init() &priority=5
+	{
+	Metrics::add_filter(TEST_METRIC, 
+		[$name="foo-bar",
+		 $break_interval=3secs]);
+		
+	if ( Cluster::local_node_type() == Cluster::WORKER )
+		{
+		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+		}
+	}

testing/btest/core/leaks/remote.bro

@@ -0,0 +1,79 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-diff sender/test.log
+# @TEST-EXEC: btest-diff sender/test.failure.log
+# @TEST-EXEC: btest-diff sender/test.success.log
+# @TEST-EXEC: cmp receiver/test.log sender/test.log
+# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
+# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+
+# This is the common part loaded by both sender and receiver.
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { LOG };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Log]);
+	Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]);
+}
+
+#####
+
+@TEST-START-FILE sender.bro
+
+module Test;
+
+@load frameworks/communication/listen
+
+function fail(rec: Log): bool
+	{
+	return rec$status != "success";
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]);
+
+	local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	local r: Log = [$t=network_time(), $id=cid, $status="success"];
+
+	# Log something.
+	Log::write(Test::LOG, r);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	disconnect(p);
+	}
+@TEST-END-FILE
+
+@TEST-START-FILE receiver.bro
+
+#####
+
+redef Communication::nodes += {
+    ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
+};
+
+@TEST-END-FILE

testing/external/scripts/perftools-adapt-paths

@@ -7,4 +7,4 @@
 
 cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1
 
-grep -q "No leaks found" $1
+grep -qv "detected leaks of" $1

testing/scripts/diff-canon-notice-policy

@@ -0,0 +1,10 @@
+#! /usr/bin/awk -f
+#
+# A diff canonifier that removes the priorities in notice_policy.log.
+
+/^#/ && $2 == "notice_policy" { filter = 1; }
+
+filter == 1 && /^[^#]/        { sub("^[0-9]*", "X"); }
+
+{ print; }
+

testing/scripts/diff-canonifier-external

@@ -6,4 +6,6 @@
     | `dirname $0`/diff-remove-uids \
     | `dirname $0`/diff-remove-mime-types \
     | `dirname $0`/diff-remove-x509-names \
+    | `dirname $0`/diff-canon-notice-policy \
     | `dirname $0`/diff-sort
+