From 908b1a17d1b08a8473695316e56eb98f7b005cbd Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 15 Nov 2011 09:51:02 -0500
Subject: [PATCH 001/238] Adding PPPoE support to Bro.

- Still needs a small test tracefile and test.
---
 src/PktSrc.cc | 36 +++++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 9 deletions(-)

diff --git a/src/PktSrc.cc b/src/PktSrc.cc
index 68b9785e6f..d86952a61f 100644
--- a/src/PktSrc.cc
+++ b/src/PktSrc.cc
@@ -208,16 +208,34 @@ void PktSrc::Process()
 		// Get protocol being carried from the ethernet frame.
 		protocol = (data[12] << 8) + data[13];
 
-		// MPLS carried over the ethernet frame.
-		if ( protocol == 0x8847 )
-			have_mpls = true;
-
-		// VLAN carried over ethernet frame.
-		else if ( protocol == 0x8100 )
+		switch ( protocol ) 
 			{
-			data += get_link_header_size(datalink);
-			data += 4; // Skip the vlan header
-			pkt_hdr_size = 0;
+			// MPLS carried over the ethernet frame.
+			case 0x8847:
+				have_mpls = true;
+				break;
+		
+			// VLAN carried over the ethernet frame.
+			case 0x8100:
+				data += get_link_header_size(datalink);
+				data += 4; // Skip the vlan header
+				pkt_hdr_size = 0;
+				break;
+			
+			// PPPoE carried over the ethernet frame.
+			case 0x8864:
+				data += get_link_header_size(datalink);
+				protocol = (data[6] << 8) + data[7];
+				data += 8; // Skip the PPPoE session and PPP header
+				pkt_hdr_size = 0;
+				if ( protocol != 0x0021 && protocol != 0x0057 )
+					{
+					// Neither IPv4 nor IPv6.
+					sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data);
+					data = 0;
+					return;
+					}
+				break;
 			}
 
 		break;

From 95f000738bc8f04559b0f3d8ba98ae369a9c640c Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 12:40:09 -0400
Subject: [PATCH 002/238] ElasticSearch log writer implementation test 1 -
 writes out JSON to file instead of sending it over HTTP for now.

---
 src/logging/Manager.cc               |   9 +
 src/logging/writers/ElasticSearch.cc | 242 +++++++++++++++++++++++++++
 src/logging/writers/ElasticSearch.h  |  67 ++++++++
 3 files changed, 318 insertions(+)
 create mode 100644 src/logging/writers/ElasticSearch.cc
 create mode 100644 src/logging/writers/ElasticSearch.h

diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index baf832e6a9..d338ac97f8 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -17,6 +17,10 @@
 #include "writers/Ascii.h"
 #include "writers/None.h"
 
+#ifdef USE_ELASTICSEARCH
+#include "writers/ElasticSearch.h"
+#endif
+
 #ifdef USE_DATASERIES
 #include "writers/DataSeries.h"
 #endif
@@ -35,6 +39,11 @@ struct WriterDefinition {
 WriterDefinition log_writers[] = {
 	{ BifEnum::Log::WRITER_NONE,  "None", 0, writer::None::Instantiate },
 	{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
+
+#ifdef USE_ELASTICSEARCH
+	{ BifEnum::Log::WRITER_ASCII, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
+#endif
+
 #ifdef USE_DATASERIES
 	{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
 #endif
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
new file mode 100644
index 0000000000..eb83f26542
--- /dev/null
+++ b/src/logging/writers/ElasticSearch.cc
@@ -0,0 +1,242 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "config.h"
+
+#ifdef USE_ELASTICSEARCH
+
+#include <string>
+#include <errno.h>
+
+#include "util.h"
+
+#include "NetVar.h"
+#include "threading/SerialTypes.h"
+
+#include "ElasticSearch.h"
+
+using namespace logging;
+using namespace writer;
+using threading::Value;
+using threading::Field;
+
+#define MAX_EVENT_SIZE 1024
+
+ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
+	{
+	cluster_name_len = BifConst::LogElasticSearch::cluster_name->Len();
+	cluster_name = new char[cluster_name_len];
+	memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len);
+
+	server_host_len = BifConst::LogElasticSearch::server_host->Len();
+	server_host = new char[server_host_len];
+	memcpy(server_host, BifConst::LogElasticSearch::server_host->Bytes(), server_host_len);
+
+	index_name_len = BifConst::LogElasticSearch::index_name->Len();
+	index_name = new char[index_name_len];
+	memcpy(index_name, BifConst::LogElasticSearch::index_name->Bytes(), index_name_len);
+
+	type_prefix_len = BifConst::LogElasticSearch::type_prefix->Len();
+	type_prefix = new char[type_prefix_len];
+	memcpy(type_prefix, BifConst::LogElasticSearch::type_prefix->Bytes(), type_prefix_len);
+
+	server_port = BifConst::LogElasticSearch::server_port;
+	batch_size = BifConst::LogElasticSearch::batch_size;
+
+	buffer = safe_malloc(MAX_EVENT_SIZE * batch_size);
+	current_offset = 0;
+	buffer[current_offset] = "\0";
+	counter = 0;
+	}
+
+ElasticSearch::~ElasticSearch()
+	{
+	delete [] cluster_name;
+	delete [] server_host;
+	delete [] index_name;
+	delete [] type_prefix;
+	delete [] buffer;
+	}
+
+bool ElasticSearch::DoInit(string path, int num_fields, const Field* const * fields)
+	{
+	  //TODO: Determine what, if anything, needs to be done here.
+	return true;
+	}
+
+bool ElasticSearch::DoFlush()
+	{
+	  //TODO: Send flush command to ElasticSearch
+	return true;
+	}
+
+bool ElasticSearch::DoFinish()
+	{
+	return WriterBackend::DoFinish();
+	}
+
+char* ElasticSearch::FormatField(const char* field_name, const char* field_value)
+{
+  char* result = new char[MAX_EVENT_SIZE];
+  strcpy(result, "\"");
+  strcpy(result, field_name);
+  strcpy(result, "\":\"");
+  strcpy(result, field_value);
+  strcpy(result, "\"");
+  return result;
+  
+}
+
+bool ElasticSearch::BatchIndex()
+{
+  file = fopen("/tmp/batch.test", 'w');
+  fwrite(buffer, current_offset, 1, file);
+  fclose(file);
+  file = 0;
+}
+
+char* ElasticSearch::AddFieldToBuffer(Value* val, const Field* field)
+	{
+    	if ( ! val->present )
+		{
+		  return "";
+		}
+
+	switch ( val->type ) {
+
+	case TYPE_BOOL:
+		return FormatField(field->name, val->val.int_val ? "T" : "F");
+
+	case TYPE_INT:
+		return FormatField(field->name, val->val.int_val);
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		return FormatField(field->name, val->val.uint_val);
+
+	case TYPE_PORT:
+		return FormatField(field->name, val->val.port_val.port);
+
+	case TYPE_SUBNET:
+	        return FormatField(field->name, Render(val->val.subnet_val));
+
+	case TYPE_ADDR:
+	        return FormatField(field->name, Render(val->val.addr_val));
+
+	case TYPE_INTERVAL:
+	case TYPE_TIME:	
+	case TYPE_DOUBLE:
+		return FormatField(field->name, val->val.double_val);
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		{
+		int size = val->val.string_val->size();
+		const char* data = val->val.string_val->data();
+
+		if ( ! size )
+			  return "";
+		return FormatField(field->name, val->val.string_val->data());
+		}
+
+	case TYPE_TABLE:
+		{
+		if ( ! val->val.set_val.size )
+		  return "";
+		
+		char* tmp = new char[MAX_EVENT_SIZE];
+		strcpy(tmp, "{");
+		for ( int j = 0; j < val->val.set_val.size; j++ )
+			{
+			  char* result = AddFieldToBuffer(val->val.set_val.vals[j], field);
+			  bool resultSeen = false;
+			  if ( result ){
+			    if ( resultSeen )
+			      strcpy(tmp, ",");
+			    strcpy(tmp, result);
+			  }
+			}
+		return FormatField(field->name, tmp);
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( ! val->val.vector_val.size )
+		  return "";
+		
+		char* tmp = new char[MAX_EVENT_SIZE];
+		strcpy(tmp, "{");
+		for ( int j = 0; j < val->val.vector_val.size; j++ )
+			{
+			  char* result = AddFieldToBuffer(val->val.vector_val.vals[j], field);
+			  bool resultSeen = false;
+			  if ( result ){
+			    if ( resultSeen )
+			      strcpy(tmp, ",");
+			    strcpy(tmp, result);
+			  }
+			}
+		return FormatField(field->name, tmp);
+		}
+
+	default:
+	  return "";
+	}
+
+	}
+
+bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
+			     Value** vals)
+	{
+	  // Our action line looks like:
+	  //   {"index":"$index_name","type":"$type_prefix$path"}\n{
+	  
+	  bool resultSeen = false;
+
+	  for ( int i = 0; i < num_fields; i++ )
+		{
+		char* result = DoWriteOne(vals[i], fields[i]);
+		if ( result ) {
+		  if ( ! resultSeen ) {
+		    strcpy(buffer[current_offset], "{\"index\":\"");
+		    strcat(buffer[current_offset], index_name);
+		    strcat(buffer[current_offset], "\",\"type\":\"");
+		    strcat(buffer[current_offset], type_prefix);
+		    strcat(buffer[current_offset], Path());
+		    strcat(buffer[current_offset], "\"}\n{");
+		    current_offset = strlen(buffer);
+		    resultSeen = true;
+		  }
+		  else {
+		    strcat(buffer[current_offset], ",");
+		    current_offset += 1;
+		  }
+		  strcat(buffer[current_offset], result);
+		  current_offset += strlen(result);
+		}
+	      }
+
+	  if ( resultSeen ) {
+	    strcat(buffer[current_offset], "}\n");
+	    current_offset += 2;
+	    counter += 1;
+	    if ( counter >= batch_size )
+	      BatchIndex();
+	  }
+	return true;
+	}
+
+bool ElasticSearch::DoRotate(string rotated_path, double open, double close, bool terminating)
+	{
+	  //TODO: Determine what, if anything, needs to be done here.
+	return true;
+	}
+
+bool ElasticSearch::DoSetBuf(bool enabled)
+	{
+	// Nothing to do.
+	return true;
+	}
+
+#endif
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
new file mode 100644
index 0000000000..870290a6e0
--- /dev/null
+++ b/src/logging/writers/ElasticSearch.h
@@ -0,0 +1,67 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Log writer for writing to an ElasticSearch database
+
+#ifndef LOGGING_WRITER_ELASTICSEARCH_H
+#define LOGGING_WRITER_ELASTICSEARCH_H
+
+#include "../WriterBackend.h"
+
+namespace logging { namespace writer {
+
+class ElasticSearch : public WriterBackend {
+public:
+	ElasticSearch(WriterFrontend* frontend);
+	~ElasticSearch();
+
+	static WriterBackend* Instantiate(WriterFrontend* frontend)
+		{ return new ElasticSearch(frontend); }
+	static string LogExt();
+
+protected:
+	// Overidden from WriterBackend.
+
+	virtual bool DoInit(string path, int num_fields,
+			    const threading::Field* const * fields);
+
+	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
+			     threading::Value** vals);
+	virtual bool DoSetBuf(bool enabled);
+	virtual bool DoRotate(string rotated_path, double open,
+			      double close, bool terminating);
+	virtual bool DoFlush();
+	virtual bool DoFinish();
+
+private:
+	char* AddFieldToBuffer(threading::Value* val, const threading::Field* field);
+	char* FormatField(const char* field_name, const char* field_value);
+	bool BatchIndex();
+	
+	char* buffer;
+	int current_offset;
+	int counter;
+
+	// From scripts
+	char* cluster_name;
+	int cluster_name_len;
+
+	char* server_host;
+	int server_host_len;
+
+	uint64 server_port;
+
+	char* index_name;
+	int index_name_len;
+
+	char* type_prefix;
+	int type_prefix_len;
+	
+	uint64 batch_size;
+
+};
+
+}
+}
+
+
+#endif

From 7bee0b0d8e91d1b8c09934c671ebb7cd9607cdca Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 22:07:20 -0400
Subject: [PATCH 003/238] Added sending messages to ElasticSearch over HTTP.

---
 CMakeLists.txt                                |  10 +
 config.h.in                                   |   3 +
 configure                                     |   5 +
 scripts/base/frameworks/logging/__load__.bro  |   1 +
 .../logging/writers/elasticsearch.bro         |  25 ++
 src/CMakeLists.txt                            |   1 +
 src/logging.bif                               |  11 +
 src/logging/Manager.cc                        |   6 +-
 src/logging/writers/ElasticSearch.cc          | 317 +++++++++++-------
 src/logging/writers/ElasticSearch.h           |  13 +-
 src/types.bif                                 |   1 +
 11 files changed, 266 insertions(+), 127 deletions(-)
 create mode 100644 scripts/base/frameworks/logging/writers/elasticsearch.bro

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 28b702ab01..404cdfeeb5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -122,6 +122,14 @@ if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
 	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
 endif()
 
+set(USE_LIBCURL false)
+find_package(CURL)
+if (CURL_FOUND)
+  set(USE_LIBCURL true)
+  include_directories(BEFORE ${CURL_INCLUDE_DIR})
+  list(APPEND OPTLIBS ${CURL_LIBRARIES})
+endif()
+
 if (ENABLE_PERFTOOLS_DEBUG)
     # Just a no op to prevent CMake from complaining about manually-specified
     # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
@@ -209,11 +217,13 @@ message(
     "\nBroccoli:          ${INSTALL_BROCCOLI}"
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
+    "\nElasticSearch:     ${INSTALL_ELASTICSEARCH}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
     "\nGoogle perftools:  ${USE_PERFTOOLS}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
     "\nDataSeries:        ${USE_DATASERIES}"
+    "\nlibCURL:           ${USE_LIBCURL}"
     "\n"
     "\n================================================================\n"
 )
diff --git a/config.h.in b/config.h.in
index c2cb3ec1dc..66121cefbf 100644
--- a/config.h.in
+++ b/config.h.in
@@ -117,6 +117,9 @@
 /* Use the DataSeries writer. */
 #cmakedefine USE_DATASERIES
 
+/* Build the ElasticSearch writer. */
+#cmakedefine INSTALL_ELASTICSEARCH
+
 /* Version number of package */
 #define VERSION "@VERSION@"
 
diff --git a/configure b/configure
index 3258d4abfc..7ea5613a6d 100755
--- a/configure
+++ b/configure
@@ -35,6 +35,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-auxtools     don't build or install auxiliary tools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
+    --enable-elasticsearch build the elasticsearch writer
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -98,6 +99,7 @@ append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
+append_cache_entry INSTALL_ELASTICSEARCH BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
@@ -156,6 +158,9 @@ while [ $# -ne 0 ]; do
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS    BOOL   false
             ;;
+	--enable-elasticsearch)
+	    append_cache_entry INSTALL_ELASTICSEARCH BOOL true
+	    ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;
diff --git a/scripts/base/frameworks/logging/__load__.bro b/scripts/base/frameworks/logging/__load__.bro
index 17e03e2ef7..7dafc45397 100644
--- a/scripts/base/frameworks/logging/__load__.bro
+++ b/scripts/base/frameworks/logging/__load__.bro
@@ -2,3 +2,4 @@
 @load ./postprocessors
 @load ./writers/ascii
 @load ./writers/dataseries
+@load ./writers/elasticsearch
\ No newline at end of file
diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
new file mode 100644
index 0000000000..82dbcc43d4
--- /dev/null
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -0,0 +1,25 @@
+module LogElasticSearch;
+
+export {
+        ## Name of the ES cluster
+	const cluster_name = "elasticsearch" &redef;
+
+	## ES Server
+	const server_host = "127.0.0.1" &redef;
+
+	## ES Port
+	const server_port = 9200 &redef;
+
+	## Name of the ES index
+	const index_name = "bro-logs" &redef;
+
+	## The ES type prefix comes before the name of the related log.
+	## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.
+	const type_prefix = "" &redef;
+
+	## The batch size is the number of messages that will be queued up before 
+	## they are sent to be bulk indexed.
+	## Note: this is mainly a memory usage parameter.
+	const batch_size = 10000 &redef;
+}
+
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 6a68d1e7c5..fbbb01fd22 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -419,6 +419,7 @@ set(bro_SRCS
     logging/WriterFrontend.cc
     logging/writers/Ascii.cc
     logging/writers/DataSeries.cc
+    logging/writers/ElasticSearch.cc
     logging/writers/None.cc
 
     input/Manager.cc
diff --git a/src/logging.bif b/src/logging.bif
index efc6ed0b4b..308ea78b7a 100644
--- a/src/logging.bif
+++ b/src/logging.bif
@@ -81,3 +81,14 @@ const extent_size: count;
 const dump_schema: bool;
 const use_integer_for_time: bool;
 const num_threads: count;
+
+# Options for the ElasticSearch writer.
+
+module LogElasticSearch;
+
+const cluster_name: string;
+const server_host: string;
+const server_port: count;
+const index_name: string;
+const type_prefix: string;
+const batch_size: count;
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index d338ac97f8..ddfed0f70f 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -17,7 +17,7 @@
 #include "writers/Ascii.h"
 #include "writers/None.h"
 
-#ifdef USE_ELASTICSEARCH
+#ifdef INSTALL_ELASTICSEARCH
 #include "writers/ElasticSearch.h"
 #endif
 
@@ -40,8 +40,8 @@ WriterDefinition log_writers[] = {
 	{ BifEnum::Log::WRITER_NONE,  "None", 0, writer::None::Instantiate },
 	{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
 
-#ifdef USE_ELASTICSEARCH
-	{ BifEnum::Log::WRITER_ASCII, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
+#ifdef INSTALL_ELASTICSEARCH
+	{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
 #endif
 
 #ifdef USE_DATASERIES
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index eb83f26542..61f3734f87 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -2,7 +2,7 @@
 
 #include "config.h"
 
-#ifdef USE_ELASTICSEARCH
+#ifdef INSTALL_ELASTICSEARCH
 
 #include <string>
 #include <errno.h>
@@ -12,6 +12,9 @@
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
 
+#include <curl/curl.h>
+#include <curl/easy.h>
+
 #include "ElasticSearch.h"
 
 using namespace logging;
@@ -24,28 +27,35 @@ using threading::Field;
 ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	{
 	cluster_name_len = BifConst::LogElasticSearch::cluster_name->Len();
-	cluster_name = new char[cluster_name_len];
+	cluster_name = new char[cluster_name_len + 1];
 	memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len);
+	cluster_name[cluster_name_len] = 0;
 
 	server_host_len = BifConst::LogElasticSearch::server_host->Len();
-	server_host = new char[server_host_len];
+	server_host = new char[server_host_len + 1];
 	memcpy(server_host, BifConst::LogElasticSearch::server_host->Bytes(), server_host_len);
+	server_host[server_host_len] = 0;
 
 	index_name_len = BifConst::LogElasticSearch::index_name->Len();
-	index_name = new char[index_name_len];
+	index_name = new char[index_name_len + 1];
 	memcpy(index_name, BifConst::LogElasticSearch::index_name->Bytes(), index_name_len);
+	index_name[index_name_len] = 0;
 
 	type_prefix_len = BifConst::LogElasticSearch::type_prefix->Len();
-	type_prefix = new char[type_prefix_len];
+	type_prefix = new char[type_prefix_len + 1];
 	memcpy(type_prefix, BifConst::LogElasticSearch::type_prefix->Bytes(), type_prefix_len);
+	type_prefix[type_prefix_len] = 0;
 
 	server_port = BifConst::LogElasticSearch::server_port;
 	batch_size = BifConst::LogElasticSearch::batch_size;
 
-	buffer = safe_malloc(MAX_EVENT_SIZE * batch_size);
+	buffer = (char *)safe_malloc(MAX_EVENT_SIZE * batch_size);
 	current_offset = 0;
-	buffer[current_offset] = "\0";
+	buffer[current_offset] = 0;
 	counter = 0;
+
+	curl_handle = HTTPSetup();
+	curl_result = new char[1024];
 	}
 
 ElasticSearch::~ElasticSearch()
@@ -74,115 +84,129 @@ bool ElasticSearch::DoFinish()
 	return WriterBackend::DoFinish();
 	}
 
-char* ElasticSearch::FormatField(const char* field_name, const char* field_value)
-{
-  char* result = new char[MAX_EVENT_SIZE];
-  strcpy(result, "\"");
-  strcpy(result, field_name);
-  strcpy(result, "\":\"");
-  strcpy(result, field_value);
-  strcpy(result, "\"");
-  return result;
-  
-}
-
 bool ElasticSearch::BatchIndex()
 {
-  file = fopen("/tmp/batch.test", 'w');
-  fwrite(buffer, current_offset, 1, file);
-  fclose(file);
-  file = 0;
+  return HTTPSend();
+}
+
+char* ElasticSearch::FieldToString(Value* val, const Field* field)
+{
+  char* result = new char[MAX_EVENT_SIZE];
+
+  switch ( val->type ) {
+
+    // ElasticSearch defines bools as: 0 == false, everything else == true. So we treat it as an int.
+  case TYPE_BOOL:
+  case TYPE_INT:
+    sprintf(result, "%d", (int) val->val.int_val); return result;
+
+  case TYPE_COUNT:
+  case TYPE_COUNTER:
+    sprintf(result, "%d", (int) val->val.uint_val); return result;
+    
+  case TYPE_PORT:
+    sprintf(result, "%d", (int) val->val.port_val.port); return result;
+
+  case TYPE_SUBNET:
+    sprintf(result, "\"%s\"", Render(val->val.subnet_val).c_str()); return result;
+    
+  case TYPE_ADDR:
+    sprintf(result, "\"%s\"", Render(val->val.addr_val).c_str()); return result;
+
+  case TYPE_INTERVAL:
+  case TYPE_TIME:	
+    sprintf(result, "\"%d\"", (int) (val->val.double_val * 1000)); return result;
+  case TYPE_DOUBLE:
+    sprintf(result, "\"%s\"", Render(val->val.double_val).c_str()); return result;
+
+  case TYPE_ENUM:
+  case TYPE_STRING:
+  case TYPE_FILE:
+  case TYPE_FUNC:
+    {
+      int size = val->val.string_val->size();
+      const char* data = val->val.string_val->data();
+      
+      if ( ! size )
+	return 0;
+      sprintf(result, "\"%s\"", data); return result;
+    }
+
+  case TYPE_TABLE:
+    {
+      char* tmp = new char[MAX_EVENT_SIZE];
+      int tmp_offset = 0;
+      strcpy(tmp, "{");
+      tmp_offset = 1;
+      bool result_seen = false;
+      for ( int j = 0; j < val->val.set_val.size; j++ )
+	{
+	  char* sub_field = FieldToString(val->val.set_val.vals[j], field);
+	  if ( sub_field ){
+	    
+	    if ( result_seen ){
+	      strcpy(tmp + tmp_offset, ",");
+	      tmp_offset += 1;
+	    }
+	    else
+	      result_seen = true;
+	    
+	    sprintf(tmp + tmp_offset, "\"%s\":%s", field->name.c_str(), sub_field);
+	    tmp_offset = strlen(tmp);
+	  }
+	}
+      strcpy(tmp + tmp_offset, "}");
+      tmp_offset += 1;
+      sprintf(result, "%s", tmp); 
+      return result;
+    }
+    
+  case TYPE_VECTOR:
+    {
+      char* tmp = new char[MAX_EVENT_SIZE];
+      int tmp_offset = 0;
+      strcpy(tmp, "{");
+      tmp_offset = 1;
+      bool result_seen = false;
+      for ( int j = 0; j < val->val.vector_val.size; j++ )
+	{
+	  char* sub_field = FieldToString(val->val.vector_val.vals[j], field);
+	  if ( sub_field ){
+	    
+	    if ( result_seen ){
+	      strcpy(tmp + tmp_offset, ",");
+	      tmp_offset += 1;
+	    }
+	    else
+	      result_seen = true;
+	    
+	    sprintf(tmp + tmp_offset, "\"%s\":%s", field->name.c_str(), sub_field);
+	    tmp_offset = strlen(tmp);
+	  }
+	}
+      strcpy(tmp + tmp_offset, "}");
+      tmp_offset += 1;
+      sprintf(result, "%s", tmp); 
+      return result;
+    }
+
+  default:
+    {
+      return (char *)"{}";
+    }
+
+  }
+
 }
 
 char* ElasticSearch::AddFieldToBuffer(Value* val, const Field* field)
 	{
     	if ( ! val->present )
-		{
-		  return "";
-		}
-
-	switch ( val->type ) {
-
-	case TYPE_BOOL:
-		return FormatField(field->name, val->val.int_val ? "T" : "F");
-
-	case TYPE_INT:
-		return FormatField(field->name, val->val.int_val);
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		return FormatField(field->name, val->val.uint_val);
-
-	case TYPE_PORT:
-		return FormatField(field->name, val->val.port_val.port);
-
-	case TYPE_SUBNET:
-	        return FormatField(field->name, Render(val->val.subnet_val));
-
-	case TYPE_ADDR:
-	        return FormatField(field->name, Render(val->val.addr_val));
-
-	case TYPE_INTERVAL:
-	case TYPE_TIME:	
-	case TYPE_DOUBLE:
-		return FormatField(field->name, val->val.double_val);
-
-	case TYPE_ENUM:
-	case TYPE_STRING:
-	case TYPE_FILE:
-	case TYPE_FUNC:
-		{
-		int size = val->val.string_val->size();
-		const char* data = val->val.string_val->data();
-
-		if ( ! size )
-			  return "";
-		return FormatField(field->name, val->val.string_val->data());
-		}
-
-	case TYPE_TABLE:
-		{
-		if ( ! val->val.set_val.size )
-		  return "";
-		
-		char* tmp = new char[MAX_EVENT_SIZE];
-		strcpy(tmp, "{");
-		for ( int j = 0; j < val->val.set_val.size; j++ )
-			{
-			  char* result = AddFieldToBuffer(val->val.set_val.vals[j], field);
-			  bool resultSeen = false;
-			  if ( result ){
-			    if ( resultSeen )
-			      strcpy(tmp, ",");
-			    strcpy(tmp, result);
-			  }
-			}
-		return FormatField(field->name, tmp);
-		}
-
-	case TYPE_VECTOR:
-		{
-		if ( ! val->val.vector_val.size )
-		  return "";
-		
-		char* tmp = new char[MAX_EVENT_SIZE];
-		strcpy(tmp, "{");
-		for ( int j = 0; j < val->val.vector_val.size; j++ )
-			{
-			  char* result = AddFieldToBuffer(val->val.vector_val.vals[j], field);
-			  bool resultSeen = false;
-			  if ( result ){
-			    if ( resultSeen )
-			      strcpy(tmp, ",");
-			    strcpy(tmp, result);
-			  }
-			}
-		return FormatField(field->name, tmp);
-		}
-
-	default:
-	  return "";
-	}
+	  return 0;
+	
+	char* result = new char[MAX_EVENT_SIZE];
+	sprintf(result, "\"%s\":%s", field->name.c_str(), FieldToString(val, field));
+	return result;
 
 	}
 
@@ -190,39 +214,37 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
 	  // Our action line looks like:
-	  //   {"index":"$index_name","type":"$type_prefix$path"}\n{
+	  //   {"index":{"_index":"$index_name","_type":"$type_prefix$path"}}\n{
 	  
 	  bool resultSeen = false;
 
 	  for ( int i = 0; i < num_fields; i++ )
 		{
-		char* result = DoWriteOne(vals[i], fields[i]);
+		char* result = AddFieldToBuffer(vals[i], fields[i]);
 		if ( result ) {
 		  if ( ! resultSeen ) {
-		    strcpy(buffer[current_offset], "{\"index\":\"");
-		    strcat(buffer[current_offset], index_name);
-		    strcat(buffer[current_offset], "\",\"type\":\"");
-		    strcat(buffer[current_offset], type_prefix);
-		    strcat(buffer[current_offset], Path());
-		    strcat(buffer[current_offset], "\"}\n{");
-		    current_offset = strlen(buffer);
+		    current_offset += sprintf(buffer + current_offset, "{\"index\":{\"_index\":\"%s\",\"_type\":\"%s%s\"}\n{", index_name, type_prefix, Path().c_str());
 		    resultSeen = true;
 		  }
 		  else {
-		    strcat(buffer[current_offset], ",");
+		    strcat(buffer, ",");
 		    current_offset += 1;
 		  }
-		  strcat(buffer[current_offset], result);
+		  strcat(buffer, result);
 		  current_offset += strlen(result);
 		}
 	      }
 
 	  if ( resultSeen ) {
-	    strcat(buffer[current_offset], "}\n");
+	    strcat(buffer, "}\n");
 	    current_offset += 2;
 	    counter += 1;
-	    if ( counter >= batch_size )
+	    if ( counter >= batch_size ){
 	      BatchIndex();
+	      current_offset = 0;
+	      buffer[current_offset] = 0;
+	      counter = 0;
+	    }
 	  }
 	return true;
 	}
@@ -239,4 +261,55 @@ bool ElasticSearch::DoSetBuf(bool enabled)
 	return true;
 	}
 
+// HTTP Functions start here.
+
+CURL* ElasticSearch::HTTPSetup()
+{
+  char URL[2048];
+  CURL* handle;
+  struct curl_slist *headers=NULL;
+  
+  handle = curl_easy_init();
+  if ( ! handle )
+    return handle;
+  
+  sprintf(URL, "http://%s:%d/_bulk", server_host, (int) server_port);
+  curl_easy_setopt(handle, CURLOPT_URL, URL);
+
+  headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
+  curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
+
+  curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
+  curl_easy_setopt(handle, CURLOPT_POST, 1); // All requests are POSTs
+
+  // HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. The best (only?) way to disable that is to
+  // just use HTTP 1.0
+  curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+  return handle;
+  
+}
+
+bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata){
+  //TODO: Do some verification on the result?
+  return true;
+}
+
+bool ElasticSearch::HTTPSend(){
+  CURLcode return_code;
+  
+  curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, curl_result);
+  curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer);
+  curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, current_offset);
+  
+  return_code = curl_easy_perform(curl_handle);
+  switch(return_code) {
+  case CURLE_COULDNT_CONNECT:
+  case CURLE_COULDNT_RESOLVE_HOST:
+  case CURLE_WRITE_ERROR:
+    return false;
+  default:
+    return true;
+  }
+}
+
 #endif
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index 870290a6e0..ad3729f6da 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -5,6 +5,7 @@
 #ifndef LOGGING_WRITER_ELASTICSEARCH_H
 #define LOGGING_WRITER_ELASTICSEARCH_H
 
+#include <curl/curl.h>
 #include "../WriterBackend.h"
 
 namespace logging { namespace writer {
@@ -34,12 +35,20 @@ protected:
 
 private:
 	char* AddFieldToBuffer(threading::Value* val, const threading::Field* field);
-	char* FormatField(const char* field_name, const char* field_value);
+	char* FieldToString(threading::Value* val, const threading::Field* field);
 	bool BatchIndex();
 	
+	CURL* HTTPSetup();
+	bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
+	bool HTTPSend();
+	
+	// Buffers, etc.
 	char* buffer;
 	int current_offset;
-	int counter;
+	uint64 counter;
+
+	CURL* curl_handle;
+	char* curl_result;
 
 	// From scripts
 	char* cluster_name;
diff --git a/src/types.bif b/src/types.bif
index 76bac3e0e2..9b387b2c52 100644
--- a/src/types.bif
+++ b/src/types.bif
@@ -163,6 +163,7 @@ enum Writer %{
 	WRITER_NONE,
 	WRITER_ASCII,
 	WRITER_DATASERIES,
+	WRITER_ELASTICSEARCH,
 %}
 
 enum ID %{

From c8e770a499b3b9457bcb9908f574c4a7b4a9ddae Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 22:51:21 -0400
Subject: [PATCH 004/238] Some basic documentation in
 doc/logging-elasticsearch.rst

---
 doc/logging-elasticsearch.rst | 97 +++++++++++++++++++++++++++++++++++
 1 file changed, 97 insertions(+)
 create mode 100644 doc/logging-elasticsearch.rst

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
new file mode 100644
index 0000000000..a3fb759c85
--- /dev/null
+++ b/doc/logging-elasticsearch.rst
@@ -0,0 +1,97 @@
+
+========================================
+Indexed Logging Output with ElasticSearch
+========================================
+
+.. rst-class:: opening
+
+   Bro's default ASCII log format is not exactly the most efficient
+   way for storing and searching large volumes of data. ElasticSearch
+   is a new and exciting technology for dealing with tons of data.
+   ElasticSearch is a search engine built on top of Apache's Lucene
+   project. It scales very well, both for distributed indexing and 
+   distributed searching.
+
+.. contents::
+
+Installing ElasticSearch
+------------------------
+
+ElasticSearch requires a JRE to run. Please download the latest version
+from: <http://www.elasticsearch.org/download/>. Once extracted, start
+ElasticSearch with::
+
+# ./bin/elasticsearch
+
+Compiling Bro with ElasticSearch Support
+----------------------------------------
+
+First, ensure that you have libcurl installed. Secondly, set the 
+``--enable-elasticsearch`` option::
+
+    # ./configure --enable-elasticsearch
+    [...]
+    ====================|  Bro Build Summary  |=====================
+    [...]
+    ElasticSearch:     true
+    [...]
+    libCURL:           true
+    [...]
+    ================================================================
+
+Activating ElasticSearch
+------------------------
+
+The direct way to use ElasticSearch is to switch *all* log files over to
+ElasticSearch. To do that, just add ``redef
+Log::default_writer=Log::WRITER_ELASTICSEARCH;`` to your ``local.bro``.
+For testing, you can also just pass that on the command line::
+
+    bro -r trace.pcap Log::default_writer=Log::WRITER_ELASTICSEARCH
+
+With that, Bro will now write all its output into ElasticSearch. You can 
+inspect these using ElasticSearch's REST-ful interface. For more
+information, see: <http://www.elasticsearch.org/guide/reference/api/>.
+
+There is also a rudimentary web interface to ElasticSearch, available at:
+<http://mobz.github.com/elasticsearch-head/>.
+
+You can also switch only individual files over to ElasticSearch by adding
+code like this to your ``local.bro``::
+
+.. code:: bro
+
+    event bro_init()
+        {
+        local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
+        f$writer = Log::WRITER_ELASTICSEARCH;               # Change writer type.
+        Log::add_filter(Conn::LOG, f);                   # Replace filter with adapted version.
+        }
+
+Configuring ElasticSearch
+-------------------------
+
+Bro's ElasticSearch writer comes with a few configuraiton options::
+
+- cluster_name:: Currently unused.
+- server_host:: Where to send the data. Default localhost.
+- server_port:: What port to send the data to. Default 9200.
+- index_name:: ElasticSearch indexes are like databases in a standard DB model.
+This is the name of the index to which to send the data. Default bro-logs.
+- type_prefix:: ElasticSearch types are like tables in a standard DB model.
+This is a prefix that gets prepended to Bro log names.
+Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc.
+Default: none.
+- batch_size:: How many messages to buffer before sending to ElasticSearch.
+This is mainly a memory optimization - changing this doesn't seem to affect 
+indexing performance that much. Default: 10,000.
+
+TODO
+----
+
+Lots.
+
+- Perform multicast discovery for server.
+- Better error detection.
+- Dynamic index names.
+- Better defaults (don't index loaded-plugins, for instance).

From d09fc15b2d376da9bfab3c5b45eff0049589f217 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 22:54:19 -0400
Subject: [PATCH 005/238] Minor documentation formatting change

---
 doc/logging-elasticsearch.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index a3fb759c85..6c490a8b23 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -59,7 +59,7 @@ There is also a rudimentary web interface to ElasticSearch, available at:
 You can also switch only individual files over to ElasticSearch by adding
 code like this to your ``local.bro``::
 
-.. code:: bro
+.. code::bro
 
     event bro_init()
         {

From b3216906fe53e807070d6ce8587a7a01f9fea4f1 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 22:56:38 -0400
Subject: [PATCH 006/238] Minor documentation formatting change

---
 doc/logging-elasticsearch.rst | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index 6c490a8b23..7a16acb0f1 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -77,14 +77,14 @@ Bro's ElasticSearch writer comes with a few configuraiton options::
 - server_host:: Where to send the data. Default localhost.
 - server_port:: What port to send the data to. Default 9200.
 - index_name:: ElasticSearch indexes are like databases in a standard DB model.
-This is the name of the index to which to send the data. Default bro-logs.
+  This is the name of the index to which to send the data. Default bro-logs.
 - type_prefix:: ElasticSearch types are like tables in a standard DB model.
-This is a prefix that gets prepended to Bro log names.
-Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc.
-Default: none.
+  This is a prefix that gets prepended to Bro log names.
+  Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc.
+  Default: none.
 - batch_size:: How many messages to buffer before sending to ElasticSearch.
-This is mainly a memory optimization - changing this doesn't seem to affect 
-indexing performance that much. Default: 10,000.
+  This is mainly a memory optimization - changing this doesn't seem to affect 
+  indexing performance that much. Default: 10,000.
 
 TODO
 ----

From 360d7e2eda7e2a4b14f43021e197f6b65b28328e Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 22:59:29 -0400
Subject: [PATCH 007/238] Minor documentation formatting change

---
 doc/logging-elasticsearch.rst | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index 7a16acb0f1..4229748854 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -73,18 +73,12 @@ Configuring ElasticSearch
 
 Bro's ElasticSearch writer comes with a few configuraiton options::
 
-- cluster_name:: Currently unused.
-- server_host:: Where to send the data. Default localhost.
-- server_port:: What port to send the data to. Default 9200.
-- index_name:: ElasticSearch indexes are like databases in a standard DB model.
-  This is the name of the index to which to send the data. Default bro-logs.
-- type_prefix:: ElasticSearch types are like tables in a standard DB model.
-  This is a prefix that gets prepended to Bro log names.
-  Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc.
-  Default: none.
-- batch_size:: How many messages to buffer before sending to ElasticSearch.
-  This is mainly a memory optimization - changing this doesn't seem to affect 
-  indexing performance that much. Default: 10,000.
+- cluster_name: Currently unused.
+- server_host:  Where to send the data. Default localhost.
+- server_port:  What port to send the data to. Default 9200.
+- index_name:   ElasticSearch indexes are like databases in a standard DB model. This is the name of the index to which to send the data. Default bro-logs.
+- type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
+- batch_size:   How many messages to buffer before sending to ElasticSearch. This is mainly a memory optimization - changing this doesn't seem to affect indexing performance that much. Default: 10,000.
 
 TODO
 ----

From 3d8b86c00a7d5cb4a4dd52ef08e6d06d42ee88a3 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 23:00:10 -0400
Subject: [PATCH 008/238] Minor documentation formatting change

---
 doc/logging-elasticsearch.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index 4229748854..3b630c4bff 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -76,7 +76,8 @@ Bro's ElasticSearch writer comes with a few configuraiton options::
 - cluster_name: Currently unused.
 - server_host:  Where to send the data. Default localhost.
 - server_port:  What port to send the data to. Default 9200.
-- index_name:   ElasticSearch indexes are like databases in a standard DB model. This is the name of the index to which to send the data. Default bro-logs.
+- index_name:   ElasticSearch indexes are like databases in a standard DB model. 
+  This is the name of the index to which to send the data. Default bro-logs.
 - type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
 - batch_size:   How many messages to buffer before sending to ElasticSearch. This is mainly a memory optimization - changing this doesn't seem to affect indexing performance that much. Default: 10,000.
 

From bf852b51f5bac6ba6b0dd14e4f6ab5fb73b68195 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 23:01:18 -0400
Subject: [PATCH 009/238] Minor documentation formatting change

---
 doc/logging-elasticsearch.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index 3b630c4bff..f891212ccd 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -76,8 +76,10 @@ Bro's ElasticSearch writer comes with a few configuraiton options::
 - cluster_name: Currently unused.
 - server_host:  Where to send the data. Default localhost.
 - server_port:  What port to send the data to. Default 9200.
+
 - index_name:   ElasticSearch indexes are like databases in a standard DB model. 
   This is the name of the index to which to send the data. Default bro-logs.
+
 - type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
 - batch_size:   How many messages to buffer before sending to ElasticSearch. This is mainly a memory optimization - changing this doesn't seem to affect indexing performance that much. Default: 10,000.
 

From 5915a2d304fd4d90f799057bfeb7425ee29e95ef Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Sun, 3 Jun 2012 23:02:07 -0400
Subject: [PATCH 010/238] Minor documentation formatting change

---
 doc/logging-elasticsearch.rst | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index f891212ccd..b3cf062de4 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -71,16 +71,19 @@ code like this to your ``local.bro``::
 Configuring ElasticSearch
 -------------------------
 
-Bro's ElasticSearch writer comes with a few configuraiton options::
+Bro's ElasticSearch writer comes with a few configuration options::
 
 - cluster_name: Currently unused.
+
 - server_host:  Where to send the data. Default localhost.
+
 - server_port:  What port to send the data to. Default 9200.
 
 - index_name:   ElasticSearch indexes are like databases in a standard DB model. 
   This is the name of the index to which to send the data. Default bro-logs.
 
 - type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
+
 - batch_size:   How many messages to buffer before sending to ElasticSearch. This is mainly a memory optimization - changing this doesn't seem to affect indexing performance that much. Default: 10,000.
 
 TODO

From 5db027e39f35c189130628bc03b0417dad951f17 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Tue, 5 Jun 2012 17:23:50 -0400
Subject: [PATCH 011/238] Fix timestamp overflow bug.

---
 src/logging/writers/ElasticSearch.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 61f3734f87..a073ea7e7c 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -115,7 +115,7 @@ char* ElasticSearch::FieldToString(Value* val, const Field* field)
 
   case TYPE_INTERVAL:
   case TYPE_TIME:	
-    sprintf(result, "\"%d\"", (int) (val->val.double_val * 1000)); return result;
+    sprintf(result, "\"%llu\"", (unsigned long long) (val->val.double_val * 1000)); return result;
   case TYPE_DOUBLE:
     sprintf(result, "\"%s\"", Render(val->val.double_val).c_str()); return result;
 

From a3b330dbc6ddb0502a004e37a8977d22395d8ef9 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Tue, 5 Jun 2012 17:24:13 -0400
Subject: [PATCH 012/238] Make default index name 'bro'.

---
 doc/logging-elasticsearch.rst                             | 2 +-
 scripts/base/frameworks/logging/writers/elasticsearch.bro | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index b3cf062de4..4fce470d4a 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -80,7 +80,7 @@ Bro's ElasticSearch writer comes with a few configuration options::
 - server_port:  What port to send the data to. Default 9200.
 
 - index_name:   ElasticSearch indexes are like databases in a standard DB model. 
-  This is the name of the index to which to send the data. Default bro-logs.
+  This is the name of the index to which to send the data. Default bro.
 
 - type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
 
diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index 82dbcc43d4..7f968d0042 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -11,7 +11,7 @@ export {
 	const server_port = 9200 &redef;
 
 	## Name of the ES index
-	const index_name = "bro-logs" &redef;
+	const index_name = "bro" &redef;
 
 	## The ES type prefix comes before the name of the related log.
 	## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.

From 894dec006909477bc9bae177fde6a739bdf73173 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Tue, 5 Jun 2012 23:01:36 -0400
Subject: [PATCH 013/238] No quotes for times and doubles, since ES won't
 interpret them as numbers then.

---
 src/logging/writers/ElasticSearch.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index a073ea7e7c..3a7635a6a2 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -115,9 +115,9 @@ char* ElasticSearch::FieldToString(Value* val, const Field* field)
 
   case TYPE_INTERVAL:
   case TYPE_TIME:	
-    sprintf(result, "\"%llu\"", (unsigned long long) (val->val.double_val * 1000)); return result;
+    sprintf(result, "%llu", (unsigned long long) (val->val.double_val * 1000)); return result;
   case TYPE_DOUBLE:
-    sprintf(result, "\"%s\"", Render(val->val.double_val).c_str()); return result;
+    sprintf(result, "%s", Render(val->val.double_val).c_str()); return result;
 
   case TYPE_ENUM:
   case TYPE_STRING:

From 5e05e548ff0e5b640a448c29d6c64f1c378bdb10 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Tue, 5 Jun 2012 23:20:28 -0400
Subject: [PATCH 014/238] Change time printf format to use the more compatible
 PRIu64.

---
 src/logging/writers/ElasticSearch.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 3a7635a6a2..1817ce63ef 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -115,7 +115,7 @@ char* ElasticSearch::FieldToString(Value* val, const Field* field)
 
   case TYPE_INTERVAL:
   case TYPE_TIME:	
-    sprintf(result, "%llu", (unsigned long long) (val->val.double_val * 1000)); return result;
+    sprintf(result, "%"PRIu64"", (uint64) (val->val.double_val * 1000)); return result;
   case TYPE_DOUBLE:
     sprintf(result, "%s", Render(val->val.double_val).c_str()); return result;
 

From 22efa452796fff41e3ebf91d0e241f21bb7ae8e7 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Fri, 15 Jun 2012 10:48:22 -0400
Subject: [PATCH 015/238] Merging in latest changes from Bro master.

---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 cmake        | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 6f43a8115d..b4094cb75e 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 6f43a8115d8e6483a50957c5d21c5d69270ab3aa
+Subproject commit b4094cb75e0a7769123f7db1f5d73f3f9f1c3977
diff --git a/aux/bro-aux b/aux/bro-aux
index c6391412e9..2038e3de04 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit c6391412e902e896836450ab98910309b2ca2d9b
+Subproject commit 2038e3de042115c3caa706426e16c830c1fd1e9e
diff --git a/aux/broccoli b/aux/broccoli
index 0d139c09d5..4e17842743 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 0d139c09d5a9c8623ecc2a5f395178f0ddcd7e16
+Subproject commit 4e17842743fef8df6abf0588c7ca86c6937a2b6d
diff --git a/aux/broctl b/aux/broctl
index 880f3e48d3..892b60edb9 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 880f3e48d33bb28d17184656f858a4a0e2e1574c
+Subproject commit 892b60edb967bb456872638f22ba994e84530137
diff --git a/cmake b/cmake
index 2a72c5e08e..96f3d92aca 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130
+Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b

From d3bb4617e96a8ec725e8d103b35813ff5d48f58a Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Fri, 15 Jun 2012 11:21:24 -0400
Subject: [PATCH 016/238] Configuration logic - if libcurl is found, enable
 elasticsearch plugin.

---
 CMakeLists.txt | 1 +
 configure      | 4 ----
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 404cdfeeb5..4b1cccf8dc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -128,6 +128,7 @@ if (CURL_FOUND)
   set(USE_LIBCURL true)
   include_directories(BEFORE ${CURL_INCLUDE_DIR})
   list(APPEND OPTLIBS ${CURL_LIBRARIES})
+  set(INSTALL_ELASTICSEARCH true)
 endif()
 
 if (ENABLE_PERFTOOLS_DEBUG)
diff --git a/configure b/configure
index 7ea5613a6d..801fb1e801 100755
--- a/configure
+++ b/configure
@@ -35,7 +35,6 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-auxtools     don't build or install auxiliary tools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
-    --enable-elasticsearch build the elasticsearch writer
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -158,9 +157,6 @@ while [ $# -ne 0 ]; do
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS    BOOL   false
             ;;
-	--enable-elasticsearch)
-	    append_cache_entry INSTALL_ELASTICSEARCH BOOL true
-	    ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;

From 0bb8b69c95191b7e25296770010b201f67f9cc9c Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 15 Jun 2012 16:30:54 -0400
Subject: [PATCH 017/238] Reworked bulk operation string construction to use
 ODesc and added json escaping.

---
 src/logging/writers/ElasticSearch.cc | 410 ++++++++++++---------------
 src/logging/writers/ElasticSearch.h  |  20 +-
 2 files changed, 187 insertions(+), 243 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 1817ce63ef..d8c4bee306 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -8,6 +8,7 @@
 #include <errno.h>
 
 #include "util.h"
+#include "BroString.h"
 
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
@@ -22,38 +23,16 @@ using namespace writer;
 using threading::Value;
 using threading::Field;
 
-#define MAX_EVENT_SIZE 1024
-
 ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	{
 	cluster_name_len = BifConst::LogElasticSearch::cluster_name->Len();
 	cluster_name = new char[cluster_name_len + 1];
 	memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len);
 	cluster_name[cluster_name_len] = 0;
-
-	server_host_len = BifConst::LogElasticSearch::server_host->Len();
-	server_host = new char[server_host_len + 1];
-	memcpy(server_host, BifConst::LogElasticSearch::server_host->Bytes(), server_host_len);
-	server_host[server_host_len] = 0;
-
-	index_name_len = BifConst::LogElasticSearch::index_name->Len();
-	index_name = new char[index_name_len + 1];
-	memcpy(index_name, BifConst::LogElasticSearch::index_name->Bytes(), index_name_len);
-	index_name[index_name_len] = 0;
-
-	type_prefix_len = BifConst::LogElasticSearch::type_prefix->Len();
-	type_prefix = new char[type_prefix_len + 1];
-	memcpy(type_prefix, BifConst::LogElasticSearch::type_prefix->Bytes(), type_prefix_len);
-	type_prefix[type_prefix_len] = 0;
-
-	server_port = BifConst::LogElasticSearch::server_port;
-	batch_size = BifConst::LogElasticSearch::batch_size;
-
-	buffer = (char *)safe_malloc(MAX_EVENT_SIZE * batch_size);
-	current_offset = 0;
-	buffer[current_offset] = 0;
+	
+	buffer.Clear();
 	counter = 0;
-
+	
 	curl_handle = HTTPSetup();
 	curl_result = new char[1024];
 	}
@@ -61,21 +40,17 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 ElasticSearch::~ElasticSearch()
 	{
 	delete [] cluster_name;
-	delete [] server_host;
-	delete [] index_name;
-	delete [] type_prefix;
-	delete [] buffer;
 	}
 
 bool ElasticSearch::DoInit(string path, int num_fields, const Field* const * fields)
 	{
-	  //TODO: Determine what, if anything, needs to be done here.
+	//TODO: Determine what, if anything, needs to be done here.
 	return true;
 	}
 
 bool ElasticSearch::DoFlush()
 	{
-	  //TODO: Send flush command to ElasticSearch
+	//TODO: Send flush command to ElasticSearch
 	return true;
 	}
 
@@ -84,174 +59,155 @@ bool ElasticSearch::DoFinish()
 	return WriterBackend::DoFinish();
 	}
 
-bool ElasticSearch::BatchIndex()
-{
-  return HTTPSend();
-}
-
-char* ElasticSearch::FieldToString(Value* val, const Field* field)
-{
-  char* result = new char[MAX_EVENT_SIZE];
-
-  switch ( val->type ) {
-
-    // ElasticSearch defines bools as: 0 == false, everything else == true. So we treat it as an int.
-  case TYPE_BOOL:
-  case TYPE_INT:
-    sprintf(result, "%d", (int) val->val.int_val); return result;
-
-  case TYPE_COUNT:
-  case TYPE_COUNTER:
-    sprintf(result, "%d", (int) val->val.uint_val); return result;
-    
-  case TYPE_PORT:
-    sprintf(result, "%d", (int) val->val.port_val.port); return result;
-
-  case TYPE_SUBNET:
-    sprintf(result, "\"%s\"", Render(val->val.subnet_val).c_str()); return result;
-    
-  case TYPE_ADDR:
-    sprintf(result, "\"%s\"", Render(val->val.addr_val).c_str()); return result;
-
-  case TYPE_INTERVAL:
-  case TYPE_TIME:	
-    sprintf(result, "%"PRIu64"", (uint64) (val->val.double_val * 1000)); return result;
-  case TYPE_DOUBLE:
-    sprintf(result, "%s", Render(val->val.double_val).c_str()); return result;
-
-  case TYPE_ENUM:
-  case TYPE_STRING:
-  case TYPE_FILE:
-  case TYPE_FUNC:
-    {
-      int size = val->val.string_val->size();
-      const char* data = val->val.string_val->data();
-      
-      if ( ! size )
-	return 0;
-      sprintf(result, "\"%s\"", data); return result;
-    }
-
-  case TYPE_TABLE:
-    {
-      char* tmp = new char[MAX_EVENT_SIZE];
-      int tmp_offset = 0;
-      strcpy(tmp, "{");
-      tmp_offset = 1;
-      bool result_seen = false;
-      for ( int j = 0; j < val->val.set_val.size; j++ )
+bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 	{
-	  char* sub_field = FieldToString(val->val.set_val.vals[j], field);
-	  if ( sub_field ){
-	    
-	    if ( result_seen ){
-	      strcpy(tmp + tmp_offset, ",");
-	      tmp_offset += 1;
-	    }
-	    else
-	      result_seen = true;
-	    
-	    sprintf(tmp + tmp_offset, "\"%s\":%s", field->name.c_str(), sub_field);
-	    tmp_offset = strlen(tmp);
-	  }
+	switch ( val->type ) 
+		{
+		// ElasticSearch defines bools as: 0 == false, everything else == true. So we treat it as an int.
+		case TYPE_BOOL:
+		case TYPE_INT:
+			buffer.Add(val->val.int_val);
+			break;
+		
+		case TYPE_COUNT:
+		case TYPE_COUNTER:
+			buffer.Add(val->val.uint_val);
+			break;
+		
+		case TYPE_PORT:
+			buffer.Add(val->val.port_val.port);
+			break;
+		
+		case TYPE_SUBNET:
+			buffer.AddRaw("\"", 1);
+			buffer.Add(Render(val->val.subnet_val));
+			buffer.AddRaw("\"", 1);
+			break;
+		
+		case TYPE_ADDR:
+			buffer.AddRaw("\"", 1);
+			buffer.Add(Render(val->val.addr_val));
+			buffer.AddRaw("\"", 1);
+			break;
+		
+		case TYPE_DOUBLE:
+			buffer.Add(val->val.double_val);
+			break;
+		
+		case TYPE_INTERVAL:
+		case TYPE_TIME:
+			// ElasticSearch uses milliseconds for timestamps
+			buffer.Add((uint64_t) (val->val.double_val * 1000));
+			break;
+		
+		case TYPE_ENUM:
+		case TYPE_STRING:
+		case TYPE_FILE:
+		case TYPE_FUNC:
+			{
+			buffer.AddRaw("\"", 1);
+			for ( uint i = 0; i < val->val.string_val->size(); ++i )
+				{
+				char c = val->val.string_val->data()[i];
+				// HTML entity encode special characters.
+				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' )
+					{
+					buffer.AddRaw("&#", 2);
+					buffer.Add((uint8_t) c);
+					buffer.AddRaw(";", 1);
+					}
+				else
+					buffer.AddRaw(&c, 1);
+				}
+			buffer.AddRaw("\"", 1);
+			break;
+			}
+		
+		case TYPE_TABLE:
+			{
+			buffer.AddRaw("[", 1);
+			for ( int j = 0; j < val->val.set_val.size; j++ )
+				{
+				if ( j > 0 )
+					buffer.AddRaw(",", 1);
+				AddFieldValueToBuffer(val->val.set_val.vals[j], field);
+				}
+			buffer.AddRaw("]", 1);
+			break;
+			}
+			
+		case TYPE_VECTOR:
+			{
+			buffer.AddRaw("[", 1);
+			for ( int j = 0; j < val->val.vector_val.size; j++ )
+				{
+				if ( j > 0 )
+					buffer.AddRaw(",", 1);
+				AddFieldValueToBuffer(val->val.vector_val.vals[j], field);
+				}
+			buffer.AddRaw("]", 1);
+			break;
+			}
+		
+		default:
+			return false;
+		}
+	return true;
 	}
-      strcpy(tmp + tmp_offset, "}");
-      tmp_offset += 1;
-      sprintf(result, "%s", tmp); 
-      return result;
-    }
-    
-  case TYPE_VECTOR:
-    {
-      char* tmp = new char[MAX_EVENT_SIZE];
-      int tmp_offset = 0;
-      strcpy(tmp, "{");
-      tmp_offset = 1;
-      bool result_seen = false;
-      for ( int j = 0; j < val->val.vector_val.size; j++ )
+
+bool ElasticSearch::AddFieldToBuffer(Value* val, const Field* field)
 	{
-	  char* sub_field = FieldToString(val->val.vector_val.vals[j], field);
-	  if ( sub_field ){
-	    
-	    if ( result_seen ){
-	      strcpy(tmp + tmp_offset, ",");
-	      tmp_offset += 1;
-	    }
-	    else
-	      result_seen = true;
-	    
-	    sprintf(tmp + tmp_offset, "\"%s\":%s", field->name.c_str(), sub_field);
-	    tmp_offset = strlen(tmp);
-	  }
-	}
-      strcpy(tmp + tmp_offset, "}");
-      tmp_offset += 1;
-      sprintf(result, "%s", tmp); 
-      return result;
-    }
-
-  default:
-    {
-      return (char *)"{}";
-    }
-
-  }
-
-}
-
-char* ElasticSearch::AddFieldToBuffer(Value* val, const Field* field)
-	{
-    	if ( ! val->present )
-	  return 0;
+	if ( ! val->present )
+		return false;
 	
-	char* result = new char[MAX_EVENT_SIZE];
-	sprintf(result, "\"%s\":%s", field->name.c_str(), FieldToString(val, field));
-	return result;
-
+	buffer.AddRaw("\"", 1);
+	buffer.Add(field->name);
+	buffer.AddRaw("\":", 2);
+	AddFieldValueToBuffer(val, field);
+	return true;
 	}
 
 bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
-	  // Our action line looks like:
-	  //   {"index":{"_index":"$index_name","_type":"$type_prefix$path"}}\n{
-	  
-	  bool resultSeen = false;
-
-	  for ( int i = 0; i < num_fields; i++ )
+	// Our action line looks like:
+	//   {"index":{"_index":"$index_name","_type":"$type_prefix$path"}}\n
+	if ( counter == 0 )
 		{
-		char* result = AddFieldToBuffer(vals[i], fields[i]);
-		if ( result ) {
-		  if ( ! resultSeen ) {
-		    current_offset += sprintf(buffer + current_offset, "{\"index\":{\"_index\":\"%s\",\"_type\":\"%s%s\"}\n{", index_name, type_prefix, Path().c_str());
-		    resultSeen = true;
-		  }
-		  else {
-		    strcat(buffer, ",");
-		    current_offset += 1;
-		  }
-		  strcat(buffer, result);
-		  current_offset += strlen(result);
+		buffer.AddRaw("{\"index\":{\"_index\":\"", 20);
+		buffer.AddN((const char*) BifConst::LogElasticSearch::index_name->Bytes(),
+		            BifConst::LogElasticSearch::index_name->Len());
+		buffer.AddRaw("\",\"_type\":\"", 11);
+		buffer.AddN((const char*) BifConst::LogElasticSearch::type_prefix->Bytes(),
+		            BifConst::LogElasticSearch::type_prefix->Len());
+		buffer.Add(Path());
+		buffer.AddRaw("\"}\n", 3);
+		}
+	
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( i == 0 )
+			buffer.AddRaw("{", 1);
+		else if ( buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
+			buffer.AddRaw(",", 1);
+		AddFieldToBuffer(vals[i], fields[i]);
+		}
+		
+	buffer.AddRaw("}\n", 2);
+	
+	counter++;
+	if ( counter >= BifConst::LogElasticSearch::batch_size )
+		{
+		HTTPSend();
+		buffer.Clear();
+		counter = 0;
 		}
-	      }
-
-	  if ( resultSeen ) {
-	    strcat(buffer, "}\n");
-	    current_offset += 2;
-	    counter += 1;
-	    if ( counter >= batch_size ){
-	      BatchIndex();
-	      current_offset = 0;
-	      buffer[current_offset] = 0;
-	      counter = 0;
-	    }
-	  }
 	return true;
 	}
 
 bool ElasticSearch::DoRotate(string rotated_path, double open, double close, bool terminating)
 	{
-	  //TODO: Determine what, if anything, needs to be done here.
+	//TODO: Determine what, if anything, needs to be done here.
 	return true;
 	}
 
@@ -264,52 +220,54 @@ bool ElasticSearch::DoSetBuf(bool enabled)
 // HTTP Functions start here.
 
 CURL* ElasticSearch::HTTPSetup()
-{
-  char URL[2048];
-  CURL* handle;
-  struct curl_slist *headers=NULL;
-  
-  handle = curl_easy_init();
-  if ( ! handle )
-    return handle;
-  
-  sprintf(URL, "http://%s:%d/_bulk", server_host, (int) server_port);
-  curl_easy_setopt(handle, CURLOPT_URL, URL);
+	{
+	const char *URL = fmt("http://%s:%d/_bulk", BifConst::LogElasticSearch::server_host->CheckString(),
+	                                            (int) BifConst::LogElasticSearch::server_port);;
+	CURL* handle;
+	struct curl_slist *headers=NULL;
+	
+	handle = curl_easy_init();
+	if ( ! handle )
+		return handle;
+	
+	//sprintf(URL, "http://%s:%d/_bulk", BifConst::LogElasticSearch::server_host->CheckString(), (int) BifConst::LogElasticSearch::server_port);
+	curl_easy_setopt(handle, CURLOPT_URL, URL);
+	
+	headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
+	curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
+	
+	curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
+	curl_easy_setopt(handle, CURLOPT_POST, 1); // All requests are POSTs
+	
+	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. The best (only?) way to disable that is to
+	// just use HTTP 1.0
+	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+	return handle;
+	}
 
-  headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
-  curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
-
-  curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
-  curl_easy_setopt(handle, CURLOPT_POST, 1); // All requests are POSTs
-
-  // HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. The best (only?) way to disable that is to
-  // just use HTTP 1.0
-  curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
-  return handle;
-  
-}
-
-bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata){
-  //TODO: Do some verification on the result?
-  return true;
-}
+bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
+	{
+	//TODO: Do some verification on the result?
+	return true;
+	}
 
 bool ElasticSearch::HTTPSend(){
-  CURLcode return_code;
-  
-  curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, curl_result);
-  curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer);
-  curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, current_offset);
-  
-  return_code = curl_easy_perform(curl_handle);
-  switch(return_code) {
-  case CURLE_COULDNT_CONNECT:
-  case CURLE_COULDNT_RESOLVE_HOST:
-  case CURLE_WRITE_ERROR:
-    return false;
-  default:
-    return true;
-  }
+	CURLcode return_code;
+	
+	curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, curl_result);
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, buffer.Len());
+	
+	return_code = curl_easy_perform(curl_handle);
+	switch(return_code) {
+	case CURLE_COULDNT_CONNECT:
+	case CURLE_COULDNT_RESOLVE_HOST:
+	case CURLE_WRITE_ERROR:
+		return false;
+	
+	default:
+		return true;
+	}
 }
 
 #endif
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index ad3729f6da..21e9bdfe08 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -34,17 +34,15 @@ protected:
 	virtual bool DoFinish();
 
 private:
-	char* AddFieldToBuffer(threading::Value* val, const threading::Field* field);
-	char* FieldToString(threading::Value* val, const threading::Field* field);
-	bool BatchIndex();
+	bool AddFieldToBuffer(threading::Value* val, const threading::Field* field);
+	bool AddFieldValueToBuffer(threading::Value* val, const threading::Field* field);
 	
 	CURL* HTTPSetup();
 	bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
 	bool HTTPSend();
 	
 	// Buffers, etc.
-	char* buffer;
-	int current_offset;
+	ODesc buffer;
 	uint64 counter;
 
 	CURL* curl_handle;
@@ -54,19 +52,7 @@ private:
 	char* cluster_name;
 	int cluster_name_len;
 
-	char* server_host;
-	int server_host_len;
-
-	uint64 server_port;
-
-	char* index_name;
-	int index_name_len;
-
-	char* type_prefix;
-	int type_prefix_len;
-	
 	uint64 batch_size;
-
 };
 
 }

From a4df914ab7cab585abb1b456a048a5ae5e0f5e65 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 15 Jun 2012 20:53:09 -0400
Subject: [PATCH 018/238] Reduce the batch size to 1000 and add a maximum time
 interval for batches.

---
 .../logging/writers/elasticsearch.bro         |  7 ++-
 src/logging.bif                               |  1 +
 src/logging/writers/ElasticSearch.cc          | 53 +++++++++++++------
 src/logging/writers/ElasticSearch.h           |  9 ++--
 4 files changed, 50 insertions(+), 20 deletions(-)

diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index 7f968d0042..e2d14a68e3 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -20,6 +20,11 @@ export {
 	## The batch size is the number of messages that will be queued up before 
 	## they are sent to be bulk indexed.
 	## Note: this is mainly a memory usage parameter.
-	const batch_size = 10000 &redef;
+	const batch_size = 1000 &redef;
+	
+	## The maximum amount of wall-clock time that is allowed to pass without
+	## finishing a bulk log send.  This represents the maximum delay you 
+	## would like to have with your logs before they show up in ElasticSearch.
+	const max_batch_interval = 1min &redef;
 }
 
diff --git a/src/logging.bif b/src/logging.bif
index 308ea78b7a..5434ac3705 100644
--- a/src/logging.bif
+++ b/src/logging.bif
@@ -92,3 +92,4 @@ const server_port: count;
 const index_name: string;
 const type_prefix: string;
 const batch_size: count;
+const max_batch_interval: interval;
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index d8c4bee306..ed1c046143 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -32,6 +32,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	
 	buffer.Clear();
 	counter = 0;
+	last_send = current_time();
 	
 	curl_handle = HTTPSetup();
 	curl_result = new char[1024];
@@ -58,12 +59,21 @@ bool ElasticSearch::DoFinish()
 	{
 	return WriterBackend::DoFinish();
 	}
+	
+bool ElasticSearch::BatchIndex()
+	{
+	HTTPSend();
+	buffer.Clear();
+	counter = 0;
+	last_send = current_time();
+	return true;
+	}
 
 bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 	{
 	switch ( val->type ) 
 		{
-		// ElasticSearch defines bools as: 0 == false, everything else == true. So we treat it as an int.
+		// ES treats 0 as false and any other value as true so bool types go here.
 		case TYPE_BOOL:
 		case TYPE_INT:
 			buffer.Add(val->val.int_val);
@@ -197,11 +207,8 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 	
 	counter++;
 	if ( counter >= BifConst::LogElasticSearch::batch_size )
-		{
-		HTTPSend();
-		buffer.Clear();
-		counter = 0;
-		}
+		BatchIndex();
+	
 	return true;
 	}
 
@@ -217,6 +224,18 @@ bool ElasticSearch::DoSetBuf(bool enabled)
 	return true;
 	}
 
+bool ElasticSearch::DoHeartbeat(double network_time, double current_time)
+	{
+	if ( last_send > 0 &&
+	     current_time-last_send > BifConst::LogElasticSearch::max_batch_interval )
+		{
+		BatchIndex();
+		}
+	
+	return true;
+	}
+
+
 // HTTP Functions start here.
 
 CURL* ElasticSearch::HTTPSetup()
@@ -251,7 +270,8 @@ bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
 	return true;
 	}
 
-bool ElasticSearch::HTTPSend(){
+bool ElasticSearch::HTTPSend()
+	{
 	CURLcode return_code;
 	
 	curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, curl_result);
@@ -259,15 +279,16 @@ bool ElasticSearch::HTTPSend(){
 	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, buffer.Len());
 	
 	return_code = curl_easy_perform(curl_handle);
-	switch(return_code) {
-	case CURLE_COULDNT_CONNECT:
-	case CURLE_COULDNT_RESOLVE_HOST:
-	case CURLE_WRITE_ERROR:
-		return false;
-	
-	default:
-		return true;
+	switch ( return_code ) 
+		{
+		case CURLE_COULDNT_CONNECT:
+		case CURLE_COULDNT_RESOLVE_HOST:
+		case CURLE_WRITE_ERROR:
+			return false;
+		
+		default:
+			return true;
+		}
 	}
-}
 
 #endif
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index 21e9bdfe08..a366dd7020 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -32,10 +32,12 @@ protected:
 			      double close, bool terminating);
 	virtual bool DoFlush();
 	virtual bool DoFinish();
+	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
 	bool AddFieldToBuffer(threading::Value* val, const threading::Field* field);
 	bool AddFieldValueToBuffer(threading::Value* val, const threading::Field* field);
+	bool BatchIndex();
 	
 	CURL* HTTPSetup();
 	bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
@@ -44,14 +46,15 @@ private:
 	// Buffers, etc.
 	ODesc buffer;
 	uint64 counter;
-
+	double last_send;
+	
 	CURL* curl_handle;
 	char* curl_result;
-
+	
 	// From scripts
 	char* cluster_name;
 	int cluster_name_len;
-
+	
 	uint64 batch_size;
 };
 

From ca5eb5382ab9d67ce340e64b1692ef681c3167da Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 15 Jun 2012 21:06:06 -0400
Subject: [PATCH 019/238] Flush logs to ES daemon as Bro is shutting down.

---
 src/logging/writers/ElasticSearch.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index ed1c046143..5e1efa504e 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -51,12 +51,12 @@ bool ElasticSearch::DoInit(string path, int num_fields, const Field* const * fie
 
 bool ElasticSearch::DoFlush()
 	{
-	//TODO: Send flush command to ElasticSearch
 	return true;
 	}
 
 bool ElasticSearch::DoFinish()
 	{
+	BatchIndex();
 	return WriterBackend::DoFinish();
 	}
 	
@@ -120,7 +120,7 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 				{
 				char c = val->val.string_val->data()[i];
 				// HTML entity encode special characters.
-				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' )
+				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
 					{
 					buffer.AddRaw("&#", 2);
 					buffer.Add((uint8_t) c);

From 8334dceadb748a93effda4828db2439554fb532f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 15 Jun 2012 22:19:51 -0400
Subject: [PATCH 020/238] Changed the escaping method.

---
 src/logging/writers/ElasticSearch.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 5e1efa504e..a2019df9fe 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -122,9 +122,13 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 				// HTML entity encode special characters.
 				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
 					{
-					buffer.AddRaw("&#", 2);
-					buffer.Add((uint8_t) c);
-					buffer.AddRaw(";", 1);
+					static const char hex_chars[] = "0123456789abcdef";
+					buffer.AddRaw("\\u00", 4);
+					buffer.AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
+					buffer.AddRaw(&hex_chars[c & 0x0f], 1);
+					//buffer.AddRaw("&#//", 2);
+					//buffer.Add((uint8_t) c);
+					//buffer.AddRaw(";", 1);
 					}
 				else
 					buffer.AddRaw(&c, 1);

From b1561437e9d3bd8dfcf3fded3ff7ceca274d70e4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 16 Jun 2012 00:35:40 -0400
Subject: [PATCH 021/238] Forgot to call the parent method for DoHeartBeat.

---
 src/logging/writers/ElasticSearch.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index a2019df9fe..46282404a6 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -236,7 +236,7 @@ bool ElasticSearch::DoHeartbeat(double network_time, double current_time)
 		BatchIndex();
 		}
 	
-	return true;
+	return WriterBackend::DoHeartbeat(network_time, current_time);
 	}
 
 

From cd8169dda3150918a29eca21ca1fd7e7dcfc6ed2 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sat, 16 Jun 2012 22:22:40 -0400
Subject: [PATCH 022/238] Bug fix and feature.

- Fixed bug with how data is sent to elasticsearch.

- Added a feature to only allow data of a certain
  size to be buffered before sending to the
  elasticsearch server.  Configured with the
  LogElasticSearch::max_byte_size variable.
---
 .../logging/writers/elasticsearch.bro         |  6 +++-
 src/logging.bif                               |  3 +-
 src/logging/writers/ElasticSearch.cc          | 29 +++++++++----------
 3 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index e2d14a68e3..b262201c85 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -20,11 +20,15 @@ export {
 	## The batch size is the number of messages that will be queued up before 
 	## they are sent to be bulk indexed.
 	## Note: this is mainly a memory usage parameter.
-	const batch_size = 1000 &redef;
+	const max_batch_size = 1000 &redef;
 	
 	## The maximum amount of wall-clock time that is allowed to pass without
 	## finishing a bulk log send.  This represents the maximum delay you 
 	## would like to have with your logs before they show up in ElasticSearch.
 	const max_batch_interval = 1min &redef;
+	
+	## The maximum byte size for a buffered JSON string to send to the bulk
+	## insert API.
+	const max_byte_size = 1024 * 1024 &redef;
 }
 
diff --git a/src/logging.bif b/src/logging.bif
index 5434ac3705..cbae66efdb 100644
--- a/src/logging.bif
+++ b/src/logging.bif
@@ -91,5 +91,6 @@ const server_host: string;
 const server_port: count;
 const index_name: string;
 const type_prefix: string;
-const batch_size: count;
+const max_batch_size: count;
 const max_batch_interval: interval;
+const max_byte_size: count;
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 46282404a6..fd028e9b68 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -186,31 +186,27 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 	{
 	// Our action line looks like:
 	//   {"index":{"_index":"$index_name","_type":"$type_prefix$path"}}\n
-	if ( counter == 0 )
-		{
-		buffer.AddRaw("{\"index\":{\"_index\":\"", 20);
-		buffer.AddN((const char*) BifConst::LogElasticSearch::index_name->Bytes(),
-		            BifConst::LogElasticSearch::index_name->Len());
-		buffer.AddRaw("\",\"_type\":\"", 11);
-		buffer.AddN((const char*) BifConst::LogElasticSearch::type_prefix->Bytes(),
-		            BifConst::LogElasticSearch::type_prefix->Len());
-		buffer.Add(Path());
-		buffer.AddRaw("\"}\n", 3);
-		}
+	buffer.AddRaw("{\"index\":{\"_index\":\"", 20);
+	buffer.AddN((const char*) BifConst::LogElasticSearch::index_name->Bytes(),
+	            BifConst::LogElasticSearch::index_name->Len());
+	buffer.AddRaw("\",\"_type\":\"", 11);
+	buffer.AddN((const char*) BifConst::LogElasticSearch::type_prefix->Bytes(),
+	            BifConst::LogElasticSearch::type_prefix->Len());
+	buffer.Add(Path());
+	buffer.AddRaw("\"}\n", 3);
 	
+	buffer.AddRaw("{", 1);
 	for ( int i = 0; i < num_fields; i++ )
 		{
-		if ( i == 0 )
-			buffer.AddRaw("{", 1);
-		else if ( buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
+		if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
 			buffer.AddRaw(",", 1);
 		AddFieldToBuffer(vals[i], fields[i]);
 		}
-		
 	buffer.AddRaw("}\n", 2);
 	
 	counter++;
-	if ( counter >= BifConst::LogElasticSearch::batch_size )
+	if ( counter >= BifConst::LogElasticSearch::max_batch_size ||
+	     uint(buffer.Len()) >= BifConst::LogElasticSearch::max_byte_size )
 		BatchIndex();
 	
 	return true;
@@ -219,6 +215,7 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 bool ElasticSearch::DoRotate(string rotated_path, double open, double close, bool terminating)
 	{
 	//TODO: Determine what, if anything, needs to be done here.
+	
 	return true;
 	}
 

From 57980c86e62dc4dbbd95efe8cc761f84ac1c2d85 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sun, 17 Jun 2012 21:41:29 -0400
Subject: [PATCH 023/238] Bug fixes.

- The curl handle is now cleaned up correctly.

- Interval values are now treated as doubles.  Treating them as uint64_t
  was wrong because intervals can be negative.  There is also no obvious
  benefit in elasticsearch to converting the value to milliseconds.
---
 src/logging/writers/ElasticSearch.cc | 7 +++----
 src/logging/writers/ElasticSearch.h  | 1 -
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index fd028e9b68..402a2f21ad 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -35,8 +35,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	last_send = current_time();
 	
 	curl_handle = HTTPSetup();
-	curl_result = new char[1024];
-	}
+}
 
 ElasticSearch::~ElasticSearch()
 	{
@@ -57,6 +56,7 @@ bool ElasticSearch::DoFlush()
 bool ElasticSearch::DoFinish()
 	{
 	BatchIndex();
+	curl_easy_cleanup(curl_handle);
 	return WriterBackend::DoFinish();
 	}
 	
@@ -101,10 +101,10 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 			break;
 		
 		case TYPE_DOUBLE:
+		case TYPE_INTERVAL:
 			buffer.Add(val->val.double_val);
 			break;
 		
-		case TYPE_INTERVAL:
 		case TYPE_TIME:
 			// ElasticSearch uses milliseconds for timestamps
 			buffer.Add((uint64_t) (val->val.double_val * 1000));
@@ -275,7 +275,6 @@ bool ElasticSearch::HTTPSend()
 	{
 	CURLcode return_code;
 	
-	curl_easy_setopt(curl_handle, CURLOPT_WRITEDATA, curl_result);
 	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
 	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, buffer.Len());
 	
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index a366dd7020..bd1351214b 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -49,7 +49,6 @@ private:
 	double last_send;
 	
 	CURL* curl_handle;
-	char* curl_result;
 	
 	// From scripts
 	char* cluster_name;

From cb7eac212e33ec60e21886a793e73b346aba0ba1 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Sun, 17 Jun 2012 22:55:11 -0400
Subject: [PATCH 024/238] Small updates and a little standardization for
 config.h.in naming.

---
 CMakeLists.txt                       | 11 ++++++-----
 config.h.in                          |  7 +++++--
 configure                            |  1 -
 doc/logging-elasticsearch.rst        |  9 ++++-----
 src/logging/Manager.cc               |  4 ++--
 src/logging/writers/ElasticSearch.cc |  6 +++---
 src/main.cc                          | 12 ++++++++++++
 7 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4b1cccf8dc..14cf66ac19 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -122,13 +122,13 @@ if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
 	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
 endif()
 
-set(USE_LIBCURL false)
+set(USE_CURL false)
 find_package(CURL)
 if (CURL_FOUND)
-  set(USE_LIBCURL true)
+  set(USE_CURL true)
   include_directories(BEFORE ${CURL_INCLUDE_DIR})
   list(APPEND OPTLIBS ${CURL_LIBRARIES})
-  set(INSTALL_ELASTICSEARCH true)
+  set(USE_ELASTICSEARCH true)
 endif()
 
 if (ENABLE_PERFTOOLS_DEBUG)
@@ -218,13 +218,14 @@ message(
     "\nBroccoli:          ${INSTALL_BROCCOLI}"
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
-    "\nElasticSearch:     ${INSTALL_ELASTICSEARCH}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
     "\nGoogle perftools:  ${USE_PERFTOOLS}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
+    "\ncURL:              ${USE_CURL}"
+    "\n"
     "\nDataSeries:        ${USE_DATASERIES}"
-    "\nlibCURL:           ${USE_LIBCURL}"
+    "\nElasticSearch:     ${USE_ELASTICSEARCH}"
     "\n"
     "\n================================================================\n"
 )
diff --git a/config.h.in b/config.h.in
index 66121cefbf..aa286736fd 100644
--- a/config.h.in
+++ b/config.h.in
@@ -114,11 +114,14 @@
 /* Analyze Mobile IPv6 traffic */
 #cmakedefine ENABLE_MOBILE_IPV6
 
+/* Use libCurl. */
+#cmakedefine USE_CURL
+
 /* Use the DataSeries writer. */
 #cmakedefine USE_DATASERIES
 
-/* Build the ElasticSearch writer. */
-#cmakedefine INSTALL_ELASTICSEARCH
+/* Use the ElasticSearch writer. */
+#cmakedefine USE_ELASTICSEARCH
 
 /* Version number of package */
 #define VERSION "@VERSION@"
diff --git a/configure b/configure
index 801fb1e801..3258d4abfc 100755
--- a/configure
+++ b/configure
@@ -98,7 +98,6 @@ append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
-append_cache_entry INSTALL_ELASTICSEARCH BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index 4fce470d4a..26b49f3a0b 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -26,16 +26,15 @@ ElasticSearch with::
 Compiling Bro with ElasticSearch Support
 ----------------------------------------
 
-First, ensure that you have libcurl installed. Secondly, set the 
-``--enable-elasticsearch`` option::
+First, ensure that you have libcurl installed the run configure.::
 
-    # ./configure --enable-elasticsearch
+    # ./configure
     [...]
     ====================|  Bro Build Summary  |=====================
     [...]
-    ElasticSearch:     true
+    cURL:              true
     [...]
-    libCURL:           true
+    ElasticSearch:     true
     [...]
     ================================================================
 
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 5c1203fd91..5562b3b867 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -17,7 +17,7 @@
 #include "writers/Ascii.h"
 #include "writers/None.h"
 
-#ifdef INSTALL_ELASTICSEARCH
+#ifdef USE_ELASTICSEARCH
 #include "writers/ElasticSearch.h"
 #endif
 
@@ -40,7 +40,7 @@ WriterDefinition log_writers[] = {
 	{ BifEnum::Log::WRITER_NONE,  "None", 0, writer::None::Instantiate },
 	{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
 
-#ifdef INSTALL_ELASTICSEARCH
+#ifdef USE_ELASTICSEARCH
 	{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
 #endif
 
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 402a2f21ad..494c48f286 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -2,7 +2,7 @@
 
 #include "config.h"
 
-#ifdef INSTALL_ELASTICSEARCH
+#ifdef USE_ELASTICSEARCH
 
 #include <string>
 #include <errno.h>
@@ -261,7 +261,7 @@ CURL* ElasticSearch::HTTPSetup()
 	
 	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. The best (only?) way to disable that is to
 	// just use HTTP 1.0
-	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+	//curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 	return handle;
 	}
 
@@ -275,8 +275,8 @@ bool ElasticSearch::HTTPSend()
 	{
 	CURLcode return_code;
 	
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, buffer.Len());
 	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
-	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE, buffer.Len());
 	
 	return_code = curl_easy_perform(curl_handle);
 	switch ( return_code ) 
diff --git a/src/main.cc b/src/main.cc
index b1d0a4d723..04aa83b832 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -12,6 +12,10 @@
 #include <getopt.h>
 #endif
 
+#ifdef USE_CURL
+#include <curl/curl.h>
+#endif
+
 #ifdef USE_IDMEF
 extern "C" {
 #include <libidmef/idmefxml.h>
@@ -716,6 +720,10 @@ int main(int argc, char** argv)
 	SSL_library_init();
 	SSL_load_error_strings();
 
+#ifdef USE_CURL
+	curl_global_init(CURL_GLOBAL_ALL);
+#endif
+
 	// FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't
 	// seed the PRNG. We should do this here (but at least Linux, FreeBSD
 	// and Solaris provide /dev/urandom).
@@ -1066,6 +1074,10 @@ int main(int argc, char** argv)
 		done_with_network();
 		net_delete();
 
+#ifdef USE_CURL
+		curl_global_cleanup();
+#endif
+		
 		terminate_bro();
 
 		// Close files after net_delete(), because net_delete()

From 52ceee8c869ac04d0bbf74ec8296ef983e182742 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 18 Jun 2012 01:31:52 -0400
Subject: [PATCH 025/238] Fixed a bug with messed up time value passing to
 elasticsearch.

---
 src/logging/writers/ElasticSearch.cc | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 494c48f286..75a4e0514f 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -106,9 +106,19 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 			break;
 		
 		case TYPE_TIME:
-			// ElasticSearch uses milliseconds for timestamps
-			buffer.Add((uint64_t) (val->val.double_val * 1000));
+			{
+			// ElasticSearch uses milliseconds for timestamps and json only
+			// supports signed ints (uints can be too large).
+			uint64_t ts = (uint64_t) (val->val.double_val * 1000);
+			if ( ts >= INT64_MAX )
+				{
+				Error(Fmt("time value too large: %" PRIu64, ts));
+				buffer.AddRaw("null", 4);
+				}
+			else
+				buffer.Add(ts);
 			break;
+			}
 		
 		case TYPE_ENUM:
 		case TYPE_STRING:
@@ -261,7 +271,7 @@ CURL* ElasticSearch::HTTPSetup()
 	
 	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. The best (only?) way to disable that is to
 	// just use HTTP 1.0
-	//curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 	return handle;
 	}
 

From 95f4390cb632f2c445ed45a6670b6ad8266e49c5 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 18 Jun 2012 02:03:43 -0400
Subject: [PATCH 026/238] Adding an extra header.

---
 src/logging/writers/ElasticSearch.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 75a4e0514f..c137505811 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -6,6 +6,7 @@
 
 #include <string>
 #include <errno.h>
+#include <stdint.h>
 
 #include "util.h"
 #include "BroString.h"

From 7fc96a8c0f012d22f6cba7a38869108c7f9fef4d Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 18 Jun 2012 15:49:00 -0400
Subject: [PATCH 027/238] Adding a define to make the stdint C macros
 available.

---
 src/logging/writers/ElasticSearch.cc | 1 -
 src/util.h                           | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index c137505811..75a4e0514f 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -6,7 +6,6 @@
 
 #include <string>
 #include <errno.h>
-#include <stdint.h>
 
 #include "util.h"
 #include "BroString.h"
diff --git a/src/util.h b/src/util.h
index 6b237edfd8..559a155626 100644
--- a/src/util.h
+++ b/src/util.h
@@ -13,6 +13,7 @@
 // Expose C99 functionality from inttypes.h, which would otherwise not be
 // available in C++.
 #define __STDC_FORMAT_MACROS
+#define __STDC_LIMIT_MACROS
 #include <inttypes.h>
 
 #if __STDC__

From 84e91b8b8d18e310c0f61372a19434c19dfdd709 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 9 Jul 2012 16:38:05 -0400
Subject: [PATCH 028/238] Bringing elasticsearch branch up to date with master.

---
 scripts/base/frameworks/logging/__load__.bro  |   3 +-
 .../logging/writers/elasticsearch.bro         |   2 +-
 src/logging.bif                               |   2 +-
 src/logging/writers/ElasticSearch.cc          | 221 ++++++++++++------
 src/logging/writers/ElasticSearch.h           |  22 +-
 5 files changed, 167 insertions(+), 83 deletions(-)

diff --git a/scripts/base/frameworks/logging/__load__.bro b/scripts/base/frameworks/logging/__load__.bro
index 2c2a6d2f59..b65cb1dea3 100644
--- a/scripts/base/frameworks/logging/__load__.bro
+++ b/scripts/base/frameworks/logging/__load__.bro
@@ -2,4 +2,5 @@
 @load ./postprocessors
 @load ./writers/ascii
 @load ./writers/dataseries
-@load ./writers/elasticsearch@load ./writers/none
+@load ./writers/elasticsearch
+@load ./writers/none
diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index b262201c85..93c6c98705 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -11,7 +11,7 @@ export {
 	const server_port = 9200 &redef;
 
 	## Name of the ES index
-	const index_name = "bro" &redef;
+	const index_prefix = "bro" &redef;
 
 	## The ES type prefix comes before the name of the related log.
 	## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.
diff --git a/src/logging.bif b/src/logging.bif
index 23b9378b26..3cdb414d80 100644
--- a/src/logging.bif
+++ b/src/logging.bif
@@ -89,7 +89,7 @@ module LogElasticSearch;
 const cluster_name: string;
 const server_host: string;
 const server_port: count;
-const index_name: string;
+const index_prefix: string;
 const type_prefix: string;
 const max_batch_size: count;
 const max_batch_interval: interval;
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 75a4e0514f..6d2f8363cc 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -30,8 +30,17 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len);
 	cluster_name[cluster_name_len] = 0;
 	
+	index_prefix = string((const char*) BifConst::LogElasticSearch::index_prefix->Bytes(), BifConst::LogElasticSearch::index_prefix->Len());
+	
+	es_server = string(Fmt("http://%s:%d", BifConst::LogElasticSearch::server_host->Bytes(),
+	                                       (int) BifConst::LogElasticSearch::server_port));
+	bulk_url = string(Fmt("%s/_bulk", es_server.c_str()));
+	
+	http_headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
 	buffer.Clear();
 	counter = 0;
+	current_index = string();
+	prev_index = string();
 	last_send = current_time();
 	
 	curl_handle = HTTPSetup();
@@ -42,67 +51,84 @@ ElasticSearch::~ElasticSearch()
 	delete [] cluster_name;
 	}
 
-bool ElasticSearch::DoInit(string path, int num_fields, const Field* const * fields)
+bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
 	{
-	//TODO: Determine what, if anything, needs to be done here.
 	return true;
 	}
 
 bool ElasticSearch::DoFlush()
 	{
+	BatchIndex();
 	return true;
 	}
 
 bool ElasticSearch::DoFinish()
 	{
 	BatchIndex();
+	curl_slist_free_all(http_headers);
 	curl_easy_cleanup(curl_handle);
 	return WriterBackend::DoFinish();
 	}
 	
 bool ElasticSearch::BatchIndex()
 	{
-	HTTPSend();
+	curl_easy_reset(curl_handle);
+	curl_easy_setopt(curl_handle, CURLOPT_URL, bulk_url.c_str());
+	curl_easy_setopt(curl_handle, CURLOPT_POST, 1);
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)buffer.Len());
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
+	HTTPSend(curl_handle);
+	
 	buffer.Clear();
 	counter = 0;
 	last_send = current_time();
+	
 	return true;
 	}
 
-bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
+bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val)
 	{
 	switch ( val->type ) 
 		{
 		// ES treats 0 as false and any other value as true so bool types go here.
 		case TYPE_BOOL:
 		case TYPE_INT:
-			buffer.Add(val->val.int_val);
+			b->Add(val->val.int_val);
 			break;
 		
 		case TYPE_COUNT:
 		case TYPE_COUNTER:
-			buffer.Add(val->val.uint_val);
+			{
+			// ElasticSearch doesn't seem to support unsigned 64bit ints.
+			if ( val->val.uint_val >= INT64_MAX )
+				{
+				Error(Fmt("count value too large: %" PRIu64, val->val.uint_val));
+				b->AddRaw("null", 4);
+				}
+			else
+				b->Add(val->val.uint_val);
 			break;
+			}
 		
 		case TYPE_PORT:
-			buffer.Add(val->val.port_val.port);
+			b->Add(val->val.port_val.port);
 			break;
 		
 		case TYPE_SUBNET:
-			buffer.AddRaw("\"", 1);
-			buffer.Add(Render(val->val.subnet_val));
-			buffer.AddRaw("\"", 1);
+			b->AddRaw("\"", 1);
+			b->Add(Render(val->val.subnet_val));
+			b->AddRaw("\"", 1);
 			break;
 		
 		case TYPE_ADDR:
-			buffer.AddRaw("\"", 1);
-			buffer.Add(Render(val->val.addr_val));
-			buffer.AddRaw("\"", 1);
+			b->AddRaw("\"", 1);
+			b->Add(Render(val->val.addr_val));
+			b->AddRaw("\"", 1);
 			break;
 		
 		case TYPE_DOUBLE:
 		case TYPE_INTERVAL:
-			buffer.Add(val->val.double_val);
+			b->Add(val->val.double_val);
 			break;
 		
 		case TYPE_TIME:
@@ -113,10 +139,10 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 			if ( ts >= INT64_MAX )
 				{
 				Error(Fmt("time value too large: %" PRIu64, ts));
-				buffer.AddRaw("null", 4);
+				b->AddRaw("null", 4);
 				}
 			else
-				buffer.Add(ts);
+				b->Add(ts);
 			break;
 			}
 		
@@ -125,51 +151,48 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 		case TYPE_FILE:
 		case TYPE_FUNC:
 			{
-			buffer.AddRaw("\"", 1);
+			b->AddRaw("\"", 1);
 			for ( uint i = 0; i < val->val.string_val->size(); ++i )
 				{
 				char c = val->val.string_val->data()[i];
-				// HTML entity encode special characters.
+				// 2byte Unicode escape special characters.
 				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
 					{
 					static const char hex_chars[] = "0123456789abcdef";
-					buffer.AddRaw("\\u00", 4);
-					buffer.AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
-					buffer.AddRaw(&hex_chars[c & 0x0f], 1);
-					//buffer.AddRaw("&#//", 2);
-					//buffer.Add((uint8_t) c);
-					//buffer.AddRaw(";", 1);
+					b->AddRaw("\\u00", 4);
+					b->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
+					b->AddRaw(&hex_chars[c & 0x0f], 1);
 					}
 				else
-					buffer.AddRaw(&c, 1);
+					b->AddRaw(&c, 1);
 				}
-			buffer.AddRaw("\"", 1);
+			b->AddRaw("\"", 1);
 			break;
 			}
 		
 		case TYPE_TABLE:
 			{
-			buffer.AddRaw("[", 1);
+			b->AddRaw("[", 1);
 			for ( int j = 0; j < val->val.set_val.size; j++ )
 				{
 				if ( j > 0 )
-					buffer.AddRaw(",", 1);
-				AddFieldValueToBuffer(val->val.set_val.vals[j], field);
+					b->AddRaw(",", 1);
+				AddValueToBuffer(b, val->val.set_val.vals[j]);
 				}
-			buffer.AddRaw("]", 1);
+			b->AddRaw("]", 1);
 			break;
 			}
 			
 		case TYPE_VECTOR:
 			{
-			buffer.AddRaw("[", 1);
+			b->AddRaw("[", 1);
 			for ( int j = 0; j < val->val.vector_val.size; j++ )
 				{
 				if ( j > 0 )
-					buffer.AddRaw(",", 1);
-				AddFieldValueToBuffer(val->val.vector_val.vals[j], field);
+					b->AddRaw(",", 1);
+				AddValueToBuffer(b, val->val.vector_val.vals[j]);
 				}
-			buffer.AddRaw("]", 1);
+			b->AddRaw("]", 1);
 			break;
 			}
 		
@@ -179,38 +202,37 @@ bool ElasticSearch::AddFieldValueToBuffer(Value* val, const Field* field)
 	return true;
 	}
 
-bool ElasticSearch::AddFieldToBuffer(Value* val, const Field* field)
+bool ElasticSearch::AddFieldToBuffer(ODesc *b, Value* val, const Field* field)
 	{
 	if ( ! val->present )
 		return false;
 	
-	buffer.AddRaw("\"", 1);
-	buffer.Add(field->name);
-	buffer.AddRaw("\":", 2);
-	AddFieldValueToBuffer(val, field);
+	b->AddRaw("\"", 1);
+	b->Add(field->name);
+	b->AddRaw("\":", 2);
+	AddValueToBuffer(b, val);
 	return true;
 	}
 
 bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
+	if ( current_index.empty() )
+		UpdateIndex(network_time, Info().rotation_interval, Info().rotation_base);
+	
 	// Our action line looks like:
-	//   {"index":{"_index":"$index_name","_type":"$type_prefix$path"}}\n
 	buffer.AddRaw("{\"index\":{\"_index\":\"", 20);
-	buffer.AddN((const char*) BifConst::LogElasticSearch::index_name->Bytes(),
-	            BifConst::LogElasticSearch::index_name->Len());
+	buffer.Add(current_index);
 	buffer.AddRaw("\",\"_type\":\"", 11);
-	buffer.AddN((const char*) BifConst::LogElasticSearch::type_prefix->Bytes(),
-	            BifConst::LogElasticSearch::type_prefix->Len());
-	buffer.Add(Path());
-	buffer.AddRaw("\"}\n", 3);
+	buffer.Add(Info().path);
+	buffer.AddRaw("\"}}\n", 4);
 	
 	buffer.AddRaw("{", 1);
 	for ( int i = 0; i < num_fields; i++ )
 		{
 		if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
 			buffer.AddRaw(",", 1);
-		AddFieldToBuffer(vals[i], fields[i]);
+		AddFieldToBuffer(&buffer, vals[i], fields[i]);
 		}
 	buffer.AddRaw("}\n", 2);
 	
@@ -221,10 +243,63 @@ bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
 	
 	return true;
 	}
+	
+bool ElasticSearch::UpdateIndex(double now, double rinterval, double rbase)
+	{
+	if ( rinterval == 0 )
+		{
+		// if logs aren't being rotated, don't use a rotation oriented index name.
+		current_index = index_prefix;
+		}
+	else
+		{
+		double nr = calc_next_rotate(now, rinterval, rbase);
+		double interval_beginning = now - (rinterval - nr);
+		
+		struct tm tm;
+		char buf[128];
+		time_t teatime = (time_t)interval_beginning;
+		gmtime_r(&teatime, &tm);
+		strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm);
+		
+		prev_index = current_index;
+		current_index = index_prefix + "-" + buf;
+		}
+	
+		//printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str());
+	return true;
+	}
+	
 
 bool ElasticSearch::DoRotate(string rotated_path, double open, double close, bool terminating)
 	{
-	//TODO: Determine what, if anything, needs to be done here.
+	// Update the currently used index to the new rotation interval.
+	UpdateIndex(close, Info().rotation_interval, Info().rotation_base);
+	
+	// Only do this stuff if there was a previous index.
+	if ( ! prev_index.empty() )
+		{
+		// FIXME: I think this section is taking too long and causing the thread to die.
+		
+		// Compress the previous index
+		//curl_easy_reset(curl_handle);
+		//curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_settings", es_server.c_str(), prev_index.c_str()));
+		//curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PUT");
+		//curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "{\"index\":{\"store.compress.stored\":\"true\"}}");
+		//curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) 42);
+		//HTTPSend(curl_handle);
+		
+		// Optimize the previous index.
+		// TODO: make this into variables.
+		//curl_easy_reset(curl_handle);
+		//curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_optimize?max_num_segments=1&wait_for_merge=false", es_server.c_str(), prev_index.c_str()));
+		//HTTPSend(curl_handle);
+		}
+	
+	//if ( ! FinishedRotation(current_index, prev_index, open, close, terminating) )
+	//	{
+	//	Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
+	//	}
 	
 	return true;
 	}
@@ -237,7 +312,7 @@ bool ElasticSearch::DoSetBuf(bool enabled)
 
 bool ElasticSearch::DoHeartbeat(double network_time, double current_time)
 	{
-	if ( last_send > 0 &&
+	if ( last_send > 0 && buffer.Len() > 0 &&
 	     current_time-last_send > BifConst::LogElasticSearch::max_batch_interval )
 		{
 		BatchIndex();
@@ -247,31 +322,15 @@ bool ElasticSearch::DoHeartbeat(double network_time, double current_time)
 	}
 
 
-// HTTP Functions start here.
-
 CURL* ElasticSearch::HTTPSetup()
 	{
-	const char *URL = fmt("http://%s:%d/_bulk", BifConst::LogElasticSearch::server_host->CheckString(),
-	                                            (int) BifConst::LogElasticSearch::server_port);;
-	CURL* handle;
-	struct curl_slist *headers=NULL;
-	
-	handle = curl_easy_init();
+	CURL* handle = curl_easy_init();
 	if ( ! handle )
-		return handle;
+		{
+		Error("cURL did not initialize correctly.");
+		return 0;
+		}
 	
-	//sprintf(URL, "http://%s:%d/_bulk", BifConst::LogElasticSearch::server_host->CheckString(), (int) BifConst::LogElasticSearch::server_port);
-	curl_easy_setopt(handle, CURLOPT_URL, URL);
-	
-	headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
-	curl_easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
-	
-	curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
-	curl_easy_setopt(handle, CURLOPT_POST, 1); // All requests are POSTs
-	
-	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. The best (only?) way to disable that is to
-	// just use HTTP 1.0
-	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 	return handle;
 	}
 
@@ -281,14 +340,16 @@ bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
 	return true;
 	}
 
-bool ElasticSearch::HTTPSend()
+bool ElasticSearch::HTTPSend(CURL *handle)
 	{
-	CURLcode return_code;
+	curl_easy_setopt(handle, CURLOPT_HTTPHEADER, http_headers);
+	curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
+	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. 
+	// The best (only?) way to disable that is to just use HTTP 1.0
+	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 	
-	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, buffer.Len());
-	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
+	CURLcode return_code = curl_easy_perform(handle);
 	
-	return_code = curl_easy_perform(curl_handle);
 	switch ( return_code ) 
 		{
 		case CURLE_COULDNT_CONNECT:
@@ -296,6 +357,16 @@ bool ElasticSearch::HTTPSend()
 		case CURLE_WRITE_ERROR:
 			return false;
 		
+		case CURLE_OK:
+			{
+			uint http_code = 0;
+			curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
+			if ( http_code != 200 )
+				Error(Fmt("Received a non-successful status code back from ElasticSearch server."));
+			
+			return true;
+			}
+		
 		default:
 			return true;
 		}
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index bd1351214b..375845b002 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -22,8 +22,8 @@ public:
 protected:
 	// Overidden from WriterBackend.
 
-	virtual bool DoInit(string path, int num_fields,
-			    const threading::Field* const * fields);
+	virtual bool DoInit(const WriterInfo& info, int num_fields,
+			    const threading::Field* const* fields);
 
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals);
@@ -35,18 +35,22 @@ protected:
 	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
-	bool AddFieldToBuffer(threading::Value* val, const threading::Field* field);
-	bool AddFieldValueToBuffer(threading::Value* val, const threading::Field* field);
+	bool AddFieldToBuffer(ODesc *b, threading::Value* val, const threading::Field* field);
+	bool AddValueToBuffer(ODesc *b, threading::Value* val);
 	bool BatchIndex();
+	bool SendMappings();
+	bool UpdateIndex(double now, double rinterval, double rbase);
 	
 	CURL* HTTPSetup();
 	bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
-	bool HTTPSend();
+	bool HTTPSend(CURL *handle);
 	
 	// Buffers, etc.
 	ODesc buffer;
 	uint64 counter;
 	double last_send;
+	string current_index;
+	string prev_index;
 	
 	CURL* curl_handle;
 	
@@ -54,6 +58,14 @@ private:
 	char* cluster_name;
 	int cluster_name_len;
 	
+	string es_server;
+	string bulk_url;
+	
+	struct curl_slist *http_headers;
+	
+	string path;
+	string index_prefix;
+	
 	uint64 batch_size;
 };
 

From 9b70ee8799ec9b52528eb750abfd34bed2278422 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 9 Jul 2012 16:50:42 -0400
Subject: [PATCH 029/238] Tiny updates.

---
 doc/logging-elasticsearch.rst        | 3 +--
 src/logging/writers/ElasticSearch.cc | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index 26b49f3a0b..b6d22cf5fa 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -78,7 +78,7 @@ Bro's ElasticSearch writer comes with a few configuration options::
 
 - server_port:  What port to send the data to. Default 9200.
 
-- index_name:   ElasticSearch indexes are like databases in a standard DB model. 
+- index_prefix:   ElasticSearch indexes are like databases in a standard DB model. 
   This is the name of the index to which to send the data. Default bro.
 
 - type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
@@ -92,5 +92,4 @@ Lots.
 
 - Perform multicast discovery for server.
 - Better error detection.
-- Dynamic index names.
 - Better defaults (don't index loaded-plugins, for instance).
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 6d2f8363cc..4461508083 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -266,7 +266,7 @@ bool ElasticSearch::UpdateIndex(double now, double rinterval, double rbase)
 		current_index = index_prefix + "-" + buf;
 		}
 	
-		//printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str());
+	//printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str());
 	return true;
 	}
 	

From 6e5382da548a4d8ffbd73089a3a502778d477176 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 10 Jul 2012 23:49:31 -0400
Subject: [PATCH 030/238] Re-adding the needed call to FinishedRotation in the
 ES writer plugin.

---
 src/logging/writers/ElasticSearch.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 4461508083..1b8dfa495d 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -296,10 +296,10 @@ bool ElasticSearch::DoRotate(string rotated_path, double open, double close, boo
 		//HTTPSend(curl_handle);
 		}
 	
-	//if ( ! FinishedRotation(current_index, prev_index, open, close, terminating) )
-	//	{
-	//	Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
-	//	}
+	if ( ! FinishedRotation(current_index, prev_index, open, close, terminating) )
+		{
+		Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
+		}
 	
 	return true;
 	}

From 5607e86ad3e8349426d0205fc8867050079d24d4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 12 Jul 2012 12:55:34 -0400
Subject: [PATCH 031/238] Reporter warnings and error now print to stderr by
 default.

- Changed the geoip warnings to Info.
---
 scripts/base/frameworks/reporter/main.bro | 26 +++++++++++++++++++----
 src/bro.bif                               |  4 ++--
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro
index 3c19005364..8b45819442 100644
--- a/scripts/base/frameworks/reporter/main.bro
+++ b/scripts/base/frameworks/reporter/main.bro
@@ -11,7 +11,7 @@ module Reporter;
 export {
 	## The reporter logging stream identifier.
 	redef enum Log::ID += { LOG };
-
+	
 	## An indicator of reporter message severity.
 	type Level: enum { 
 		## Informational, not needing specific attention.
@@ -36,24 +36,42 @@ export {
 		## Not all reporter messages will have locations in them though.
 		location: string &log &optional;
 	};
+	
+	## Send reporter error messages to STDERR by default.  The option to 
+	## turn it off is presented here in case Bro is being run by some 
+	## external harness and shouldn't output anything to the console.
+	const errors_to_stderr = T &redef;
+	
+	## Send reporter warning messages to STDERR by default.  The option to 
+	## turn it off is presented here in case Bro is being run by some 
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
 }
 
+global stderr: file;
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Reporter::LOG, [$columns=Info]);
+	
+	if ( errors_to_stderr || warnings_to_stderr )
+		stderr = open("/dev/stderr");
 	}
 
-event reporter_info(t: time, msg: string, location: string)
+event reporter_info(t: time, msg: string, location: string) &priority=-5
 	{
 	Log::write(Reporter::LOG, [$ts=t, $level=INFO, $message=msg, $location=location]);
 	}
 	
-event reporter_warning(t: time, msg: string, location: string)
+event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
-event reporter_error(t: time, msg: string, location: string)
+event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( errors_to_stderr )
+		print stderr, fmt("ERROR: %s", msg);
+	
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}
diff --git a/src/bro.bif b/src/bro.bif
index f18d3ba1b5..1d4aeed4d6 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -3764,7 +3764,7 @@ static GeoIP* open_geoip_db(GeoIPDBTypes type)
 		geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE);
 
 	if ( ! geoip )
-		reporter->Warning("Failed to open GeoIP database: %s",
+		reporter->Info("Failed to open GeoIP database: %s",
 		                  GeoIPDBFileName[type]);
 	return geoip;
 	}
@@ -3804,7 +3804,7 @@ function lookup_location%(a: addr%) : geo_location
 			if ( ! geoip )
 				builtin_error("Can't initialize GeoIP City/Country database");
 			else
-				reporter->Warning("Fell back to GeoIP Country database");
+				reporter->Info("Fell back to GeoIP Country database");
 			}
 		else
 			have_city_db = true;

From e1bd9609264a4d067e3c58016806877f0f859c8d Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Fri, 13 Jul 2012 02:20:41 -0700
Subject: [PATCH 032/238] Unblock SIGFPE, SIGILL, SIGSEGV and SIGBUS for
 threads.

According to POSIX, behavior is unspecified if a specific thread receives one of those signals (because of e.g. executing an invalid instruction) if the signal is blocked.

This resulted in segfaults in threads not propagating to the main thread.

Adresses #848
---
 src/threading/BasicThread.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index 3dda6b5e8c..6ce5ad5f52 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -164,6 +164,13 @@ void* BasicThread::launcher(void *arg)
 	// process.
 	sigset_t mask_set;
 	sigfillset(&mask_set);
+	// Unblock the signals where according to POSIX the result is undefined if they are blocked
+	// in a thread and received by that thread. If those are not unblocked, threads will just
+	// hang when they crash without the user being notified.
+	sigdelset(&mask_set, SIGFPE);
+	sigdelset(&mask_set, SIGILL);
+	sigdelset(&mask_set, SIGSEGV);
+	sigdelset(&mask_set, SIGBUS);
 	int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
 	assert(res == 0);  //
 

From f43576cff346bcecde12bd477f444d532e4b0632 Mon Sep 17 00:00:00 2001
From: Vlad Grigorescu <vladg@cmu.edu>
Date: Fri, 13 Jul 2012 14:04:24 -0400
Subject: [PATCH 033/238] Fix some Info:Record field documentation.

---
 .../base/frameworks/communication/main.bro    |  2 +-
 scripts/base/protocols/conn/main.bro          | 25 +++++++++--------
 scripts/base/protocols/dns/main.bro           |  8 +++---
 scripts/base/protocols/ftp/main.bro           |  2 ++
 scripts/base/protocols/http/main.bro          |  2 ++
 scripts/base/protocols/irc/main.bro           |  2 ++
 scripts/base/protocols/smtp/main.bro          | 28 +++++++++++++++----
 scripts/base/protocols/socks/main.bro         |  8 ++++--
 scripts/base/protocols/ssh/main.bro           | 10 ++++---
 scripts/base/protocols/ssl/main.bro           | 10 ++++---
 scripts/base/protocols/syslog/main.bro        |  4 ++-
 11 files changed, 67 insertions(+), 34 deletions(-)

diff --git a/scripts/base/frameworks/communication/main.bro b/scripts/base/frameworks/communication/main.bro
index ceae357f78..81f9a383b9 100644
--- a/scripts/base/frameworks/communication/main.bro
+++ b/scripts/base/frameworks/communication/main.bro
@@ -42,7 +42,7 @@ export {
 	type Info: record {
 		## The network time at which a communication event occurred.
 		ts:                  time   &log;
-		## The peer name (if any) for which a communication event is concerned.
+		## The peer name (if any) with which a communication event is concerned.
 		peer:                string &log &optional;
 		## Where the communication event message originated from, that is,
 		## either from the scripting layer or inside the Bro process.
diff --git a/scripts/base/protocols/conn/main.bro b/scripts/base/protocols/conn/main.bro
index 5796c3c6b1..6cc2510027 100644
--- a/scripts/base/protocols/conn/main.bro
+++ b/scripts/base/protocols/conn/main.bro
@@ -17,7 +17,7 @@ export {
 	type Info: record {
 		## This is the time of the first packet.
 		ts:           time            &log;
-		## A unique identifier of a connection.
+		## A unique identifier of the connection.
 		uid:          string          &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:           conn_id         &log;
@@ -61,7 +61,7 @@ export {
 		## be left empty at all times.
 		local_orig:   bool            &log &optional;
 		
-		## Indicates the number of bytes missed in content gaps which is
+		## Indicates the number of bytes missed in content gaps, which is
 		## representative of packet loss.  A value other than zero will 
 		## normally cause protocol analysis to fail but some analysis may 
 		## have been completed prior to the packet loss.
@@ -83,23 +83,24 @@ export {
 		## i       inconsistent packet (e.g. SYN+RST bits both set)
 		## ======  ====================================================
 		##
-		## If the letter is in upper case it means the event comes from the
-		## originator and lower case then means the responder.
-		## Also, there is compression. We only record one "d" in each direction,
-		## for instance. I.e., we just record that data went in that direction.
-		## This history is not meant to encode how much data that happened to
-		## be.
+		## If the event comes from the originator, the letter is in upper-case; if it comes
+		## from the responder, it's in lower-case. Multiple packets of the same type will
+		## only be noted once (e.g. we only record one "d" in each direction, regardless of 
+		## how many data packets were seen.)
 		history:      string          &log &optional;
-		## Number of packets the originator sent.
+		## Number of packets that the originator sent.
 		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_pkts:     count      &log &optional;
-		## Number IP level bytes the originator sent (as seen on the wire,
+		## Number of IP level bytes that the originator sent (as seen on the wire,
 		## taken from IP total_length header field).
 		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_ip_bytes: count      &log &optional;
-		## Number of packets the responder sent. See ``orig_pkts``.
+		## Number of packets that the responder sent.
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		resp_pkts:     count      &log &optional;
-		## Number IP level bytes the responder sent. See ``orig_pkts``.
+		## Number og IP level bytes that the responder sent (as seen on the wire,
+		## taken from IP total_length header field).
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		resp_ip_bytes: count      &log &optional;
 		## If this connection was over a tunnel, indicate the 
 		## *uid* values for any encapsulating parent connections
diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index c50a8bdc54..600de4beaf 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -45,16 +45,16 @@ export {
 		AA:            bool               &log &default=F;
 		## The Truncation bit specifies that the message was truncated.
 		TC:            bool               &log &default=F;
-		## The Recursion Desired bit indicates to a name server to recursively
-		## purse the query.
+		## The Recursion Desired bit in a request message indicates that
+		## the client wants recursive service for this query.
 		RD:            bool               &log &default=F;
-		## The Recursion Available bit in a response message indicates if
+		## The Recursion Available bit in a response message indicates that
 		## the name server supports recursive queries.
 		RA:            bool               &log &default=F;
 		## A reserved field that is currently supposed to be zero in all
 		## queries and responses.
 		Z:             count              &log &default=0;
-		## The set of resource descriptions in answer of the query.
+		## The set of resource descriptions in the query answer.
 		answers:       vector of string   &log &optional;
 		## The caching intervals of the associated RRs described by the
 		## ``answers`` field.
diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index 7c5bbaefdc..d20bc92d8a 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -28,7 +28,9 @@ export {
 	type Info: record {
 		## Time when the command was sent.
 		ts:               time        &log;
+		## Unique ID for the connection.
 		uid:              string      &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:               conn_id     &log;
 		## User name for the current FTP session.
 		user:             string      &log &default="<unknown>";
diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro
index 6571548145..f4377e03de 100644
--- a/scripts/base/protocols/http/main.bro
+++ b/scripts/base/protocols/http/main.bro
@@ -22,7 +22,9 @@ export {
 	type Info: record {
 		## Timestamp for when the request happened.
 		ts:                      time      &log;
+		## Unique ID for the connection.
 		uid:                     string    &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:                      conn_id   &log;
 		## Represents the pipelined depth into the connection of this 
 		## request/response transaction.
diff --git a/scripts/base/protocols/irc/main.bro b/scripts/base/protocols/irc/main.bro
index 2bf2a9bbb9..1cf542b8ea 100644
--- a/scripts/base/protocols/irc/main.bro
+++ b/scripts/base/protocols/irc/main.bro
@@ -11,7 +11,9 @@ export {
 	type Info: record {
 		## Timestamp when the command was seen.
 		ts:       time        &log;
+		## Unique ID for the connection.
 		uid:      string      &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:       conn_id     &log;
 		## Nick name given for the connection.
 		nick:     string      &log &optional;
diff --git a/scripts/base/protocols/smtp/main.bro b/scripts/base/protocols/smtp/main.bro
index 513b85e342..03b3d36a24 100644
--- a/scripts/base/protocols/smtp/main.bro
+++ b/scripts/base/protocols/smtp/main.bro
@@ -8,33 +8,51 @@ export {
 	redef enum Log::ID += { LOG };
 
 	type Info: record {
+		## Time when the message was first seen.
 		ts:                time            &log;
+		## Unique ID for the connection.
 		uid:               string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:                conn_id         &log;
-		## This is a number that indicates the number of messages deep into
-		## this connection where this particular message was transferred.
+		## A count to represent the depth of this message transaction in a single 
+		## connection where multiple messages were transferred.
 		trans_depth:       count           &log;
+		## Contents of the Helo header.
 		helo:              string          &log &optional;
+		## Contents of the From header.
 		mailfrom:          string          &log &optional;
+		## Contents of the Rcpt header.
 		rcptto:            set[string]     &log &optional;
+		## Contents of the Date header.
 		date:              string          &log &optional;
+		## Contents of the From header.
 		from:              string          &log &optional;
+		## Contents of the To header.
 		to:                set[string]     &log &optional;
+		## Contents of the ReplyTo header.
 		reply_to:          string          &log &optional;
+		## Contents of the MsgID header.
 		msg_id:            string          &log &optional;
+		## Contents of the In-Reply-To header.
 		in_reply_to:       string          &log &optional;
+		## Contents of the Subject header.
 		subject:           string          &log &optional;
+		## Contents of the X-Origininating-IP header.
 		x_originating_ip:  addr            &log &optional;
+		## Contents of the first Received header.
 		first_received:    string          &log &optional;
+		## Contents of the second Received header.
 		second_received:   string          &log &optional;
-		## The last message the server sent to the client.
+		## The last message that the server sent to the client.
 		last_reply:        string          &log &optional;
+		## The message transmission path, as extracted from the headers.
 		path:              vector of addr  &log &optional;
+		## Value of the User-Agent header from the client.
 		user_agent:        string          &log &optional;
 		
-		## Indicate if the "Received: from" headers should still be processed.
+		## Indicates if the "Received: from" headers should still be processed.
 		process_received_from: bool        &default=T;
-		## Indicates if client activity has been seen, but not yet logged
+		## Indicates if client activity has been seen, but not yet logged.
 		has_client_activity:  bool            &default=F;
 	};
 	
diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro
index 052e666371..79ae4baa19 100644
--- a/scripts/base/protocols/socks/main.bro
+++ b/scripts/base/protocols/socks/main.bro
@@ -9,19 +9,21 @@ export {
 	type Info: record {
 		## Time when the proxy connection was first detected.
 		ts:          time            &log;
+		## Unique ID for the tunnel - may correspond to connection uid or be non-existent.
 		uid:         string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:          conn_id         &log;
 		## Protocol version of SOCKS.
 		version:     count           &log;
-		## Username for the proxy if extracted from the network.
+		## Username for the proxy if extracted from the network..
 		user:        string          &log &optional;
 		## Server status for the attempt at using the proxy.
 		status:      string          &log &optional;
-		## Client requested SOCKS address.  Could be an address, a name or both.
+		## Client requested SOCKS address. Could be an address, a name or both.
 		request:     SOCKS::Address  &log &optional;
 		## Client requested port.
 		request_p:   port            &log &optional;
-		## Server bound address.  Could be an address, a name or both.
+		## Server bound address. Could be an address, a name or both.
 		bound:       SOCKS::Address  &log &optional;
 		## Server bound port.
 		bound_p:     port            &log &optional;
diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro
index 0d3439bb1f..cd20f4e913 100644
--- a/scripts/base/protocols/ssh/main.bro
+++ b/scripts/base/protocols/ssh/main.bro
@@ -26,21 +26,23 @@ export {
 	type Info: record {
 		## Time when the SSH connection began.
 		ts:              time         &log;
+		## Unique ID for the connection.
 		uid:             string       &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
 		## Indicates if the login was heuristically guessed to be "success"
 		## or "failure".
 		status:          string       &log &optional;
 		## Direction of the connection.  If the client was a local host 
-		## logging into an external host, this would be OUTBOUD.  INBOUND
+		## logging into an external host, this would be OUTBOUND. INBOUND
 		## would be set for the opposite situation.
 		# TODO: handle local-local and remote-remote better.
 		direction:       Direction    &log &optional;
-		## Software string given by the client.
+		## Software string from the client.
 		client:          string       &log &optional;
-		## Software string given by the server.
+		## Software string from the server.
 		server:          string       &log &optional;
-		## Amount of data returned from the server.  This is currently
+		## Amount of data returned from the server. This is currently
 		## the only measure of the success heuristic and it is logged to 
 		## assist analysts looking at the logs to make their own determination
 		## about the success on a case-by-case basis.
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index b5f74d5122..f61e0d68ab 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -9,13 +9,15 @@ export {
 	redef enum Log::ID += { LOG };
 
 	type Info: record {
-		## Time when the SSL connection began.
+		## Time when the SSL connection was first detected.
 		ts:               time             &log;
-		uid:              string           &log;
+		## Unique ID for the connection.
+		uid:         string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:               conn_id          &log;
-		## SSL/TLS version the server offered.
+		## SSL/TLS version that the server offered.
 		version:          string           &log &optional;
-		## SSL/TLS cipher suite the server chose.
+		## SSL/TLS cipher suite that the server chose.
 		cipher:           string           &log &optional;
 		## Value of the Server Name Indicator SSL/TLS extension.  It 
 		## indicates the server name that the client was requesting.
diff --git a/scripts/base/protocols/syslog/main.bro b/scripts/base/protocols/syslog/main.bro
index 79f89d5e71..61334e3f2b 100644
--- a/scripts/base/protocols/syslog/main.bro
+++ b/scripts/base/protocols/syslog/main.bro
@@ -9,9 +9,11 @@ export {
 	redef enum Log::ID += { LOG };
 	
 	type Info: record {
-		## Timestamp of when the syslog message was seen.
+		## Timestamp when the syslog message was seen.
 		ts:        time            &log;
+		## Unique ID for the connection.
 		uid:       string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:        conn_id         &log;
 		## Protocol over which the message was seen.
 		proto:     transport_proto &log;

From 8279de25c98e54e526a7d7abacd4c69d28b3300a Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 13 Jul 2012 14:25:31 -0500
Subject: [PATCH 034/238] Remove baselines for some leak-detecting unit tests.

They were drifting from the non-leak-detecting unit tests and the point
of these was just leak detecting anyway, don't need the redundancy.
---
 .../btest/Baseline/core.leaks.ayiya/conn.log  | 15 ----
 .../btest/Baseline/core.leaks.ayiya/http.log  | 10 ---
 .../Baseline/core.leaks.ayiya/tunnel.log      | 11 ---
 .../btest/Baseline/core.leaks.teredo/conn.log | 28 -------
 .../btest/Baseline/core.leaks.teredo/http.log | 11 ---
 .../btest/Baseline/core.leaks.teredo/output   | 83 -------------------
 .../Baseline/core.leaks.teredo/tunnel.log     | 13 ---
 testing/btest/core/leaks/ayiya.test           |  3 -
 testing/btest/core/leaks/teredo.bro           |  4 -
 9 files changed, 178 deletions(-)
 delete mode 100644 testing/btest/Baseline/core.leaks.ayiya/conn.log
 delete mode 100644 testing/btest/Baseline/core.leaks.ayiya/http.log
 delete mode 100644 testing/btest/Baseline/core.leaks.ayiya/tunnel.log
 delete mode 100644 testing/btest/Baseline/core.leaks.teredo/conn.log
 delete mode 100644 testing/btest/Baseline/core.leaks.teredo/http.log
 delete mode 100644 testing/btest/Baseline/core.leaks.teredo/output
 delete mode 100644 testing/btest/Baseline/core.leaks.teredo/tunnel.log

diff --git a/testing/btest/Baseline/core.leaks.ayiya/conn.log b/testing/btest/Baseline/core.leaks.ayiya/conn.log
deleted file mode 100644
index 5c23b4c404..0000000000
--- a/testing/btest/Baseline/core.leaks.ayiya/conn.log
+++ /dev/null
@@ -1,15 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	conn
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1257655301.595604	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	tcp	http	2.101052	2981	4665	S1	-	0	ShADad	10	3605	11	5329	k6kgXLOoSKl
-1257655296.585034	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	udp	ayiya	20.879001	5129	6109	SF	-	0	Dd	21	5717	13	6473	(empty)
-1257655293.629048	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	udp	ayiya	-	-	-	SHR	-	0	d	0	0	1	176	(empty)
-1257655296.585333	FrJExwHcSal	::	135	ff02::1:ff00:2	136	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	k6kgXLOoSKl
-1257655293.629048	arKYeMETxOg	2001:4978:f:4c::1	128	2001:4978:f:4c::2	129	icmp	-	23.834987	168	56	OTH	-	0	-	3	312	1	104	UWkUyAuUGXf,k6kgXLOoSKl
-1257655296.585188	TEfuqmmG4bh	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff00:2	130	icmp	-	0.919988	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
-1257655296.585151	j4u32Pc5bif	fe80::216:cbff:fe9a:4cb9	131	ff02::2:f901:d225	130	icmp	-	0.719947	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
-1257655296.585034	nQcgTWjvg4c	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff9a:4cb9	130	icmp	-	4.922880	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
diff --git a/testing/btest/Baseline/core.leaks.ayiya/http.log b/testing/btest/Baseline/core.leaks.ayiya/http.log
deleted file mode 100644
index 7cef1a1b8e..0000000000
--- a/testing/btest/Baseline/core.leaks.ayiya/http.log
+++ /dev/null
@@ -1,10 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1257655301.652206	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	10102	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-1257655302.514424	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	2	GET	ipv6.google.com	/csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
-1257655303.603569	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	3	GET	ipv6.google.com	/gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
diff --git a/testing/btest/Baseline/core.leaks.ayiya/tunnel.log b/testing/btest/Baseline/core.leaks.ayiya/tunnel.log
deleted file mode 100644
index 512f49b6ee..0000000000
--- a/testing/btest/Baseline/core.leaks.ayiya/tunnel.log
+++ /dev/null
@@ -1,11 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	tunnel
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	action	tunnel_type
-#types	time	string	addr	port	addr	port	enum	enum
-1257655293.629048	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::DISCOVER	Tunnel::AYIYA
-1257655296.585034	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::DISCOVER	Tunnel::AYIYA
-1257655317.464035	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::CLOSE	Tunnel::AYIYA
-1257655317.464035	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::CLOSE	Tunnel::AYIYA
diff --git a/testing/btest/Baseline/core.leaks.teredo/conn.log b/testing/btest/Baseline/core.leaks.teredo/conn.log
deleted file mode 100644
index 151230886b..0000000000
--- a/testing/btest/Baseline/core.leaks.teredo/conn.log
+++ /dev/null
@@ -1,28 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	conn
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	parents
-#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
-1210953047.736921	arKYeMETxOg	192.168.2.16	1576	75.126.130.163	80	tcp	-	0.000357	0	0	SHR	-	0	fA	1	40	1	40	(empty)
-1210953050.867067	k6kgXLOoSKl	192.168.2.16	1577	75.126.203.78	80	tcp	-	0.000387	0	0	SHR	-	0	fA	1	40	1	40	(empty)
-1210953057.833364	5OKnoww6xl4	192.168.2.16	1577	75.126.203.78	80	tcp	-	0.079208	0	0	SH	-	0	Fa	1	40	1	40	(empty)
-1210953058.007081	VW0XPVINV8a	192.168.2.16	1576	75.126.130.163	80	tcp	-	-	-	-	RSTOS0	-	0	R	1	40	0	0	(empty)
-1210953057.834454	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	tcp	http	0.407908	790	171	RSTO	-	0	ShADadR	6	1038	4	335	(empty)
-1210953058.350065	fRFu0wcOle6	192.168.2.16	1920	192.168.2.1	53	udp	dns	0.223055	66	438	SF	-	0	Dd	2	122	2	494	(empty)
-1210953058.577231	qSsw6ESzHV4	192.168.2.16	137	192.168.2.255	137	udp	dns	1.499261	150	0	S0	-	0	D	3	234	0	0	(empty)
-1210953074.264819	Tw8jXtpTGu6	192.168.2.16	1920	192.168.2.1	53	udp	dns	0.297723	123	598	SF	-	0	Dd	3	207	3	682	(empty)
-1210953061.312379	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	tcp	http	12.810848	1675	10467	S1	-	0	ShADad	10	2279	12	11191	GSxOnSLghOa
-1210953076.058333	EAr0uf4mhq	192.168.2.16	1578	75.126.203.78	80	tcp	-	-	-	-	RSTRH	-	0	r	0	0	1	40	(empty)
-1210953074.055744	h5DsfNtYzi1	192.168.2.16	1577	75.126.203.78	80	tcp	-	-	-	-	RSTRH	-	0	r	0	0	1	40	(empty)
-1210953074.057124	P654jzLoe3a	192.168.2.16	1576	75.126.130.163	80	tcp	-	-	-	-	RSTRH	-	0	r	0	0	1	40	(empty)
-1210953074.570439	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	tcp	http	0.466677	469	3916	SF	-	0	ShADadFf	7	757	6	4164	(empty)
-1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	udp	teredo	8.928880	129	48	SF	-	0	Dd	2	185	1	76	(empty)
-1210953060.829233	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	udp	teredo	13.293994	2359	11243	SF	-	0	Dd	12	2695	13	11607	(empty)
-1210953058.933954	iE6yhOq3SF	0.0.0.0	68	255.255.255.255	67	udp	-	-	-	-	S0	-	0	D	1	328	0	0	(empty)
-1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	udp	teredo	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
-1210953046.591933	UWkUyAuUGXf	192.168.2.16	138	192.168.2.255	138	udp	-	28.448321	416	0	S0	-	0	D	2	472	0	0	(empty)
-1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
-1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c
-1210953052.202579	j4u32Pc5bif	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	nQcgTWjvg4c
diff --git a/testing/btest/Baseline/core.leaks.teredo/http.log b/testing/btest/Baseline/core.leaks.teredo/http.log
deleted file mode 100644
index b3cf832083..0000000000
--- a/testing/btest/Baseline/core.leaks.teredo/http.log
+++ /dev/null
@@ -1,11 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	http
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
-#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1210953057.917183	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	1	POST	download913.avast.com	/cgi-bin/iavs4stats.cgi	-	Syncer/4.80 (av_pro-1169;f)	589	0	204	<empty>	-	-	-	(empty)	-	-	-	text/plain	-	-
-1210953061.585996	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-1210953073.381474	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-1210953074.674817	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	1	GET	www.wireshark.org	/	http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	11845	200	OK	-	-	-	(empty)	-	-	-	text/xml	-	-
diff --git a/testing/btest/Baseline/core.leaks.teredo/output b/testing/btest/Baseline/core.leaks.teredo/output
deleted file mode 100644
index 02d5a41e74..0000000000
--- a/testing/btest/Baseline/core.leaks.teredo/output
+++ /dev/null
@@ -1,83 +0,0 @@
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.80, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=24, nxt=58, hlim=255, src=fe80::8000:ffff:ffff:fffd, dst=ff02::2, exts=[]]
-    auth: [id=, value=, nonce=14796129349558001544, confirm=0]
-auth: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.80, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=24, nxt=58, hlim=255, src=fe80::8000:ffff:ffff:fffd, dst=ff02::2, exts=[]]
-    auth: [id=, value=, nonce=14796129349558001544, confirm=0]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.81, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=48, nxt=58, hlim=255, src=fe80::8000:f227:bec8:61af, dst=fe80::8000:ffff:ffff:fffd, exts=[]]
-    auth: [id=, value=, nonce=14796129349558001544, confirm=0]
-    origin: [p=3797/udp, a=70.55.215.234]
-auth: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.81, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=48, nxt=58, hlim=255, src=fe80::8000:f227:bec8:61af, dst=fe80::8000:ffff:ffff:fffd, exts=[]]
-    auth: [id=, value=, nonce=14796129349558001544, confirm=0]
-    origin: [p=3797/udp, a=70.55.215.234]
-origin: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.81, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=48, nxt=58, hlim=255, src=fe80::8000:f227:bec8:61af, dst=fe80::8000:ffff:ffff:fffd, exts=[]]
-    auth: [id=, value=, nonce=14796129349558001544, confirm=0]
-    origin: [p=3797/udp, a=70.55.215.234]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=21, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-bubble: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=21, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.80, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=12, nxt=58, hlim=21, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.80, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=0, src=fe80::708d:fe83:4114:a512, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-    origin: [p=32900/udp, a=83.170.1.38]
-origin: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.80, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=0, src=fe80::708d:fe83:4114:a512, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-    origin: [p=32900/udp, a=83.170.1.38]
-bubble: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=65.55.158.80, resp_p=3544/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=0, src=fe80::708d:fe83:4114:a512, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-    origin: [p=32900/udp, a=83.170.1.38]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=0, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=fe80::708d:fe83:4114:a512, exts=[]]
-bubble: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=0, nxt=59, hlim=0, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=fe80::708d:fe83:4114:a512, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=12, nxt=58, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=24, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=24, nxt=6, hlim=245, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=817, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=514, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=898, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=812, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=1232, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=717, nxt=6, hlim=58, src=2001:4860:0:2001::68, dst=2001:0:4137:9e50:8000:f12a:b9c8:2815, exts=[]]
-packet: [orig_h=192.168.2.16, orig_p=3797/udp, resp_h=83.170.1.38, resp_p=32900/udp]
-    ip6: [class=0, flow=0, len=20, nxt=6, hlim=128, src=2001:0:4137:9e50:8000:f12a:b9c8:2815, dst=2001:4860:0:2001::68, exts=[]]
diff --git a/testing/btest/Baseline/core.leaks.teredo/tunnel.log b/testing/btest/Baseline/core.leaks.teredo/tunnel.log
deleted file mode 100644
index 5a2114dd1c..0000000000
--- a/testing/btest/Baseline/core.leaks.teredo/tunnel.log
+++ /dev/null
@@ -1,13 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	tunnel
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	action	tunnel_type
-#types	time	string	addr	port	addr	port	enum	enum
-1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::DISCOVER	Tunnel::TEREDO
-1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	Tunnel::DISCOVER	Tunnel::TEREDO
-1210953061.292918	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	Tunnel::DISCOVER	Tunnel::TEREDO
-1210953076.058333	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::CLOSE	Tunnel::TEREDO
-1210953076.058333	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	Tunnel::CLOSE	Tunnel::TEREDO
-1210953076.058333	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	Tunnel::CLOSE	Tunnel::TEREDO
diff --git a/testing/btest/core/leaks/ayiya.test b/testing/btest/core/leaks/ayiya.test
index adad42a822..2093924c7a 100644
--- a/testing/btest/core/leaks/ayiya.test
+++ b/testing/btest/core/leaks/ayiya.test
@@ -5,6 +5,3 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/tunnels/ayiya3.trace
-# @TEST-EXEC: btest-diff tunnel.log
-# @TEST-EXEC: btest-diff conn.log
-# @TEST-EXEC: btest-diff http.log
diff --git a/testing/btest/core/leaks/teredo.bro b/testing/btest/core/leaks/teredo.bro
index 9902f1258b..be298f4d68 100644
--- a/testing/btest/core/leaks/teredo.bro
+++ b/testing/btest/core/leaks/teredo.bro
@@ -5,10 +5,6 @@
 # @TEST-GROUP: leaks
 #
 # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/tunnels/Teredo.pcap %INPUT >output
-# @TEST-EXEC: btest-diff output
-# @TEST-EXEC: btest-diff tunnel.log
-# @TEST-EXEC: btest-diff conn.log
-# @TEST-EXEC: btest-diff http.log
 
 function print_teredo(name: string, outer: connection, inner: teredo_hdr)
 	{

From 353393f9bd9c43bab74b1ba4244d82e414b0698c Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 13 Jul 2012 14:32:50 -0500
Subject: [PATCH 035/238] Fix segfault when incrementing whole vector values.

Also removed RefExpr::Eval(Val*) method since it was never called
(Clang emitted warning about this hiding overloaded virtual function
UnaryExpr::Eval(Frame*)) and doesn't appear to be necessary even if it
was called to avoid the default vector handling of UnaryExpr::Eval
(as the comment suggests as the intention).
---
 src/Expr.cc                                   |  7 ----
 src/Expr.h                                    |  4 ---
 .../btest/Baseline/language.incr-vec-expr/out |  5 +++
 testing/btest/core/leaks/incr-vec-expr.test   | 35 +++++++++++++++++++
 testing/btest/language/incr-vec-expr.test     | 27 ++++++++++++++
 5 files changed, 67 insertions(+), 11 deletions(-)
 create mode 100644 testing/btest/Baseline/language.incr-vec-expr/out
 create mode 100644 testing/btest/core/leaks/incr-vec-expr.test
 create mode 100644 testing/btest/language/incr-vec-expr.test

diff --git a/src/Expr.cc b/src/Expr.cc
index 58f5db3fd1..b62f119bae 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -1035,12 +1035,10 @@ Val* IncrExpr::Eval(Frame* f) const
 				{
 				Val* new_elt = DoSingleEval(f, elt);
 				v_vec->Assign(i, new_elt, this, OP_INCR);
-				Unref(new_elt); // was Ref()'d by Assign()
 				}
 			else
 				v_vec->Assign(i, 0, this, OP_INCR);
 			}
-		// FIXME: Is the next line needed?
 		op->Assign(f, v_vec, OP_INCR);
 		}
 
@@ -2402,11 +2400,6 @@ Expr* RefExpr::MakeLvalue()
 	return this;
 	}
 
-Val* RefExpr::Eval(Val* v) const
-	{
-	return Fold(v);
-	}
-
 void RefExpr::Assign(Frame* f, Val* v, Opcode opcode)
 	{
 	op->Assign(f, v, opcode);
diff --git a/src/Expr.h b/src/Expr.h
index f0798359c2..c16cf86612 100644
--- a/src/Expr.h
+++ b/src/Expr.h
@@ -608,10 +608,6 @@ public:
 	void Assign(Frame* f, Val* v, Opcode op = OP_ASSIGN);
 	Expr* MakeLvalue();
 
-	// Only overridden to avoid special vector handling which doesn't apply
-	// for this class.
-	Val* Eval(Val* v) const;
-
 protected:
 	friend class Expr;
 	RefExpr()	{ }
diff --git a/testing/btest/Baseline/language.incr-vec-expr/out b/testing/btest/Baseline/language.incr-vec-expr/out
new file mode 100644
index 0000000000..b6c108a2d8
--- /dev/null
+++ b/testing/btest/Baseline/language.incr-vec-expr/out
@@ -0,0 +1,5 @@
+[0, 0, 0]
+[a=0, b=test, c=[1, 2, 3]]
+[1, 1, 1]
+[a=1, b=test, c=[1, 2, 3]]
+[a=1, b=test, c=[2, 3, 4]]
diff --git a/testing/btest/core/leaks/incr-vec-expr.test b/testing/btest/core/leaks/incr-vec-expr.test
new file mode 100644
index 0000000000..d2b94a5e63
--- /dev/null
+++ b/testing/btest/core/leaks/incr-vec-expr.test
@@ -0,0 +1,35 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT
+
+type rec: record {
+    a: count;
+    b: string;
+    c: vector of count;
+};
+
+global vec: vector of count = vector(0,0,0);
+
+global v: rec = [$a=0, $b="test", $c=vector(1,2,3)];
+
+event new_connection(c: connection)
+    {
+    print vec;
+    print v;
+
+    ++vec;
+
+    print vec;
+
+    ++v$a;
+
+    print v;
+
+    ++v$c;
+
+    print v;
+    }
diff --git a/testing/btest/language/incr-vec-expr.test b/testing/btest/language/incr-vec-expr.test
new file mode 100644
index 0000000000..c9945061a2
--- /dev/null
+++ b/testing/btest/language/incr-vec-expr.test
@@ -0,0 +1,27 @@
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+type rec: record {
+    a: count;
+    b: string;
+    c: vector of count;
+};
+
+global vec: vector of count = vector(0,0,0);
+
+global v: rec = [$a=0, $b="test", $c=vector(1,2,3)];
+
+print vec;
+print v;
+
+++vec;
+
+print vec;
+
+++v$a;
+
+print v;
+
+++v$c;
+
+print v;

From 0ef91538dbd5890dc7eaf265e74c0c3a85880000 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 13 Jul 2012 16:25:58 -0500
Subject: [PATCH 036/238] Fix overrides of
 TCP_ApplicationAnalyzer::EndpointEOF.

In many cases, classes derived from TCP_ApplicationAnalyzer were
*overloading* instead of overriding EndpointEOF causing the parent
class version to become hidden in the child and also for the child's
version to never to called polymorphically from
TCP_Analyzer::EndpointEOF.  Clang gave a warning in each case.
---
 src/BitTorrent.cc        | 6 +++---
 src/BitTorrent.h         | 2 +-
 src/BitTorrentTracker.cc | 4 ++--
 src/BitTorrentTracker.h  | 2 +-
 src/DNS-binpac.cc        | 6 +++---
 src/DNS-binpac.h         | 2 +-
 src/HTTP-binpac.cc       | 6 +++---
 src/HTTP-binpac.h        | 2 +-
 src/SOCKS.cc             | 6 +++---
 src/SOCKS.h              | 2 +-
 src/SSL.cc               | 6 +++---
 src/SSL.h                | 2 +-
 12 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/src/BitTorrent.cc b/src/BitTorrent.cc
index 824e4ec98d..fa8fb09e43 100644
--- a/src/BitTorrent.cc
+++ b/src/BitTorrent.cc
@@ -106,10 +106,10 @@ void BitTorrent_Analyzer::Undelivered(int seq, int len, bool orig)
 	//	}
 	}
 
-void BitTorrent_Analyzer::EndpointEOF(TCP_Reassembler* endp)
+void BitTorrent_Analyzer::EndpointEOF(bool is_orig)
 	{
-	TCP_ApplicationAnalyzer::EndpointEOF(endp);
-	interp->FlowEOF(endp->IsOrig());
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
 	}
 
 void BitTorrent_Analyzer::DeliverWeird(const char* msg, bool orig)
diff --git a/src/BitTorrent.h b/src/BitTorrent.h
index 191b4c50d7..f083cf4fc7 100644
--- a/src/BitTorrent.h
+++ b/src/BitTorrent.h
@@ -15,7 +15,7 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(int seq, int len, bool orig);
-	virtual void EndpointEOF(TCP_Reassembler* endp);
+	virtual void EndpointEOF(bool is_orig);
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new BitTorrent_Analyzer(conn); }
diff --git a/src/BitTorrentTracker.cc b/src/BitTorrentTracker.cc
index 995a01dd63..12c5a199de 100644
--- a/src/BitTorrentTracker.cc
+++ b/src/BitTorrentTracker.cc
@@ -215,9 +215,9 @@ void BitTorrentTracker_Analyzer::Undelivered(int seq, int len, bool orig)
 		stop_resp = true;
 	}
 
-void BitTorrentTracker_Analyzer::EndpointEOF(TCP_Reassembler* endp)
+void BitTorrentTracker_Analyzer::EndpointEOF(bool is_orig)
 	{
-	TCP_ApplicationAnalyzer::EndpointEOF(endp);
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
 	}
 
 void BitTorrentTracker_Analyzer::InitBencParser(void)
diff --git a/src/BitTorrentTracker.h b/src/BitTorrentTracker.h
index d57665d104..3b9efe0430 100644
--- a/src/BitTorrentTracker.h
+++ b/src/BitTorrentTracker.h
@@ -48,7 +48,7 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(int seq, int len, bool orig);
-	virtual void EndpointEOF(TCP_Reassembler* endp);
+	virtual void EndpointEOF(bool is_orig);
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new BitTorrentTracker_Analyzer(conn); }
diff --git a/src/DNS-binpac.cc b/src/DNS-binpac.cc
index eb95ac2e1c..999f6015c0 100644
--- a/src/DNS-binpac.cc
+++ b/src/DNS-binpac.cc
@@ -63,10 +63,10 @@ void DNS_TCP_Analyzer_binpac::Done()
 	interp->FlowEOF(false);
 	}
 
-void DNS_TCP_Analyzer_binpac::EndpointEOF(TCP_Reassembler* endp)
+void DNS_TCP_Analyzer_binpac::EndpointEOF(bool is_orig)
 	{
-	TCP_ApplicationAnalyzer::EndpointEOF(endp);
-	interp->FlowEOF(endp->IsOrig());
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
 	}
 
 void DNS_TCP_Analyzer_binpac::DeliverStream(int len, const u_char* data,
diff --git a/src/DNS-binpac.h b/src/DNS-binpac.h
index 9e8cb16f69..0bbacf9192 100644
--- a/src/DNS-binpac.h
+++ b/src/DNS-binpac.h
@@ -45,7 +45,7 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(int seq, int len, bool orig);
-	virtual void EndpointEOF(TCP_Reassembler* endp);
+	virtual void EndpointEOF(bool is_orig);
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new DNS_TCP_Analyzer_binpac(conn); }
diff --git a/src/HTTP-binpac.cc b/src/HTTP-binpac.cc
index 70cf37457b..47b2c479ec 100644
--- a/src/HTTP-binpac.cc
+++ b/src/HTTP-binpac.cc
@@ -20,10 +20,10 @@ void HTTP_Analyzer_binpac::Done()
 	interp->FlowEOF(false);
 	}
 
-void HTTP_Analyzer_binpac::EndpointEOF(TCP_Reassembler* endp)
+void HTTP_Analyzer_binpac::EndpointEOF(bool is_orig)
 	{
-	TCP_ApplicationAnalyzer::EndpointEOF(endp);
-	interp->FlowEOF(endp->IsOrig());
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
 	}
 
 void HTTP_Analyzer_binpac::DeliverStream(int len, const u_char* data, bool orig)
diff --git a/src/HTTP-binpac.h b/src/HTTP-binpac.h
index 62b6fd0db3..ef7cc7dd7d 100644
--- a/src/HTTP-binpac.h
+++ b/src/HTTP-binpac.h
@@ -13,7 +13,7 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(int seq, int len, bool orig);
-	virtual void EndpointEOF(TCP_Reassembler* endp);
+	virtual void EndpointEOF(bool is_orig);
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new HTTP_Analyzer_binpac(conn); }
diff --git a/src/SOCKS.cc b/src/SOCKS.cc
index 02429aa208..4a6eda7043 100644
--- a/src/SOCKS.cc
+++ b/src/SOCKS.cc
@@ -31,10 +31,10 @@ void SOCKS_Analyzer::Done()
 	interp->FlowEOF(false);
 	}
 
-void SOCKS_Analyzer::EndpointEOF(TCP_Reassembler* endp)
+void SOCKS_Analyzer::EndpointEOF(bool is_orig)
 	{
-	TCP_ApplicationAnalyzer::EndpointEOF(endp);
-	interp->FlowEOF(endp->IsOrig());
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
 	}
 
 void SOCKS_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
diff --git a/src/SOCKS.h b/src/SOCKS.h
index c9a7338496..9753abb660 100644
--- a/src/SOCKS.h
+++ b/src/SOCKS.h
@@ -23,7 +23,7 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 	virtual void Undelivered(int seq, int len, bool orig);
-	virtual void EndpointEOF(TCP_Reassembler* endp);
+	virtual void EndpointEOF(bool is_orig);
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new SOCKS_Analyzer(conn); }
diff --git a/src/SSL.cc b/src/SSL.cc
index 218b17080b..4658bbbc16 100644
--- a/src/SSL.cc
+++ b/src/SSL.cc
@@ -23,10 +23,10 @@ void SSL_Analyzer::Done()
 	interp->FlowEOF(false);
 	}
 
-void SSL_Analyzer::EndpointEOF(TCP_Reassembler* endp)
+void SSL_Analyzer::EndpointEOF(bool is_orig)
 	{
-	TCP_ApplicationAnalyzer::EndpointEOF(endp);
-	interp->FlowEOF(endp->IsOrig());
+	TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
 	}
 
 void SSL_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
diff --git a/src/SSL.h b/src/SSL.h
index c9f8d9be91..d0ef164877 100644
--- a/src/SSL.h
+++ b/src/SSL.h
@@ -15,7 +15,7 @@ public:
 	virtual void Undelivered(int seq, int len, bool orig);
 
 	// Overriden from TCP_ApplicationAnalyzer.
-	virtual void EndpointEOF(TCP_Reassembler* endp);
+	virtual void EndpointEOF(bool is_orig);
 
 	static Analyzer* InstantiateAnalyzer(Connection* conn)
 		{ return new SSL_Analyzer(conn); }

From ce05600a717e31f36170d6c47dabd91bd914cd2d Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 13 Jul 2012 22:24:34 -0400
Subject: [PATCH 037/238] Mozilla's current certificate bundle.

---
 scripts/base/protocols/ssl/mozilla-ca-list.bro | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/scripts/base/protocols/ssl/mozilla-ca-list.bro b/scripts/base/protocols/ssl/mozilla-ca-list.bro
index 4c4dccb755..ad8e445912 100644
--- a/scripts/base/protocols/ssl/mozilla-ca-list.bro
+++ b/scripts/base/protocols/ssl/mozilla-ca-list.bro
@@ -1,5 +1,5 @@
 # Don't edit!  This file is automatically generated.
-# Generated at: 2011-10-25 11:03:20 -0500
+# Generated at: Fri Jul 13 22:22:40 -0400 2012
 @load base/protocols/ssl
 module SSL;
 redef root_certs += {
@@ -11,7 +11,6 @@ redef root_certs += {
 	["OU=DSTCA E2,O=Digital Signature Trust Co.,C=US"] = "\x30\x82\x03\x29\x30\x82\x02\x92\xA0\x03\x02\x01\x02\x02\x04\x36\x6E\xD3\xCE\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x46\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x24\x30\x22\x06\x03\x55\x04\x0A\x13\x1B\x44\x69\x67\x69\x74\x61\x6C\x20\x53\x69\x67\x6E\x61\x74\x75\x72\x65\x20\x54\x72\x75\x73\x74\x20\x43\x6F\x2E\x31\x11\x30\x0F\x06\x03\x55\x04\x0B\x13\x08\x44\x53\x54\x43\x41\x20\x45\x32\x30\x1E\x17\x0D\x39\x38\x31\x32\x30\x39\x31\x39\x31\x37\x32\x36\x5A\x17\x0D\x31\x38\x31\x32\x30\x39\x31\x39\x34\x37\x32\x36\x5A\x30\x46\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x24\x30\x22\x06\x03\x55\x04\x0A\x13\x1B\x44\x69\x67\x69\x74\x61\x6C\x20\x53\x69\x67\x6E\x61\x74\x75\x72\x65\x20\x54\x72\x75\x73\x74\x20\x43\x6F\x2E\x31\x11\x30\x0F\x06\x03\x55\x04\x0B\x13\x08\x44\x53\x54\x43\x41\x20\x45\x32\x30\x81\x9D\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8B\x00\x30\x81\x87\x02\x81\x81\x00\xBF\x93\x8F\x17\x92\xEF\x33\x13\x18\xEB\x10\x7F\x4E\x16\xBF\xFF\x06\x8F\x2A\x85\xBC\x5E\xF9\x24\xA6\x24\x88\xB6\x03\xB7\xC1\xC3\x5F\x03\x5B\xD1\x6F\xAE\x7E\x42\xEA\x66\x23\xB8\x63\x83\x56\xFB\x28\x2D\xE1\x38\x8B\xB4\xEE\xA8\x01\xE1\xCE\x1C\xB6\x88\x2A\x22\x46\x85\xFB\x9F\xA7\x70\xA9\x47\x14\x3F\xCE\xDE\x65\xF0\xA8\x71\xF7\x4F\x26\x6C\x8C\xBC\xC6\xB5\xEF\xDE\x49\x27\xFF\x48\x2A\x7D\xE8\x4D\x03\xCC\xC7\xB2\x52\xC6\x17\x31\x13\x3B\xB5\x4D\xDB\xC8\xC4\xF6\xC3\x0F\x24\x2A\xDA\x0C\x9D\xE7\x91\x5B\x80\xCD\x94\x9D\x02\x01\x03\xA3\x82\x01\x24\x30\x82\x01\x20\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x68\x06\x03\x55\x1D\x1F\x04\x61\x30\x5F\x30\x5D\xA0\x5B\xA0\x59\xA4\x57\x30\x55\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x24\x30\x22\x06\x03\x55\x04\x0A\x13\x1B\x44\x69\x67\x69\x74\x61\x6C\x20\x53\x69\x67\x6E\x61\x74\x75\x72\x65\x20\x54\x72\x75\x73\x74\x20\x43\x6F\x2E\x31\x11\x30\x0F\x06\x03\x55\x04\x0B\x13\x08\x44\x53\x54\x43\x41\x20\x45\x32\x31\x0D\x30\x0B\x06\x03\x55\x04\x03\x13\x04\x43\x52\x4C\x31\x30\x2B\x06\x03\x55\x1D\x10\x04\x24\x30\x22\x80\x0F\x31\x39\x39\x38\x31\x32\x30\x39\x31\x39\x31\x37\x32\x36\x5A\x81\x0F\x32\x30\x31\x38\x31\x32\x30\x39\x31\x39\x31\x37\x32\x36\x5A\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\x06\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x1E\x82\x4D\x28\x65\x80\x3C\xC9\x41\x6E\xAC\x35\x2E\x5A\xCB\xDE\xEE\xF8\x39\x5B\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x1E\x82\x4D\x28\x65\x80\x3C\xC9\x41\x6E\xAC\x35\x2E\x5A\xCB\xDE\xEE\xF8\x39\x5B\x30\x0C\x06\x03\x55\x1D\x13\x04\x05\x30\x03\x01\x01\xFF\x30\x19\x06\x09\x2A\x86\x48\x86\xF6\x7D\x07\x41\x00\x04\x0C\x30\x0A\x1B\x04\x56\x34\x2E\x30\x03\x02\x04\x90\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x81\x81\x00\x47\x8D\x83\xAD\x62\xF2\xDB\xB0\x9E\x45\x22\x05\xB9\xA2\xD6\x03\x0E\x38\x72\xE7\x9E\xFC\x7B\xE6\x93\xB6\x9A\xA5\xA2\x94\xC8\x34\x1D\x91\xD1\xC5\xD7\xF4\x0A\x25\x0F\x3D\x78\x81\x9E\x0F\xB1\x67\xC4\x90\x4C\x63\xDD\x5E\xA7\xE2\xBA\x9F\xF5\xF7\x4D\xA5\x31\x7B\x9C\x29\x2D\x4C\xFE\x64\x3E\xEC\xB6\x53\xFE\xEA\x9B\xED\x82\xDB\x74\x75\x4B\x07\x79\x6E\x1E\xD8\x19\x83\x73\xDE\xF5\x3E\xD0\xB5\xDE\xE7\x4B\x68\x7D\x43\x2E\x2A\x20\xE1\x7E\xA0\x78\x44\x9E\x08\xF5\x98\xF9\xC7\x7F\x1B\x1B\xD6\x06\x20\x02\x58\xA1\xC3\xA2\x03",
 	["OU=Class 3 Public Primary Certification Authority,O=VeriSign\, Inc.,C=US"] = "\x30\x82\x02\x3C\x30\x82\x01\xA5\x02\x10\x70\xBA\xE4\x1D\x10\xD9\x29\x34\xB6\x38\xCA\x7B\x03\xCC\xBA\xBF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x02\x05\x00\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x37\x30\x35\x06\x03\x55\x04\x0B\x13\x2E\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x1E\x17\x0D\x39\x36\x30\x31\x32\x39\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x38\x30\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x37\x30\x35\x06\x03\x55\x04\x0B\x13\x2E\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xC9\x5C\x59\x9E\xF2\x1B\x8A\x01\x14\xB4\x10\xDF\x04\x40\xDB\xE3\x57\xAF\x6A\x45\x40\x8F\x84\x0C\x0B\xD1\x33\xD9\xD9\x11\xCF\xEE\x02\x58\x1F\x25\xF7\x2A\xA8\x44\x05\xAA\xEC\x03\x1F\x78\x7F\x9E\x93\xB9\x9A\x00\xAA\x23\x7D\xD6\xAC\x85\xA2\x63\x45\xC7\x72\x27\xCC\xF4\x4C\xC6\x75\x71\xD2\x39\xEF\x4F\x42\xF0\x75\xDF\x0A\x90\xC6\x8E\x20\x6F\x98\x0F\xF8\xAC\x23\x5F\x70\x29\x36\xA4\xC9\x86\xE7\xB1\x9A\x20\xCB\x53\xA5\x85\xE7\x3D\xBE\x7D\x9A\xFE\x24\x45\x33\xDC\x76\x15\xED\x0F\xA2\x71\x64\x4C\x65\x2E\x81\x68\x45\xA7\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x02\x05\x00\x03\x81\x81\x00\xBB\x4C\x12\x2B\xCF\x2C\x26\x00\x4F\x14\x13\xDD\xA6\xFB\xFC\x0A\x11\x84\x8C\xF3\x28\x1C\x67\x92\x2F\x7C\xB6\xC5\xFA\xDF\xF0\xE8\x95\xBC\x1D\x8F\x6C\x2C\xA8\x51\xCC\x73\xD8\xA4\xC0\x53\xF0\x4E\xD6\x26\xC0\x76\x01\x57\x81\x92\x5E\x21\xF1\xD1\xB1\xFF\xE7\xD0\x21\x58\xCD\x69\x17\xE3\x44\x1C\x9C\x19\x44\x39\x89\x5C\xDC\x9C\x00\x0F\x56\x8D\x02\x99\xED\xA2\x90\x45\x4C\xE4\xBB\x10\xA4\x3D\xF0\x32\x03\x0E\xF1\xCE\xF8\xE8\xC9\x51\x8C\xE6\x62\x9F\xE6\x9F\xC0\x7D\xB7\x72\x9C\xC9\x36\x3A\x6B\x9F\x4E\xA8\xFF\x64\x0D\x64",
 	["OU=VeriSign Trust Network,OU=(c) 1998 VeriSign\, Inc. - For authorized use only,OU=Class 3 Public Primary Certification Authority - G2,O=VeriSign\, Inc.,C=US"] = "\x30\x82\x03\x02\x30\x82\x02\x6B\x02\x10\x7D\xD9\xFE\x07\xCF\xA8\x1E\xB7\x10\x79\x67\xFB\xA7\x89\x34\xC6\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\xC1\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x3C\x30\x3A\x06\x03\x55\x04\x0B\x13\x33\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x2D\x20\x47\x32\x31\x3A\x30\x38\x06\x03\x55\x04\x0B\x13\x31\x28\x63\x29\x20\x31\x39\x39\x38\x20\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x20\x2D\x20\x46\x6F\x72\x20\x61\x75\x74\x68\x6F\x72\x69\x7A\x65\x64\x20\x75\x73\x65\x20\x6F\x6E\x6C\x79\x31\x1F\x30\x1D\x06\x03\x55\x04\x0B\x13\x16\x56\x65\x72\x69\x53\x69\x67\x6E\x20\x54\x72\x75\x73\x74\x20\x4E\x65\x74\x77\x6F\x72\x6B\x30\x1E\x17\x0D\x39\x38\x30\x35\x31\x38\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x38\x30\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x81\xC1\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x3C\x30\x3A\x06\x03\x55\x04\x0B\x13\x33\x43\x6C\x61\x73\x73\x20\x33\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x2D\x20\x47\x32\x31\x3A\x30\x38\x06\x03\x55\x04\x0B\x13\x31\x28\x63\x29\x20\x31\x39\x39\x38\x20\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x20\x2D\x20\x46\x6F\x72\x20\x61\x75\x74\x68\x6F\x72\x69\x7A\x65\x64\x20\x75\x73\x65\x20\x6F\x6E\x6C\x79\x31\x1F\x30\x1D\x06\x03\x55\x04\x0B\x13\x16\x56\x65\x72\x69\x53\x69\x67\x6E\x20\x54\x72\x75\x73\x74\x20\x4E\x65\x74\x77\x6F\x72\x6B\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xCC\x5E\xD1\x11\x5D\x5C\x69\xD0\xAB\xD3\xB9\x6A\x4C\x99\x1F\x59\x98\x30\x8E\x16\x85\x20\x46\x6D\x47\x3F\xD4\x85\x20\x84\xE1\x6D\xB3\xF8\xA4\xED\x0C\xF1\x17\x0F\x3B\xF9\xA7\xF9\x25\xD7\xC1\xCF\x84\x63\xF2\x7C\x63\xCF\xA2\x47\xF2\xC6\x5B\x33\x8E\x64\x40\x04\x68\xC1\x80\xB9\x64\x1C\x45\x77\xC7\xD8\x6E\xF5\x95\x29\x3C\x50\xE8\x34\xD7\x78\x1F\xA8\xBA\x6D\x43\x91\x95\x8F\x45\x57\x5E\x7E\xC5\xFB\xCA\xA4\x04\xEB\xEA\x97\x37\x54\x30\x6F\xBB\x01\x47\x32\x33\xCD\xDC\x57\x9B\x64\x69\x61\xF8\x9B\x1D\x1C\x89\x4F\x5C\x67\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x81\x81\x00\x51\x4D\xCD\xBE\x5C\xCB\x98\x19\x9C\x15\xB2\x01\x39\x78\x2E\x4D\x0F\x67\x70\x70\x99\xC6\x10\x5A\x94\xA4\x53\x4D\x54\x6D\x2B\xAF\x0D\x5D\x40\x8B\x64\xD3\xD7\xEE\xDE\x56\x61\x92\x5F\xA6\xC4\x1D\x10\x61\x36\xD3\x2C\x27\x3C\xE8\x29\x09\xB9\x11\x64\x74\xCC\xB5\x73\x9F\x1C\x48\xA9\xBC\x61\x01\xEE\xE2\x17\xA6\x0C\xE3\x40\x08\x3B\x0E\xE7\xEB\x44\x73\x2A\x9A\xF1\x69\x92\xEF\x71\x14\xC3\x39\xAC\x71\xA7\x91\x09\x6F\xE4\x71\x06\xB3\xBA\x59\x57\x26\x79\x00\xF6\xF8\x0D\xA2\x33\x30\x28\xD4\xAA\x58\xA0\x9D\x9D\x69\x91\xFD",
-	["OU=VeriSign Trust Network,OU=(c) 1998 VeriSign\, Inc. - For authorized use only,OU=Class 4 Public Primary Certification Authority - G2,O=VeriSign\, Inc.,C=US"] = "\x30\x82\x03\x02\x30\x82\x02\x6B\x02\x10\x32\x88\x8E\x9A\xD2\xF5\xEB\x13\x47\xF8\x7F\xC4\x20\x37\x25\xF8\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\xC1\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x3C\x30\x3A\x06\x03\x55\x04\x0B\x13\x33\x43\x6C\x61\x73\x73\x20\x34\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x2D\x20\x47\x32\x31\x3A\x30\x38\x06\x03\x55\x04\x0B\x13\x31\x28\x63\x29\x20\x31\x39\x39\x38\x20\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x20\x2D\x20\x46\x6F\x72\x20\x61\x75\x74\x68\x6F\x72\x69\x7A\x65\x64\x20\x75\x73\x65\x20\x6F\x6E\x6C\x79\x31\x1F\x30\x1D\x06\x03\x55\x04\x0B\x13\x16\x56\x65\x72\x69\x53\x69\x67\x6E\x20\x54\x72\x75\x73\x74\x20\x4E\x65\x74\x77\x6F\x72\x6B\x30\x1E\x17\x0D\x39\x38\x30\x35\x31\x38\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x38\x30\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x81\xC1\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x31\x3C\x30\x3A\x06\x03\x55\x04\x0B\x13\x33\x43\x6C\x61\x73\x73\x20\x34\x20\x50\x75\x62\x6C\x69\x63\x20\x50\x72\x69\x6D\x61\x72\x79\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x2D\x20\x47\x32\x31\x3A\x30\x38\x06\x03\x55\x04\x0B\x13\x31\x28\x63\x29\x20\x31\x39\x39\x38\x20\x56\x65\x72\x69\x53\x69\x67\x6E\x2C\x20\x49\x6E\x63\x2E\x20\x2D\x20\x46\x6F\x72\x20\x61\x75\x74\x68\x6F\x72\x69\x7A\x65\x64\x20\x75\x73\x65\x20\x6F\x6E\x6C\x79\x31\x1F\x30\x1D\x06\x03\x55\x04\x0B\x13\x16\x56\x65\x72\x69\x53\x69\x67\x6E\x20\x54\x72\x75\x73\x74\x20\x4E\x65\x74\x77\x6F\x72\x6B\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xBA\xF0\xE4\xCF\xF9\xC4\xAE\x85\x54\xB9\x07\x57\xF9\x8F\xC5\x7F\x68\x11\xF8\xC4\x17\xB0\x44\xDC\xE3\x30\x73\xD5\x2A\x62\x2A\xB8\xD0\xCC\x1C\xED\x28\x5B\x7E\xBD\x6A\xDC\xB3\x91\x24\xCA\x41\x62\x3C\xFC\x02\x01\xBF\x1C\x16\x31\x94\x05\x97\x76\x6E\xA2\xAD\xBD\x61\x17\x6C\x4E\x30\x86\xF0\x51\x37\x2A\x50\xC7\xA8\x62\x81\xDC\x5B\x4A\xAA\xC1\xA0\xB4\x6E\xEB\x2F\xE5\x57\xC5\xB1\x2B\x40\x70\xDB\x5A\x4D\xA1\x8E\x1F\xBD\x03\x1F\xD8\x03\xD4\x8F\x4C\x99\x71\xBC\xE2\x82\xCC\x58\xE8\x98\x3A\x86\xD3\x86\x38\xF3\x00\x29\x1F\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x81\x81\x00\x85\x8C\x12\xC1\xA7\xB9\x50\x15\x7A\xCB\x3E\xAC\xB8\x43\x8A\xDC\xAA\xDD\x14\xBA\x89\x81\x7E\x01\x3C\x23\x71\x21\x88\x2F\x82\xDC\x63\xFA\x02\x45\xAC\x45\x59\xD7\x2A\x58\x44\x5B\xB7\x9F\x81\x3B\x92\x68\x3D\xE2\x37\x24\xF5\x7B\x6C\x8F\x76\x35\x96\x09\xA8\x59\x9D\xB9\xCE\x23\xAB\x74\xD6\x83\xFD\x32\x73\x27\xD8\x69\x3E\x43\x74\xF6\xAE\xC5\x89\x9A\xE7\x53\x7C\xE9\x7B\xF6\x4B\xF3\xC1\x65\x83\xDE\x8D\x8A\x9C\x3C\x88\x8D\x39\x59\xFC\xAA\x3F\x22\x8D\xA1\xC1\x66\x50\x81\x72\x4C\xED\x22\x64\x4F\x4F\xCA\x80\x91\xB6\x29",
 	["CN=GlobalSign Root CA,OU=Root CA,O=GlobalSign nv-sa,C=BE"] = "\x30\x82\x03\x75\x30\x82\x02\x5D\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x15\x4B\x5A\xC3\x94\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x39\x38\x30\x39\x30\x31\x31\x32\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x30\x31\x32\x38\x31\x32\x30\x30\x30\x30\x5A\x30\x57\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x42\x45\x31\x19\x30\x17\x06\x03\x55\x04\x0A\x13\x10\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x6E\x76\x2D\x73\x61\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x13\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDA\x0E\xE6\x99\x8D\xCE\xA3\xE3\x4F\x8A\x7E\xFB\xF1\x8B\x83\x25\x6B\xEA\x48\x1F\xF1\x2A\xB0\xB9\x95\x11\x04\xBD\xF0\x63\xD1\xE2\x67\x66\xCF\x1C\xDD\xCF\x1B\x48\x2B\xEE\x8D\x89\x8E\x9A\xAF\x29\x80\x65\xAB\xE9\xC7\x2D\x12\xCB\xAB\x1C\x4C\x70\x07\xA1\x3D\x0A\x30\xCD\x15\x8D\x4F\xF8\xDD\xD4\x8C\x50\x15\x1C\xEF\x50\xEE\xC4\x2E\xF7\xFC\xE9\x52\xF2\x91\x7D\xE0\x6D\xD5\x35\x30\x8E\x5E\x43\x73\xF2\x41\xE9\xD5\x6A\xE3\xB2\x89\x3A\x56\x39\x38\x6F\x06\x3C\x88\x69\x5B\x2A\x4D\xC5\xA7\x54\xB8\x6C\x89\xCC\x9B\xF9\x3C\xCA\xE5\xFD\x89\xF5\x12\x3C\x92\x78\x96\xD6\xDC\x74\x6E\x93\x44\x61\xD1\x8D\xC7\x46\xB2\x75\x0E\x86\xE8\x19\x8A\xD5\x6D\x6C\xD5\x78\x16\x95\xA2\xE9\xC8\x0A\x38\xEB\xF2\x24\x13\x4F\x73\x54\x93\x13\x85\x3A\x1B\xBC\x1E\x34\xB5\x8B\x05\x8C\xB9\x77\x8B\xB1\xDB\x1F\x20\x91\xAB\x09\x53\x6E\x90\xCE\x7B\x37\x74\xB9\x70\x47\x91\x22\x51\x63\x16\x79\xAE\xB1\xAE\x41\x26\x08\xC8\x19\x2B\xD1\x46\xAA\x48\xD6\x64\x2A\xD7\x83\x34\xFF\x2C\x2A\xC1\x6C\x19\x43\x4A\x07\x85\xE7\xD3\x7C\xF6\x21\x68\xEF\xEA\xF2\x52\x9F\x7F\x93\x90\xCF\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\x7B\x66\x1A\x45\x0D\x97\xCA\x89\x50\x2F\x7D\x04\xCD\x34\xA8\xFF\xFC\xFD\x4B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xD6\x73\xE7\x7C\x4F\x76\xD0\x8D\xBF\xEC\xBA\xA2\xBE\x34\xC5\x28\x32\xB5\x7C\xFC\x6C\x9C\x2C\x2B\xBD\x09\x9E\x53\xBF\x6B\x5E\xAA\x11\x48\xB6\xE5\x08\xA3\xB3\xCA\x3D\x61\x4D\xD3\x46\x09\xB3\x3E\xC3\xA0\xE3\x63\x55\x1B\xF2\xBA\xEF\xAD\x39\xE1\x43\xB9\x38\xA3\xE6\x2F\x8A\x26\x3B\xEF\xA0\x50\x56\xF9\xC6\x0A\xFD\x38\xCD\xC4\x0B\x70\x51\x94\x97\x98\x04\xDF\xC3\x5F\x94\xD5\x15\xC9\x14\x41\x9C\xC4\x5D\x75\x64\x15\x0D\xFF\x55\x30\xEC\x86\x8F\xFF\x0D\xEF\x2C\xB9\x63\x46\xF6\xAA\xFC\xDF\xBC\x69\xFD\x2E\x12\x48\x64\x9A\xE0\x95\xF0\xA6\xEF\x29\x8F\x01\xB1\x15\xB5\x0C\x1D\xA5\xFE\x69\x2C\x69\x24\x78\x1E\xB3\xA7\x1C\x71\x62\xEE\xCA\xC8\x97\xAC\x17\x5D\x8A\xC2\xF8\x47\x86\x6E\x2A\xC4\x56\x31\x95\xD0\x67\x89\x85\x2B\xF9\x6C\xA6\x5D\x46\x9D\x0C\xAA\x82\xE4\x99\x51\xDD\x70\xB7\xDB\x56\x3D\x61\xE4\x6A\xE1\x5C\xD6\xF6\xFE\x3D\xDE\x41\xCC\x07\xAE\x63\x52\xBF\x53\x53\xF4\x2B\xE9\xC7\xFD\xB6\xF7\x82\x5F\x85\xD2\x41\x18\xDB\x81\xB3\x04\x1C\xC5\x1F\xA4\x80\x6F\x15\x20\xC9\xDE\x0C\x88\x0A\x1D\xD6\x66\x55\xE2\xFC\x48\xC9\x29\x26\x69\xE0",
 	["CN=GlobalSign,O=GlobalSign,OU=GlobalSign Root CA - R2"] = "\x30\x82\x03\xBA\x30\x82\x02\xA2\xA0\x03\x02\x01\x02\x02\x0B\x04\x00\x00\x00\x00\x01\x0F\x86\x26\xE6\x0D\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x4C\x31\x20\x30\x1E\x06\x03\x55\x04\x0B\x13\x17\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x2D\x20\x52\x32\x31\x13\x30\x11\x06\x03\x55\x04\x0A\x13\x0A\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x31\x13\x30\x11\x06\x03\x55\x04\x03\x13\x0A\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x30\x1E\x17\x0D\x30\x36\x31\x32\x31\x35\x30\x38\x30\x30\x30\x30\x5A\x17\x0D\x32\x31\x31\x32\x31\x35\x30\x38\x30\x30\x30\x30\x5A\x30\x4C\x31\x20\x30\x1E\x06\x03\x55\x04\x0B\x13\x17\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x20\x2D\x20\x52\x32\x31\x13\x30\x11\x06\x03\x55\x04\x0A\x13\x0A\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x31\x13\x30\x11\x06\x03\x55\x04\x03\x13\x0A\x47\x6C\x6F\x62\x61\x6C\x53\x69\x67\x6E\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xA6\xCF\x24\x0E\xBE\x2E\x6F\x28\x99\x45\x42\xC4\xAB\x3E\x21\x54\x9B\x0B\xD3\x7F\x84\x70\xFA\x12\xB3\xCB\xBF\x87\x5F\xC6\x7F\x86\xD3\xB2\x30\x5C\xD6\xFD\xAD\xF1\x7B\xDC\xE5\xF8\x60\x96\x09\x92\x10\xF5\xD0\x53\xDE\xFB\x7B\x7E\x73\x88\xAC\x52\x88\x7B\x4A\xA6\xCA\x49\xA6\x5E\xA8\xA7\x8C\x5A\x11\xBC\x7A\x82\xEB\xBE\x8C\xE9\xB3\xAC\x96\x25\x07\x97\x4A\x99\x2A\x07\x2F\xB4\x1E\x77\xBF\x8A\x0F\xB5\x02\x7C\x1B\x96\xB8\xC5\xB9\x3A\x2C\xBC\xD6\x12\xB9\xEB\x59\x7D\xE2\xD0\x06\x86\x5F\x5E\x49\x6A\xB5\x39\x5E\x88\x34\xEC\xBC\x78\x0C\x08\x98\x84\x6C\xA8\xCD\x4B\xB4\xA0\x7D\x0C\x79\x4D\xF0\xB8\x2D\xCB\x21\xCA\xD5\x6C\x5B\x7D\xE1\xA0\x29\x84\xA1\xF9\xD3\x94\x49\xCB\x24\x62\x91\x20\xBC\xDD\x0B\xD5\xD9\xCC\xF9\xEA\x27\x0A\x2B\x73\x91\xC6\x9D\x1B\xAC\xC8\xCB\xE8\xE0\xA0\xF4\x2F\x90\x8B\x4D\xFB\xB0\x36\x1B\xF6\x19\x7A\x85\xE0\x6D\xF2\x61\x13\x88\x5C\x9F\xE0\x93\x0A\x51\x97\x8A\x5A\xCE\xAF\xAB\xD5\xF7\xAA\x09\xAA\x60\xBD\xDC\xD9\x5F\xDF\x72\xA9\x60\x13\x5E\x00\x01\xC9\x4A\xFA\x3F\xA4\xEA\x07\x03\x21\x02\x8E\x82\xCA\x03\xC2\x9B\x8F\x02\x03\x01\x00\x01\xA3\x81\x9C\x30\x81\x99\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x9B\xE2\x07\x57\x67\x1C\x1E\xC0\x6A\x06\xDE\x59\xB4\x9A\x2D\xDF\xDC\x19\x86\x2E\x30\x36\x06\x03\x55\x1D\x1F\x04\x2F\x30\x2D\x30\x2B\xA0\x29\xA0\x27\x86\x25\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x67\x6C\x6F\x62\x61\x6C\x73\x69\x67\x6E\x2E\x6E\x65\x74\x2F\x72\x6F\x6F\x74\x2D\x72\x32\x2E\x63\x72\x6C\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x9B\xE2\x07\x57\x67\x1C\x1E\xC0\x6A\x06\xDE\x59\xB4\x9A\x2D\xDF\xDC\x19\x86\x2E\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x99\x81\x53\x87\x1C\x68\x97\x86\x91\xEC\xE0\x4A\xB8\x44\x0B\xAB\x81\xAC\x27\x4F\xD6\xC1\xB8\x1C\x43\x78\xB3\x0C\x9A\xFC\xEA\x2C\x3C\x6E\x61\x1B\x4D\x4B\x29\xF5\x9F\x05\x1D\x26\xC1\xB8\xE9\x83\x00\x62\x45\xB6\xA9\x08\x93\xB9\xA9\x33\x4B\x18\x9A\xC2\xF8\x87\x88\x4E\xDB\xDD\x71\x34\x1A\xC1\x54\xDA\x46\x3F\xE0\xD3\x2A\xAB\x6D\x54\x22\xF5\x3A\x62\xCD\x20\x6F\xBA\x29\x89\xD7\xDD\x91\xEE\xD3\x5C\xA2\x3E\xA1\x5B\x41\xF5\xDF\xE5\x64\x43\x2D\xE9\xD5\x39\xAB\xD2\xA2\xDF\xB7\x8B\xD0\xC0\x80\x19\x1C\x45\xC0\x2D\x8C\xE8\xF8\x2D\xA4\x74\x56\x49\xC5\x05\xB5\x4F\x15\xDE\x6E\x44\x78\x39\x87\xA8\x7E\xBB\xF3\x79\x18\x91\xBB\xF4\x6F\x9D\xC1\xF0\x8C\x35\x8C\x5D\x01\xFB\xC3\x6D\xB9\xEF\x44\x6D\x79\x46\x31\x7E\x0A\xFE\xA9\x82\xC1\xFF\xEF\xAB\x6E\x20\xC4\x50\xC9\x5F\x9D\x4D\x9B\x17\x8C\x0C\xE5\x01\xC9\xA0\x41\x6A\x73\x53\xFA\xA5\x50\xB4\x6E\x25\x0F\xFB\x4C\x18\xF4\xFD\x52\xD9\x8E\x69\xB1\xE8\x11\x0F\xDE\x88\xD8\xFB\x1D\x49\xF7\xAA\xDE\x95\xCF\x20\x78\xC2\x60\x12\xDB\x25\x40\x8C\x6A\xFC\x7E\x42\x38\x40\x64\x12\xF7\x9E\x81\xE1\x93\x2E",
 	["emailAddress=info@valicert.com,CN=http://www.valicert.com/,OU=ValiCert Class 1 Policy Validation Authority,O=ValiCert\, Inc.,L=ValiCert Validation Network"] = "\x30\x82\x02\xE7\x30\x82\x02\x50\x02\x01\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\xBB\x31\x24\x30\x22\x06\x03\x55\x04\x07\x13\x1B\x56\x61\x6C\x69\x43\x65\x72\x74\x20\x56\x61\x6C\x69\x64\x61\x74\x69\x6F\x6E\x20\x4E\x65\x74\x77\x6F\x72\x6B\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x61\x6C\x69\x43\x65\x72\x74\x2C\x20\x49\x6E\x63\x2E\x31\x35\x30\x33\x06\x03\x55\x04\x0B\x13\x2C\x56\x61\x6C\x69\x43\x65\x72\x74\x20\x43\x6C\x61\x73\x73\x20\x31\x20\x50\x6F\x6C\x69\x63\x79\x20\x56\x61\x6C\x69\x64\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x31\x21\x30\x1F\x06\x03\x55\x04\x03\x13\x18\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x76\x61\x6C\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x2F\x31\x20\x30\x1E\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x09\x01\x16\x11\x69\x6E\x66\x6F\x40\x76\x61\x6C\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x30\x1E\x17\x0D\x39\x39\x30\x36\x32\x35\x32\x32\x32\x33\x34\x38\x5A\x17\x0D\x31\x39\x30\x36\x32\x35\x32\x32\x32\x33\x34\x38\x5A\x30\x81\xBB\x31\x24\x30\x22\x06\x03\x55\x04\x07\x13\x1B\x56\x61\x6C\x69\x43\x65\x72\x74\x20\x56\x61\x6C\x69\x64\x61\x74\x69\x6F\x6E\x20\x4E\x65\x74\x77\x6F\x72\x6B\x31\x17\x30\x15\x06\x03\x55\x04\x0A\x13\x0E\x56\x61\x6C\x69\x43\x65\x72\x74\x2C\x20\x49\x6E\x63\x2E\x31\x35\x30\x33\x06\x03\x55\x04\x0B\x13\x2C\x56\x61\x6C\x69\x43\x65\x72\x74\x20\x43\x6C\x61\x73\x73\x20\x31\x20\x50\x6F\x6C\x69\x63\x79\x20\x56\x61\x6C\x69\x64\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x31\x21\x30\x1F\x06\x03\x55\x04\x03\x13\x18\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x76\x61\x6C\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x2F\x31\x20\x30\x1E\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x09\x01\x16\x11\x69\x6E\x66\x6F\x40\x76\x61\x6C\x69\x63\x65\x72\x74\x2E\x63\x6F\x6D\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xD8\x59\x82\x7A\x89\xB8\x96\xBA\xA6\x2F\x68\x6F\x58\x2E\xA7\x54\x1C\x06\x6E\xF4\xEA\x8D\x48\xBC\x31\x94\x17\xF0\xF3\x4E\xBC\xB2\xB8\x35\x92\x76\xB0\xD0\xA5\xA5\x01\xD7\x00\x03\x12\x22\x19\x08\xF8\xFF\x11\x23\x9B\xCE\x07\xF5\xBF\x69\x1A\x26\xFE\x4E\xE9\xD1\x7F\x9D\x2C\x40\x1D\x59\x68\x6E\xA6\xF8\x58\xB0\x9D\x1A\x8F\xD3\x3F\xF1\xDC\x19\x06\x81\xA8\x0E\xE0\x3A\xDD\xC8\x53\x45\x09\x06\xE6\x0F\x70\xC3\xFA\x40\xA6\x0E\xE2\x56\x05\x0F\x18\x4D\xFC\x20\x82\xD1\x73\x55\x74\x8D\x76\x72\xA0\x1D\x9D\x1D\xC0\xDD\x3F\x71\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x81\x81\x00\x50\x68\x3D\x49\xF4\x2C\x1C\x06\x94\xDF\x95\x60\x7F\x96\x7B\x17\xFE\x4F\x71\xAD\x64\xC8\xDD\x77\xD2\xEF\x59\x55\xE8\x3F\xE8\x8E\x05\x2A\x21\xF2\x07\xD2\xB5\xA7\x52\xFE\x9C\xB1\xB6\xE2\x5B\x77\x17\x40\xEA\x72\xD6\x23\xCB\x28\x81\x32\xC3\x00\x79\x18\xEC\x59\x17\x89\xC9\xC6\x6A\x1E\x71\xC9\xFD\xB7\x74\xA5\x25\x45\x69\xC5\x48\xAB\x19\xE1\x45\x8A\x25\x6B\x19\xEE\xE5\xBB\x12\xF5\x7F\xF7\xA6\x8D\x51\xC3\xF0\x9D\x74\xB7\xA9\x3E\xA0\xA5\xFF\xB6\x49\x03\x13\xDA\x22\xCC\xED\x71\x82\x2B\x99\xCF\x3A\xB7\xF5\x2D\x72\xC8",
@@ -38,8 +37,6 @@ redef root_certs += {
 	["CN=America Online Root Certification Authority 1,O=America Online Inc.,C=US"] = "\x30\x82\x03\xA4\x30\x82\x02\x8C\xA0\x03\x02\x01\x02\x02\x01\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x63\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x1C\x30\x1A\x06\x03\x55\x04\x0A\x13\x13\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x49\x6E\x63\x2E\x31\x36\x30\x34\x06\x03\x55\x04\x03\x13\x2D\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x31\x30\x1E\x17\x0D\x30\x32\x30\x35\x32\x38\x30\x36\x30\x30\x30\x30\x5A\x17\x0D\x33\x37\x31\x31\x31\x39\x32\x30\x34\x33\x30\x30\x5A\x30\x63\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x1C\x30\x1A\x06\x03\x55\x04\x0A\x13\x13\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x49\x6E\x63\x2E\x31\x36\x30\x34\x06\x03\x55\x04\x03\x13\x2D\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x31\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xA8\x2F\xE8\xA4\x69\x06\x03\x47\xC3\xE9\x2A\x98\xFF\x19\xA2\x70\x9A\xC6\x50\xB2\x7E\xA5\xDF\x68\x4D\x1B\x7C\x0F\xB6\x97\x68\x7D\x2D\xA6\x8B\x97\xE9\x64\x86\xC9\xA3\xEF\xA0\x86\xBF\x60\x65\x9C\x4B\x54\x88\xC2\x48\xC5\x4A\x39\xBF\x14\xE3\x59\x55\xE5\x19\xB4\x74\xC8\xB4\x05\x39\x5C\x16\xA5\xE2\x95\x05\xE0\x12\xAE\x59\x8B\xA2\x33\x68\x58\x1C\xA6\xD4\x15\xB7\xD8\x9F\xD7\xDC\x71\xAB\x7E\x9A\xBF\x9B\x8E\x33\x0F\x22\xFD\x1F\x2E\xE7\x07\x36\xEF\x62\x39\xC5\xDD\xCB\xBA\x25\x14\x23\xDE\x0C\xC6\x3D\x3C\xCE\x82\x08\xE6\x66\x3E\xDA\x51\x3B\x16\x3A\xA3\x05\x7F\xA0\xDC\x87\xD5\x9C\xFC\x72\xA9\xA0\x7D\x78\xE4\xB7\x31\x55\x1E\x65\xBB\xD4\x61\xB0\x21\x60\xED\x10\x32\x72\xC5\x92\x25\x1E\xF8\x90\x4A\x18\x78\x47\xDF\x7E\x30\x37\x3E\x50\x1B\xDB\x1C\xD3\x6B\x9A\x86\x53\x07\xB0\xEF\xAC\x06\x78\xF8\x84\x99\xFE\x21\x8D\x4C\x80\xB6\x0C\x82\xF6\x66\x70\x79\x1A\xD3\x4F\xA3\xCF\xF1\xCF\x46\xB0\x4B\x0F\x3E\xDD\x88\x62\xB8\x8C\xA9\x09\x28\x3B\x7A\xC7\x97\xE1\x1E\xE5\xF4\x9F\xC0\xC0\xAE\x24\xA0\xC8\xA1\xD9\x0F\xD6\x7B\x26\x82\x69\x32\x3D\xA7\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x00\xAD\xD9\xA3\xF6\x79\xF6\x6E\x74\xA9\x7F\x33\x3D\x81\x17\xD7\x4C\xCF\x33\xDE\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x00\xAD\xD9\xA3\xF6\x79\xF6\x6E\x74\xA9\x7F\x33\x3D\x81\x17\xD7\x4C\xCF\x33\xDE\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x7C\x8A\xD1\x1F\x18\x37\x82\xE0\xB8\xB0\xA3\xED\x56\x95\xC8\x62\x61\x9C\x05\xA2\xCD\xC2\x62\x26\x61\xCD\x10\x16\xD7\xCC\xB4\x65\x34\xD0\x11\x8A\xAD\xA8\xA9\x05\x66\xEF\x74\xF3\x6D\x5F\x9D\x99\xAF\xF6\x8B\xFB\xEB\x52\xB2\x05\x98\xA2\x6F\x2A\xC5\x54\xBD\x25\xBD\x5F\xAE\xC8\x86\xEA\x46\x2C\xC1\xB3\xBD\xC1\xE9\x49\x70\x18\x16\x97\x08\x13\x8C\x20\xE0\x1B\x2E\x3A\x47\xCB\x1E\xE4\x00\x30\x95\x5B\xF4\x45\xA3\xC0\x1A\xB0\x01\x4E\xAB\xBD\xC0\x23\x6E\x63\x3F\x80\x4A\xC5\x07\xED\xDC\xE2\x6F\xC7\xC1\x62\xF1\xE3\x72\xD6\x04\xC8\x74\x67\x0B\xFA\x88\xAB\xA1\x01\xC8\x6F\xF0\x14\xAF\xD2\x99\xCD\x51\x93\x7E\xED\x2E\x38\xC7\xBD\xCE\x46\x50\x3D\x72\xE3\x79\x25\x9D\x9B\x88\x2B\x10\x20\xDD\xA5\xB8\x32\x9F\x8D\xE0\x29\xDF\x21\x74\x86\x82\xDB\x2F\x82\x30\xC6\xC7\x35\x86\xB3\xF9\x96\x5F\x46\xDB\x0C\x45\xFD\xF3\x50\xC3\x6F\xC6\xC3\x48\xAD\x46\xA6\xE1\x27\x47\x0A\x1D\x0E\x9B\xB6\xC2\x77\x7F\x63\xF2\xE0\x7D\x1A\xBE\xFC\xE0\xDF\xD7\xC7\xA7\x6C\xB0\xF9\xAE\xBA\x3C\xFD\x74\xB4\x11\xE8\x58\x0D\x80\xBC\xD3\xA8\x80\x3A\x99\xED\x75\xCC\x46\x7B",
 	["CN=America Online Root Certification Authority 2,O=America Online Inc.,C=US"] = "\x30\x82\x05\xA4\x30\x82\x03\x8C\xA0\x03\x02\x01\x02\x02\x01\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x63\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x1C\x30\x1A\x06\x03\x55\x04\x0A\x13\x13\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x49\x6E\x63\x2E\x31\x36\x30\x34\x06\x03\x55\x04\x03\x13\x2D\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x32\x30\x1E\x17\x0D\x30\x32\x30\x35\x32\x38\x30\x36\x30\x30\x30\x30\x5A\x17\x0D\x33\x37\x30\x39\x32\x39\x31\x34\x30\x38\x30\x30\x5A\x30\x63\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x1C\x30\x1A\x06\x03\x55\x04\x0A\x13\x13\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x49\x6E\x63\x2E\x31\x36\x30\x34\x06\x03\x55\x04\x03\x13\x2D\x41\x6D\x65\x72\x69\x63\x61\x20\x4F\x6E\x6C\x69\x6E\x65\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x32\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xCC\x41\x45\x1D\xE9\x3D\x4D\x10\xF6\x8C\xB1\x41\xC9\xE0\x5E\xCB\x0D\xB7\xBF\x47\x73\xD3\xF0\x55\x4D\xDD\xC6\x0C\xFA\xB1\x66\x05\x6A\xCD\x78\xB4\xDC\x02\xDB\x4E\x81\xF3\xD7\xA7\x7C\x71\xBC\x75\x63\xA0\x5D\xE3\x07\x0C\x48\xEC\x25\xC4\x03\x20\xF4\xFF\x0E\x3B\x12\xFF\x9B\x8D\xE1\xC6\xD5\x1B\xB4\x6D\x22\xE3\xB1\xDB\x7F\x21\x64\xAF\x86\xBC\x57\x22\x2A\xD6\x47\x81\x57\x44\x82\x56\x53\xBD\x86\x14\x01\x0B\xFC\x7F\x74\xA4\x5A\xAE\xF1\xBA\x11\xB5\x9B\x58\x5A\x80\xB4\x37\x78\x09\x33\x7C\x32\x47\x03\x5C\xC4\xA5\x83\x48\xF4\x57\x56\x6E\x81\x36\x27\x18\x4F\xEC\x9B\x28\xC2\xD4\xB4\xD7\x7C\x0C\x3E\x0C\x2B\xDF\xCA\x04\xD7\xC6\x8E\xEA\x58\x4E\xA8\xA4\xA5\x18\x1C\x6C\x45\x98\xA3\x41\xD1\x2D\xD2\xC7\x6D\x8D\x19\xF1\xAD\x79\xB7\x81\x3F\xBD\x06\x82\x27\x2D\x10\x58\x05\xB5\x78\x05\xB9\x2F\xDB\x0C\x6B\x90\x90\x7E\x14\x59\x38\xBB\x94\x24\x13\xE5\xD1\x9D\x14\xDF\xD3\x82\x4D\x46\xF0\x80\x39\x52\x32\x0F\xE3\x84\xB2\x7A\x43\xF2\x5E\xDE\x5F\x3F\x1D\xDD\xE3\xB2\x1B\xA0\xA1\x2A\x23\x03\x6E\x2E\x01\x15\x87\x5C\xA6\x75\x75\xC7\x97\x61\xBE\xDE\x86\xDC\xD4\x48\xDB\xBD\x2A\xBF\x4A\x55\xDA\xE8\x7D\x50\xFB\xB4\x80\x17\xB8\x94\xBF\x01\x3D\xEA\xDA\xBA\x7C\xE0\x58\x67\x17\xB9\x58\xE0\x88\x86\x46\x67\x6C\x9D\x10\x47\x58\x32\xD0\x35\x7C\x79\x2A\x90\xA2\x5A\x10\x11\x23\x35\xAD\x2F\xCC\xE4\x4A\x5B\xA7\xC8\x27\xF2\x83\xDE\x5E\xBB\x5E\x77\xE7\xE8\xA5\x6E\x63\xC2\x0D\x5D\x61\xD0\x8C\xD2\x6C\x5A\x21\x0E\xCA\x28\xA3\xCE\x2A\xE9\x95\xC7\x48\xCF\x96\x6F\x1D\x92\x25\xC8\xC6\xC6\xC1\xC1\x0C\x05\xAC\x26\xC4\xD2\x75\xD2\xE1\x2A\x67\xC0\x3D\x5B\xA5\x9A\xEB\xCF\x7B\x1A\xA8\x9D\x14\x45\xE5\x0F\xA0\x9A\x65\xDE\x2F\x28\xBD\xCE\x6F\x94\x66\x83\x48\x29\xD8\xEA\x65\x8C\xAF\x93\xD9\x64\x9F\x55\x57\x26\xBF\x6F\xCB\x37\x31\x99\xA3\x60\xBB\x1C\xAD\x89\x34\x32\x62\xB8\x43\x21\x06\x72\x0C\xA1\x5C\x6D\x46\xC5\xFA\x29\xCF\x30\xDE\x89\xDC\x71\x5B\xDD\xB6\x37\x3E\xDF\x50\xF5\xB8\x07\x25\x26\xE5\xBC\xB5\xFE\x3C\x02\xB3\xB7\xF8\xBE\x43\xC1\x87\x11\x94\x9E\x23\x6C\x17\x8A\xB8\x8A\x27\x0C\x54\x47\xF0\xA9\xB3\xC0\x80\x8C\xA0\x27\xEB\x1D\x19\xE3\x07\x8E\x77\x70\xCA\x2B\xF4\x7D\x76\xE0\x78\x67\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x4D\x45\xC1\x68\x38\xBB\x73\xA9\x69\xA1\x20\xE7\xED\xF5\x22\xA1\x23\x14\xD7\x9E\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x4D\x45\xC1\x68\x38\xBB\x73\xA9\x69\xA1\x20\xE7\xED\xF5\x22\xA1\x23\x14\xD7\x9E\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x02\x01\x00\x67\x6B\x06\xB9\x5F\x45\x3B\x2A\x4B\x33\xB3\xE6\x1B\x6B\x59\x4E\x22\xCC\xB9\xB7\xA4\x25\xC9\xA7\xC4\xF0\x54\x96\x0B\x64\xF3\xB1\x58\x4F\x5E\x51\xFC\xB2\x97\x7B\x27\x65\xC2\xE5\xCA\xE7\x0D\x0C\x25\x7B\x62\xE3\xFA\x9F\xB4\x87\xB7\x45\x46\xAF\x83\xA5\x97\x48\x8C\xA5\xBD\xF1\x16\x2B\x9B\x76\x2C\x7A\x35\x60\x6C\x11\x80\x97\xCC\xA9\x92\x52\xE6\x2B\xE6\x69\xED\xA9\xF8\x36\x2D\x2C\x77\xBF\x61\x48\xD1\x63\x0B\xB9\x5B\x52\xED\x18\xB0\x43\x42\x22\xA6\xB1\x77\xAE\xDE\x69\xC5\xCD\xC7\x1C\xA1\xB1\xA5\x1C\x10\xFB\x18\xBE\x1A\x70\xDD\xC1\x92\x4B\xBE\x29\x5A\x9D\x3F\x35\xBE\xE5\x7D\x51\xF8\x55\xE0\x25\x75\x23\x87\x1E\x5C\xDC\xBA\x9D\xB0\xAC\xB3\x69\xDB\x17\x83\xC9\xF7\xDE\x0C\xBC\x08\xDC\x91\x9E\xA8\xD0\xD7\x15\x37\x73\xA5\x35\xB8\xFC\x7E\xC5\x44\x40\x06\xC3\xEB\xF8\x22\x80\x5C\x47\xCE\x02\xE3\x11\x9F\x44\xFF\xFD\x9A\x32\xCC\x7D\x64\x51\x0E\xEB\x57\x26\x76\x3A\xE3\x1E\x22\x3C\xC2\xA6\x36\xDD\x19\xEF\xA7\xFC\x12\xF3\x26\xC0\x59\x31\x85\x4C\x9C\xD8\xCF\xDF\xA4\xCC\xCC\x29\x93\xFF\x94\x6D\x76\x5C\x13\x08\x97\xF2\xED\xA5\x0B\x4D\xDD\xE8\xC9\x68\x0E\x66\xD3\x00\x0E\x33\x12\x5B\xBC\x95\xE5\x32\x90\xA8\xB3\xC6\x6C\x83\xAD\x77\xEE\x8B\x7E\x7E\xB1\xA9\xAB\xD3\xE1\xF1\xB6\xC0\xB1\xEA\x88\xC0\xE7\xD3\x90\xE9\x28\x92\x94\x7B\x68\x7B\x97\x2A\x0A\x67\x2D\x85\x02\x38\x10\xE4\x03\x61\xD4\xDA\x25\x36\xC7\x08\x58\x2D\xA1\xA7\x51\xAF\x30\x0A\x49\xF5\xA6\x69\x87\x07\x2D\x44\x46\x76\x8E\x2A\xE5\x9A\x3B\xD7\x18\xA2\xFC\x9C\x38\x10\xCC\xC6\x3B\xD2\xB5\x17\x3A\x6F\xFD\xAE\x25\xBD\xF5\x72\x59\x64\xB1\x74\x2A\x38\x5F\x18\x4C\xDF\xCF\x71\x04\x5A\x36\xD4\xBF\x2F\x99\x9C\xE8\xD9\xBA\xB1\x95\xE6\x02\x4B\x21\xA1\x5B\xD5\xC1\x4F\x8F\xAE\x69\x6D\x53\xDB\x01\x93\xB5\x5C\x1E\x18\xDD\x64\x5A\xCA\x18\x28\x3E\x63\x04\x11\xFD\x1C\x8D\x00\x0F\xB8\x37\xDF\x67\x8A\x9D\x66\xA9\x02\x6A\x91\xFF\x13\xCA\x2F\x5D\x83\xBC\x87\x93\x6C\xDC\x24\x51\x16\x04\x25\x66\xFA\xB3\xD9\xC2\xBA\x29\xBE\x9A\x48\x38\x82\x99\xF4\xBF\x3B\x4A\x31\x19\xF9\xBF\x8E\x21\x33\x14\xCA\x4F\x54\x5F\xFB\xCE\xFB\x8F\x71\x7F\xFD\x5E\x19\xA0\x0F\x4B\x91\xB8\xC4\x54\xBC\x06\xB0\x45\x8F\x26\x91\xA2\x8E\xFE\xA9",
 	["CN=Visa eCommerce Root,OU=Visa International Service Association,O=VISA,C=US"] = "\x30\x82\x03\xA2\x30\x82\x02\x8A\xA0\x03\x02\x01\x02\x02\x10\x13\x86\x35\x4D\x1D\x3F\x06\xF2\xC1\xF9\x65\x05\xD5\x90\x1C\x62\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x6B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x0D\x30\x0B\x06\x03\x55\x04\x0A\x13\x04\x56\x49\x53\x41\x31\x2F\x30\x2D\x06\x03\x55\x04\x0B\x13\x26\x56\x69\x73\x61\x20\x49\x6E\x74\x65\x72\x6E\x61\x74\x69\x6F\x6E\x61\x6C\x20\x53\x65\x72\x76\x69\x63\x65\x20\x41\x73\x73\x6F\x63\x69\x61\x74\x69\x6F\x6E\x31\x1C\x30\x1A\x06\x03\x55\x04\x03\x13\x13\x56\x69\x73\x61\x20\x65\x43\x6F\x6D\x6D\x65\x72\x63\x65\x20\x52\x6F\x6F\x74\x30\x1E\x17\x0D\x30\x32\x30\x36\x32\x36\x30\x32\x31\x38\x33\x36\x5A\x17\x0D\x32\x32\x30\x36\x32\x34\x30\x30\x31\x36\x31\x32\x5A\x30\x6B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x0D\x30\x0B\x06\x03\x55\x04\x0A\x13\x04\x56\x49\x53\x41\x31\x2F\x30\x2D\x06\x03\x55\x04\x0B\x13\x26\x56\x69\x73\x61\x20\x49\x6E\x74\x65\x72\x6E\x61\x74\x69\x6F\x6E\x61\x6C\x20\x53\x65\x72\x76\x69\x63\x65\x20\x41\x73\x73\x6F\x63\x69\x61\x74\x69\x6F\x6E\x31\x1C\x30\x1A\x06\x03\x55\x04\x03\x13\x13\x56\x69\x73\x61\x20\x65\x43\x6F\x6D\x6D\x65\x72\x63\x65\x20\x52\x6F\x6F\x74\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xAF\x57\xDE\x56\x1E\x6E\xA1\xDA\x60\xB1\x94\x27\xCB\x17\xDB\x07\x3F\x80\x85\x4F\xC8\x9C\xB6\xD0\xF4\x6F\x4F\xCF\x99\xD8\xE1\xDB\xC2\x48\x5C\x3A\xAC\x39\x33\xC7\x1F\x6A\x8B\x26\x3D\x2B\x35\xF5\x48\xB1\x91\xC1\x02\x4E\x04\x96\x91\x7B\xB0\x33\xF0\xB1\x14\x4E\x11\x6F\xB5\x40\xAF\x1B\x45\xA5\x4A\xEF\x7E\xB6\xAC\xF2\xA0\x1F\x58\x3F\x12\x46\x60\x3C\x8D\xA1\xE0\x7D\xCF\x57\x3E\x33\x1E\xFB\x47\xF1\xAA\x15\x97\x07\x55\x66\xA5\xB5\x2D\x2E\xD8\x80\x59\xB2\xA7\x0D\xB7\x46\xEC\x21\x63\xFF\x35\xAB\xA5\x02\xCF\x2A\xF4\x4C\xFE\x7B\xF5\x94\x5D\x84\x4D\xA8\xF2\x60\x8F\xDB\x0E\x25\x3C\x9F\x73\x71\xCF\x94\xDF\x4A\xEA\xDB\xDF\x72\x38\x8C\xF3\x96\xBD\xF1\x17\xBC\xD2\xBA\x3B\x45\x5A\xC6\xA7\xF6\xC6\x17\x8B\x01\x9D\xFC\x19\xA8\x2A\x83\x16\xB8\x3A\x48\xFE\x4E\x3E\xA0\xAB\x06\x19\xE9\x53\xF3\x80\x13\x07\xED\x2D\xBF\x3F\x0A\x3C\x55\x20\x39\x2C\x2C\x00\x69\x74\x95\x4A\xBC\x20\xB2\xA9\x79\xE5\x18\x89\x91\xA8\xDC\x1C\x4D\xEF\xBB\x7E\x37\x0B\x5D\xFE\x39\xA5\x88\x52\x8C\x00\x6C\xEC\x18\x7C\x41\xBD\xF6\x8B\x75\x77\xBA\x60\x9D\x84\xE7\xFE\x2D\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x15\x38\x83\x0F\x3F\x2C\x3F\x70\x33\x1E\xCD\x46\xFE\x07\x8C\x20\xE0\xD7\xC3\xB7\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x5F\xF1\x41\x7D\x7C\x5C\x08\xB9\x2B\xE0\xD5\x92\x47\xFA\x67\x5C\xA5\x13\xC3\x03\x21\x9B\x2B\x4C\x89\x46\xCF\x59\x4D\xC9\xFE\xA5\x40\xB6\x63\xCD\xDD\x71\x28\x95\x67\x11\xCC\x24\xAC\xD3\x44\x6C\x71\xAE\x01\x20\x6B\x03\xA2\x8F\x18\xB7\x29\x3A\x7D\xE5\x16\x60\x53\x78\x3C\xC0\xAF\x15\x83\xF7\x8F\x52\x33\x24\xBD\x64\x93\x97\xEE\x8B\xF7\xDB\x18\xA8\x6D\x71\xB3\xF7\x2C\x17\xD0\x74\x25\x69\xF7\xFE\x6B\x3C\x94\xBE\x4D\x4B\x41\x8C\x4E\xE2\x73\xD0\xE3\x90\x22\x73\x43\xCD\xF3\xEF\xEA\x73\xCE\x45\x8A\xB0\xA6\x49\xFF\x4C\x7D\x9D\x71\x88\xC4\x76\x1D\x90\x5B\x1D\xEE\xFD\xCC\xF7\xEE\xFD\x60\xA5\xB1\x7A\x16\x71\xD1\x16\xD0\x7C\x12\x3C\x6C\x69\x97\xDB\xAE\x5F\x39\x9A\x70\x2F\x05\x3C\x19\x46\x04\x99\x20\x36\xD0\x60\x6E\x61\x06\xBB\x16\x42\x8C\x70\xF7\x30\xFB\xE0\xDB\x66\xA3\x00\x01\xBD\xE6\x2C\xDA\x91\x5F\xA0\x46\x8B\x4D\x6A\x9C\x3D\x3D\xDD\x05\x46\xFE\x76\xBF\xA0\x0A\x3C\xE4\x00\xE6\x27\xB7\xFF\x84\x2D\xDE\xBA\x22\x27\x96\x10\x71\xEB\x22\xED\xDF\xDF\x33\x9C\xCF\xE3\xAD\xAE\x8E\xD4\x8E\xE6\x4F\x51\xAF\x16\x92\xE0\x5C\xF6\x07\x0F",
-	["emailAddress=certificate@trustcenter.de,OU=TC TrustCenter Class 2 CA,O=TC TrustCenter for Security in Data Networks GmbH,L=Hamburg,ST=Hamburg,C=DE"] = "\x30\x82\x03\x5C\x30\x82\x02\xC5\xA0\x03\x02\x01\x02\x02\x02\x03\xEA\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x30\x81\xBC\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x45\x31\x10\x30\x0E\x06\x03\x55\x04\x08\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x3A\x30\x38\x06\x03\x55\x04\x0A\x13\x31\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x66\x6F\x72\x20\x53\x65\x63\x75\x72\x69\x74\x79\x20\x69\x6E\x20\x44\x61\x74\x61\x20\x4E\x65\x74\x77\x6F\x72\x6B\x73\x20\x47\x6D\x62\x48\x31\x22\x30\x20\x06\x03\x55\x04\x0B\x13\x19\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x43\x6C\x61\x73\x73\x20\x32\x20\x43\x41\x31\x29\x30\x27\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x09\x01\x16\x1A\x63\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x40\x74\x72\x75\x73\x74\x63\x65\x6E\x74\x65\x72\x2E\x64\x65\x30\x1E\x17\x0D\x39\x38\x30\x33\x30\x39\x31\x31\x35\x39\x35\x39\x5A\x17\x0D\x31\x31\x30\x31\x30\x31\x31\x31\x35\x39\x35\x39\x5A\x30\x81\xBC\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x45\x31\x10\x30\x0E\x06\x03\x55\x04\x08\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x3A\x30\x38\x06\x03\x55\x04\x0A\x13\x31\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x66\x6F\x72\x20\x53\x65\x63\x75\x72\x69\x74\x79\x20\x69\x6E\x20\x44\x61\x74\x61\x20\x4E\x65\x74\x77\x6F\x72\x6B\x73\x20\x47\x6D\x62\x48\x31\x22\x30\x20\x06\x03\x55\x04\x0B\x13\x19\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x43\x6C\x61\x73\x73\x20\x32\x20\x43\x41\x31\x29\x30\x27\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x09\x01\x16\x1A\x63\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x40\x74\x72\x75\x73\x74\x63\x65\x6E\x74\x65\x72\x2E\x64\x65\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xDA\x38\xE8\xED\x32\x00\x29\x71\x83\x01\x0D\xBF\x8C\x01\xDC\xDA\xC6\xAD\x39\xA4\xA9\x8A\x2F\xD5\x8B\x5C\x68\x5F\x50\xC6\x62\xF5\x66\xBD\xCA\x91\x22\xEC\xAA\x1D\x51\xD7\x3D\xB3\x51\xB2\x83\x4E\x5D\xCB\x49\xB0\xF0\x4C\x55\xE5\x6B\x2D\xC7\x85\x0B\x30\x1C\x92\x4E\x82\xD4\xCA\x02\xED\xF7\x6F\xBE\xDC\xE0\xE3\x14\xB8\x05\x53\xF2\x9A\xF4\x56\x8B\x5A\x9E\x85\x93\xD1\xB4\x82\x56\xAE\x4D\xBB\xA8\x4B\x57\x16\xBC\xFE\xF8\x58\x9E\xF8\x29\x8D\xB0\x7B\xCD\x78\xC9\x4F\xAC\x8B\x67\x0C\xF1\x9C\xFB\xFC\x57\x9B\x57\x5C\x4F\x0D\x02\x03\x01\x00\x01\xA3\x6B\x30\x69\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x33\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x08\x04\x26\x16\x24\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x74\x72\x75\x73\x74\x63\x65\x6E\x74\x65\x72\x2E\x64\x65\x2F\x67\x75\x69\x64\x65\x6C\x69\x6E\x65\x73\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x03\x81\x81\x00\x84\x52\xFB\x28\xDF\xFF\x1F\x75\x01\xBC\x01\xBE\x04\x56\x97\x6A\x74\x42\x24\x31\x83\xF9\x46\xB1\x06\x8A\x89\xCF\x96\x2C\x33\xBF\x8C\xB5\x5F\x7A\x72\xA1\x85\x06\xCE\x86\xF8\x05\x8E\xE8\xF9\x25\xCA\xDA\x83\x8C\x06\xAC\xEB\x36\x6D\x85\x91\x34\x04\x36\xF4\x42\xF0\xF8\x79\x2E\x0A\x48\x5C\xAB\xCC\x51\x4F\x78\x76\xA0\xD9\xAC\x19\xBD\x2A\xD1\x69\x04\x28\x91\xCA\x36\x10\x27\x80\x57\x5B\xD2\x5C\xF5\xC2\x5B\xAB\x64\x81\x63\x74\x51\xF4\x97\xBF\xCD\x12\x28\xF7\x4D\x66\x7F\xA7\xF0\x1C\x01\x26\x78\xB2\x66\x47\x70\x51\x64",
-	["emailAddress=certificate@trustcenter.de,OU=TC TrustCenter Class 3 CA,O=TC TrustCenter for Security in Data Networks GmbH,L=Hamburg,ST=Hamburg,C=DE"] = "\x30\x82\x03\x5C\x30\x82\x02\xC5\xA0\x03\x02\x01\x02\x02\x02\x03\xEB\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x30\x81\xBC\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x45\x31\x10\x30\x0E\x06\x03\x55\x04\x08\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x3A\x30\x38\x06\x03\x55\x04\x0A\x13\x31\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x66\x6F\x72\x20\x53\x65\x63\x75\x72\x69\x74\x79\x20\x69\x6E\x20\x44\x61\x74\x61\x20\x4E\x65\x74\x77\x6F\x72\x6B\x73\x20\x47\x6D\x62\x48\x31\x22\x30\x20\x06\x03\x55\x04\x0B\x13\x19\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x43\x6C\x61\x73\x73\x20\x33\x20\x43\x41\x31\x29\x30\x27\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x09\x01\x16\x1A\x63\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x40\x74\x72\x75\x73\x74\x63\x65\x6E\x74\x65\x72\x2E\x64\x65\x30\x1E\x17\x0D\x39\x38\x30\x33\x30\x39\x31\x31\x35\x39\x35\x39\x5A\x17\x0D\x31\x31\x30\x31\x30\x31\x31\x31\x35\x39\x35\x39\x5A\x30\x81\xBC\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x45\x31\x10\x30\x0E\x06\x03\x55\x04\x08\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x13\x07\x48\x61\x6D\x62\x75\x72\x67\x31\x3A\x30\x38\x06\x03\x55\x04\x0A\x13\x31\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x66\x6F\x72\x20\x53\x65\x63\x75\x72\x69\x74\x79\x20\x69\x6E\x20\x44\x61\x74\x61\x20\x4E\x65\x74\x77\x6F\x72\x6B\x73\x20\x47\x6D\x62\x48\x31\x22\x30\x20\x06\x03\x55\x04\x0B\x13\x19\x54\x43\x20\x54\x72\x75\x73\x74\x43\x65\x6E\x74\x65\x72\x20\x43\x6C\x61\x73\x73\x20\x33\x20\x43\x41\x31\x29\x30\x27\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x09\x01\x16\x1A\x63\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x40\x74\x72\x75\x73\x74\x63\x65\x6E\x74\x65\x72\x2E\x64\x65\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\xB6\xB4\xC1\x35\x05\x2E\x0D\x8D\xEC\xA0\x40\x6A\x1C\x0E\x27\xA6\x50\x92\x6B\x50\x1B\x07\xDE\x2E\xE7\x76\xCC\xE0\xDA\xFC\x84\xA8\x5E\x8C\x63\x6A\x2B\x4D\xD9\x4E\x02\x76\x11\xC1\x0B\xF2\x8D\x79\xCA\x00\xB6\xF1\xB0\x0E\xD7\xFB\xA4\x17\x3D\xAF\xAB\x69\x7A\x96\x27\xBF\xAF\x33\xA1\x9A\x2A\x59\xAA\xC4\xB5\x37\x08\xF2\x12\xA5\x31\xB6\x43\xF5\x32\x96\x71\x28\x28\xAB\x8D\x28\x86\xDF\xBB\xEE\xE3\x0C\x7D\x30\xD6\xC3\x52\xAB\x8F\x5D\x27\x9C\x6B\xC0\xA3\xE7\x05\x6B\x57\x49\x44\xB3\x6E\xEA\x64\xCF\xD2\x8E\x7A\x50\x77\x77\x02\x03\x01\x00\x01\xA3\x6B\x30\x69\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x86\x30\x33\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x08\x04\x26\x16\x24\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x74\x72\x75\x73\x74\x63\x65\x6E\x74\x65\x72\x2E\x64\x65\x2F\x67\x75\x69\x64\x65\x6C\x69\x6E\x65\x73\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x03\x81\x81\x00\x16\x3D\xC6\xCD\xC1\xBB\x85\x71\x85\x46\x9F\x3E\x20\x8F\x51\x28\x99\xEC\x2D\x45\x21\x63\x23\x5B\x04\xBB\x4C\x90\xB8\x88\x92\x04\x4D\xBD\x7D\x01\xA3\x3F\xF6\xEC\xCE\xF1\xDE\xFE\x7D\xE5\xE1\x3E\xBB\xC6\xAB\x5E\x0B\xDD\x3D\x96\xC4\xCB\xA9\xD4\xF9\x26\xE6\x06\x4E\x9E\x0C\xA5\x7A\xBA\x6E\xC3\x7C\x82\x19\xD1\xC7\xB1\xB1\xC3\xDB\x0D\x8E\x9B\x40\x7C\x37\x0B\xF1\x5D\xE8\xFD\x1F\x90\x88\xA5\x0E\x4E\x37\x64\x21\xA8\x4E\x8D\xB4\x9F\xF1\xDE\x48\xAD\xD5\x56\x18\x52\x29\x8B\x47\x34\x12\x09\xD4\xBB\x92\x35\xEF\x0F\xDB\x34",
 	["CN=Certum CA,O=Unizeto Sp. z o.o.,C=PL"] = "\x30\x82\x03\x0C\x30\x82\x01\xF4\xA0\x03\x02\x01\x02\x02\x03\x01\x00\x20\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x3E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x50\x4C\x31\x1B\x30\x19\x06\x03\x55\x04\x0A\x13\x12\x55\x6E\x69\x7A\x65\x74\x6F\x20\x53\x70\x2E\x20\x7A\x20\x6F\x2E\x6F\x2E\x31\x12\x30\x10\x06\x03\x55\x04\x03\x13\x09\x43\x65\x72\x74\x75\x6D\x20\x43\x41\x30\x1E\x17\x0D\x30\x32\x30\x36\x31\x31\x31\x30\x34\x36\x33\x39\x5A\x17\x0D\x32\x37\x30\x36\x31\x31\x31\x30\x34\x36\x33\x39\x5A\x30\x3E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x50\x4C\x31\x1B\x30\x19\x06\x03\x55\x04\x0A\x13\x12\x55\x6E\x69\x7A\x65\x74\x6F\x20\x53\x70\x2E\x20\x7A\x20\x6F\x2E\x6F\x2E\x31\x12\x30\x10\x06\x03\x55\x04\x03\x13\x09\x43\x65\x72\x74\x75\x6D\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xCE\xB1\xC1\x2E\xD3\x4F\x7C\xCD\x25\xCE\x18\x3E\x4F\xC4\x8C\x6F\x80\x6A\x73\xC8\x5B\x51\xF8\x9B\xD2\xDC\xBB\x00\x5C\xB1\xA0\xFC\x75\x03\xEE\x81\xF0\x88\xEE\x23\x52\xE9\xE6\x15\x33\x8D\xAC\x2D\x09\xC5\x76\xF9\x2B\x39\x80\x89\xE4\x97\x4B\x90\xA5\xA8\x78\xF8\x73\x43\x7B\xA4\x61\xB0\xD8\x58\xCC\xE1\x6C\x66\x7E\x9C\xF3\x09\x5E\x55\x63\x84\xD5\xA8\xEF\xF3\xB1\x2E\x30\x68\xB3\xC4\x3C\xD8\xAC\x6E\x8D\x99\x5A\x90\x4E\x34\xDC\x36\x9A\x8F\x81\x88\x50\xB7\x6D\x96\x42\x09\xF3\xD7\x95\x83\x0D\x41\x4B\xB0\x6A\x6B\xF8\xFC\x0F\x7E\x62\x9F\x67\xC4\xED\x26\x5F\x10\x26\x0F\x08\x4F\xF0\xA4\x57\x28\xCE\x8F\xB8\xED\x45\xF6\x6E\xEE\x25\x5D\xAA\x6E\x39\xBE\xE4\x93\x2F\xD9\x47\xA0\x72\xEB\xFA\xA6\x5B\xAF\xCA\x53\x3F\xE2\x0E\xC6\x96\x56\x11\x6E\xF7\xE9\x66\xA9\x26\xD8\x7F\x95\x53\xED\x0A\x85\x88\xBA\x4F\x29\xA5\x42\x8C\x5E\xB6\xFC\x85\x20\x00\xAA\x68\x0B\xA1\x1A\x85\x01\x9C\xC4\x46\x63\x82\x88\xB6\x22\xB1\xEE\xFE\xAA\x46\x59\x7E\xCF\x35\x2C\xD5\xB6\xDA\x5D\xF7\x48\x33\x14\x54\xB6\xEB\xD9\x6F\xCE\xCD\x88\xD6\xAB\x1B\xDA\x96\x3B\x1D\x59\x02\x03\x01\x00\x01\xA3\x13\x30\x11\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xB8\x8D\xCE\xEF\xE7\x14\xBA\xCF\xEE\xB0\x44\x92\x6C\xB4\x39\x3E\xA2\x84\x6E\xAD\xB8\x21\x77\xD2\xD4\x77\x82\x87\xE6\x20\x41\x81\xEE\xE2\xF8\x11\xB7\x63\xD1\x17\x37\xBE\x19\x76\x24\x1C\x04\x1A\x4C\xEB\x3D\xAA\x67\x6F\x2D\xD4\xCD\xFE\x65\x31\x70\xC5\x1B\xA6\x02\x0A\xBA\x60\x7B\x6D\x58\xC2\x9A\x49\xFE\x63\x32\x0B\x6B\xE3\x3A\xC0\xAC\xAB\x3B\xB0\xE8\xD3\x09\x51\x8C\x10\x83\xC6\x34\xE0\xC5\x2B\xE0\x1A\xB6\x60\x14\x27\x6C\x32\x77\x8C\xBC\xB2\x72\x98\xCF\xCD\xCC\x3F\xB9\xC8\x24\x42\x14\xD6\x57\xFC\xE6\x26\x43\xA9\x1D\xE5\x80\x90\xCE\x03\x54\x28\x3E\xF7\x3F\xD3\xF8\x4D\xED\x6A\x0A\x3A\x93\x13\x9B\x3B\x14\x23\x13\x63\x9C\x3F\xD1\x87\x27\x79\xE5\x4C\x51\xE3\x01\xAD\x85\x5D\x1A\x3B\xB1\xD5\x73\x10\xA4\xD3\xF2\xBC\x6E\x64\xF5\x5A\x56\x90\xA8\xC7\x0E\x4C\x74\x0F\x2E\x71\x3B\xF7\xC8\x47\xF4\x69\x6F\x15\xF2\x11\x5E\x83\x1E\x9C\x7C\x52\xAE\xFD\x02\xDA\x12\xA8\x59\x67\x18\xDB\xBC\x70\xDD\x9B\xB1\x69\xED\x80\xCE\x89\x40\x48\x6A\x0E\x35\xCA\x29\x66\x15\x21\x94\x2C\xE8\x60\x2A\x9B\x85\x4A\x40\xF3\x6B\x8A\x24\xEC\x06\x16\x2C\x73",
 	["CN=AAA Certificate Services,O=Comodo CA Limited,L=Salford,ST=Greater Manchester,C=GB"] = "\x30\x82\x04\x32\x30\x82\x03\x1A\xA0\x03\x02\x01\x02\x02\x01\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x7B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x1B\x30\x19\x06\x03\x55\x04\x08\x0C\x12\x47\x72\x65\x61\x74\x65\x72\x20\x4D\x61\x6E\x63\x68\x65\x73\x74\x65\x72\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x0C\x07\x53\x61\x6C\x66\x6F\x72\x64\x31\x1A\x30\x18\x06\x03\x55\x04\x0A\x0C\x11\x43\x6F\x6D\x6F\x64\x6F\x20\x43\x41\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x21\x30\x1F\x06\x03\x55\x04\x03\x0C\x18\x41\x41\x41\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x20\x53\x65\x72\x76\x69\x63\x65\x73\x30\x1E\x17\x0D\x30\x34\x30\x31\x30\x31\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x31\x32\x33\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x7B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x1B\x30\x19\x06\x03\x55\x04\x08\x0C\x12\x47\x72\x65\x61\x74\x65\x72\x20\x4D\x61\x6E\x63\x68\x65\x73\x74\x65\x72\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x0C\x07\x53\x61\x6C\x66\x6F\x72\x64\x31\x1A\x30\x18\x06\x03\x55\x04\x0A\x0C\x11\x43\x6F\x6D\x6F\x64\x6F\x20\x43\x41\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x21\x30\x1F\x06\x03\x55\x04\x03\x0C\x18\x41\x41\x41\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x20\x53\x65\x72\x76\x69\x63\x65\x73\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xBE\x40\x9D\xF4\x6E\xE1\xEA\x76\x87\x1C\x4D\x45\x44\x8E\xBE\x46\xC8\x83\x06\x9D\xC1\x2A\xFE\x18\x1F\x8E\xE4\x02\xFA\xF3\xAB\x5D\x50\x8A\x16\x31\x0B\x9A\x06\xD0\xC5\x70\x22\xCD\x49\x2D\x54\x63\xCC\xB6\x6E\x68\x46\x0B\x53\xEA\xCB\x4C\x24\xC0\xBC\x72\x4E\xEA\xF1\x15\xAE\xF4\x54\x9A\x12\x0A\xC3\x7A\xB2\x33\x60\xE2\xDA\x89\x55\xF3\x22\x58\xF3\xDE\xDC\xCF\xEF\x83\x86\xA2\x8C\x94\x4F\x9F\x68\xF2\x98\x90\x46\x84\x27\xC7\x76\xBF\xE3\xCC\x35\x2C\x8B\x5E\x07\x64\x65\x82\xC0\x48\xB0\xA8\x91\xF9\x61\x9F\x76\x20\x50\xA8\x91\xC7\x66\xB5\xEB\x78\x62\x03\x56\xF0\x8A\x1A\x13\xEA\x31\xA3\x1E\xA0\x99\xFD\x38\xF6\xF6\x27\x32\x58\x6F\x07\xF5\x6B\xB8\xFB\x14\x2B\xAF\xB7\xAA\xCC\xD6\x63\x5F\x73\x8C\xDA\x05\x99\xA8\x38\xA8\xCB\x17\x78\x36\x51\xAC\xE9\x9E\xF4\x78\x3A\x8D\xCF\x0F\xD9\x42\xE2\x98\x0C\xAB\x2F\x9F\x0E\x01\xDE\xEF\x9F\x99\x49\xF1\x2D\xDF\xAC\x74\x4D\x1B\x98\xB5\x47\xC5\xE5\x29\xD1\xF9\x90\x18\xC7\x62\x9C\xBE\x83\xC7\x26\x7B\x3E\x8A\x25\xC7\xC0\xDD\x9D\xE6\x35\x68\x10\x20\x9D\x8F\xD8\xDE\xD2\xC3\x84\x9C\x0D\x5E\xE8\x2F\xC9\x02\x03\x01\x00\x01\xA3\x81\xC0\x30\x81\xBD\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xA0\x11\x0A\x23\x3E\x96\xF1\x07\xEC\xE2\xAF\x29\xEF\x82\xA5\x7F\xD0\x30\xA4\xB4\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x7B\x06\x03\x55\x1D\x1F\x04\x74\x30\x72\x30\x38\xA0\x36\xA0\x34\x86\x32\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x63\x6F\x6D\x6F\x64\x6F\x63\x61\x2E\x63\x6F\x6D\x2F\x41\x41\x41\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x53\x65\x72\x76\x69\x63\x65\x73\x2E\x63\x72\x6C\x30\x36\xA0\x34\xA0\x32\x86\x30\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x63\x6F\x6D\x6F\x64\x6F\x2E\x6E\x65\x74\x2F\x41\x41\x41\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x53\x65\x72\x76\x69\x63\x65\x73\x2E\x63\x72\x6C\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x08\x56\xFC\x02\xF0\x9B\xE8\xFF\xA4\xFA\xD6\x7B\xC6\x44\x80\xCE\x4F\xC4\xC5\xF6\x00\x58\xCC\xA6\xB6\xBC\x14\x49\x68\x04\x76\xE8\xE6\xEE\x5D\xEC\x02\x0F\x60\xD6\x8D\x50\x18\x4F\x26\x4E\x01\xE3\xE6\xB0\xA5\xEE\xBF\xBC\x74\x54\x41\xBF\xFD\xFC\x12\xB8\xC7\x4F\x5A\xF4\x89\x60\x05\x7F\x60\xB7\x05\x4A\xF3\xF6\xF1\xC2\xBF\xC4\xB9\x74\x86\xB6\x2D\x7D\x6B\xCC\xD2\xF3\x46\xDD\x2F\xC6\xE0\x6A\xC3\xC3\x34\x03\x2C\x7D\x96\xDD\x5A\xC2\x0E\xA7\x0A\x99\xC1\x05\x8B\xAB\x0C\x2F\xF3\x5C\x3A\xCF\x6C\x37\x55\x09\x87\xDE\x53\x40\x6C\x58\xEF\xFC\xB6\xAB\x65\x6E\x04\xF6\x1B\xDC\x3C\xE0\x5A\x15\xC6\x9E\xD9\xF1\x59\x48\x30\x21\x65\x03\x6C\xEC\xE9\x21\x73\xEC\x9B\x03\xA1\xE0\x37\xAD\xA0\x15\x18\x8F\xFA\xBA\x02\xCE\xA7\x2C\xA9\x10\x13\x2C\xD4\xE5\x08\x26\xAB\x22\x97\x60\xF8\x90\x5E\x74\xD4\xA2\x9A\x53\xBD\xF2\xA9\x68\xE0\xA2\x6E\xC2\xD7\x6C\xB1\xA3\x0F\x9E\xBF\xEB\x68\xE7\x56\xF2\xAE\xF2\xE3\x2B\x38\x3A\x09\x81\xB5\x6B\x85\xD7\xBE\x2D\xED\x3F\x1A\xB7\xB2\x63\xE2\xF5\x62\x2C\x82\xD4\x6A\x00\x41\x50\xF1\x39\x83\x9F\x95\xE9\x36\x96\x98\x6E",
 	["CN=Secure Certificate Services,O=Comodo CA Limited,L=Salford,ST=Greater Manchester,C=GB"] = "\x30\x82\x04\x3F\x30\x82\x03\x27\xA0\x03\x02\x01\x02\x02\x01\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x7E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x1B\x30\x19\x06\x03\x55\x04\x08\x0C\x12\x47\x72\x65\x61\x74\x65\x72\x20\x4D\x61\x6E\x63\x68\x65\x73\x74\x65\x72\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x0C\x07\x53\x61\x6C\x66\x6F\x72\x64\x31\x1A\x30\x18\x06\x03\x55\x04\x0A\x0C\x11\x43\x6F\x6D\x6F\x64\x6F\x20\x43\x41\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x24\x30\x22\x06\x03\x55\x04\x03\x0C\x1B\x53\x65\x63\x75\x72\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x20\x53\x65\x72\x76\x69\x63\x65\x73\x30\x1E\x17\x0D\x30\x34\x30\x31\x30\x31\x30\x30\x30\x30\x30\x30\x5A\x17\x0D\x32\x38\x31\x32\x33\x31\x32\x33\x35\x39\x35\x39\x5A\x30\x7E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x1B\x30\x19\x06\x03\x55\x04\x08\x0C\x12\x47\x72\x65\x61\x74\x65\x72\x20\x4D\x61\x6E\x63\x68\x65\x73\x74\x65\x72\x31\x10\x30\x0E\x06\x03\x55\x04\x07\x0C\x07\x53\x61\x6C\x66\x6F\x72\x64\x31\x1A\x30\x18\x06\x03\x55\x04\x0A\x0C\x11\x43\x6F\x6D\x6F\x64\x6F\x20\x43\x41\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x24\x30\x22\x06\x03\x55\x04\x03\x0C\x1B\x53\x65\x63\x75\x72\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x20\x53\x65\x72\x76\x69\x63\x65\x73\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xC0\x71\x33\x82\x8A\xD0\x70\xEB\x73\x87\x82\x40\xD5\x1D\xE4\xCB\xC9\x0E\x42\x90\xF9\xDE\x34\xB9\xA1\xBA\x11\xF4\x25\x85\xF3\xCC\x72\x6D\xF2\x7B\x97\x6B\xB3\x07\xF1\x77\x24\x91\x5F\x25\x8F\xF6\x74\x3D\xE4\x80\xC2\xF8\x3C\x0D\xF3\xBF\x40\xEA\xF7\xC8\x52\xD1\x72\x6F\xEF\xC8\xAB\x41\xB8\x6E\x2E\x17\x2A\x95\x69\x0C\xCD\xD2\x1E\x94\x7B\x2D\x94\x1D\xAA\x75\xD7\xB3\x98\xCB\xAC\xBC\x64\x53\x40\xBC\x8F\xAC\xAC\x36\xCB\x5C\xAD\xBB\xDD\xE0\x94\x17\xEC\xD1\x5C\xD0\xBF\xEF\xA5\x95\xC9\x90\xC5\xB0\xAC\xFB\x1B\x43\xDF\x7A\x08\x5D\xB7\xB8\xF2\x40\x1B\x2B\x27\x9E\x50\xCE\x5E\x65\x82\x88\x8C\x5E\xD3\x4E\x0C\x7A\xEA\x08\x91\xB6\x36\xAA\x2B\x42\xFB\xEA\xC2\xA3\x39\xE5\xDB\x26\x38\xAD\x8B\x0A\xEE\x19\x63\xC7\x1C\x24\xDF\x03\x78\xDA\xE6\xEA\xC1\x47\x1A\x0B\x0B\x46\x09\xDD\x02\xFC\xDE\xCB\x87\x5F\xD7\x30\x63\x68\xA1\xAE\xDC\x32\xA1\xBA\xBE\xFE\x44\xAB\x68\xB6\xA5\x17\x15\xFD\xBD\xD5\xA7\xA7\x9A\xE4\x44\x33\xE9\x88\x8E\xFC\xED\x51\xEB\x93\x71\x4E\xAD\x01\xE7\x44\x8E\xAB\x2D\xCB\xA8\xFE\x01\x49\x48\xF0\xC0\xDD\xC7\x68\xD8\x92\xFE\x3D\x02\x03\x01\x00\x01\xA3\x81\xC7\x30\x81\xC4\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x3C\xD8\x93\x88\xC2\xC0\x82\x09\xCC\x01\x99\x06\x93\x20\xE9\x9E\x70\x09\x63\x4F\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x81\x81\x06\x03\x55\x1D\x1F\x04\x7A\x30\x78\x30\x3B\xA0\x39\xA0\x37\x86\x35\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x63\x6F\x6D\x6F\x64\x6F\x63\x61\x2E\x63\x6F\x6D\x2F\x53\x65\x63\x75\x72\x65\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x53\x65\x72\x76\x69\x63\x65\x73\x2E\x63\x72\x6C\x30\x39\xA0\x37\xA0\x35\x86\x33\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x63\x6F\x6D\x6F\x64\x6F\x2E\x6E\x65\x74\x2F\x53\x65\x63\x75\x72\x65\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x53\x65\x72\x76\x69\x63\x65\x73\x2E\x63\x72\x6C\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x87\x01\x6D\x23\x1D\x7E\x5B\x17\x7D\xC1\x61\x32\xCF\x8F\xE7\xF3\x8A\x94\x59\x66\xE0\x9E\x28\xA8\x5E\xD3\xB7\xF4\x34\xE6\xAA\x39\xB2\x97\x16\xC5\x82\x6F\x32\xA4\xE9\x8C\xE7\xAF\xFD\xEF\xC2\xE8\xB9\x4B\xAA\xA3\xF4\xE6\xDA\x8D\x65\x21\xFB\xBA\x80\xEB\x26\x28\x85\x1A\xFE\x39\x8C\xDE\x5B\x04\x04\xB4\x54\xF9\xA3\x67\x9E\x41\xFA\x09\x52\xCC\x05\x48\xA8\xC9\x3F\x21\x04\x1E\xCE\x48\x6B\xFC\x85\xE8\xC2\x7B\xAF\x7F\xB7\xCC\xF8\x5F\x3A\xFD\x35\xC6\x0D\xEF\x97\xDC\x4C\xAB\x11\xE1\x6B\xCB\x31\xD1\x6C\xFB\x48\x80\xAB\xDC\x9C\x37\xB8\x21\x14\x4B\x0D\x71\x3D\xEC\x83\x33\x6E\xD1\x6E\x32\x16\xEC\x98\xC7\x16\x8B\x59\xA6\x34\xAB\x05\x57\x2D\x93\xF7\xAA\x13\xCB\xD2\x13\xE2\xB7\x2E\x3B\xCD\x6B\x50\x17\x09\x68\x3E\xB5\x26\x57\xEE\xB6\xE0\xB6\xDD\xB9\x29\x80\x79\x7D\x8F\xA3\xF0\xA4\x28\xA4\x15\xC4\x85\xF4\x27\xD4\x6B\xBF\xE5\x5C\xE4\x65\x02\x76\x54\xB4\xE3\x37\x66\x24\xD3\x19\x61\xC8\x52\x10\xE5\x8B\x37\x9A\xB9\xA9\xF9\x1D\xBF\xEA\x99\x92\x61\x96\xFF\x01\xCD\xA1\x5F\x0D\xBC\x71\xBC\x0E\xAC\x0B\x1D\x47\x45\x1D\xC1\xEC\x7C\xEC\xFD\x29",
@@ -51,7 +48,6 @@ redef root_certs += {
 	["CN=Sonera Class2 CA,O=Sonera,C=FI"] = "\x30\x82\x03\x20\x30\x82\x02\x08\xA0\x03\x02\x01\x02\x02\x01\x1D\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x39\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x49\x31\x0F\x30\x0D\x06\x03\x55\x04\x0A\x13\x06\x53\x6F\x6E\x65\x72\x61\x31\x19\x30\x17\x06\x03\x55\x04\x03\x13\x10\x53\x6F\x6E\x65\x72\x61\x20\x43\x6C\x61\x73\x73\x32\x20\x43\x41\x30\x1E\x17\x0D\x30\x31\x30\x34\x30\x36\x30\x37\x32\x39\x34\x30\x5A\x17\x0D\x32\x31\x30\x34\x30\x36\x30\x37\x32\x39\x34\x30\x5A\x30\x39\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x46\x49\x31\x0F\x30\x0D\x06\x03\x55\x04\x0A\x13\x06\x53\x6F\x6E\x65\x72\x61\x31\x19\x30\x17\x06\x03\x55\x04\x03\x13\x10\x53\x6F\x6E\x65\x72\x61\x20\x43\x6C\x61\x73\x73\x32\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\x90\x17\x4A\x35\x9D\xCA\xF0\x0D\x96\xC7\x44\xFA\x16\x37\xFC\x48\xBD\xBD\x7F\x80\x2D\x35\x3B\xE1\x6F\xA8\x67\xA9\xBF\x03\x1C\x4D\x8C\x6F\x32\x47\xD5\x41\x68\xA4\x13\x04\xC1\x35\x0C\x9A\x84\x43\xFC\x5C\x1D\xFF\x89\xB3\xE8\x17\x18\xCD\x91\x5F\xFB\x89\xE3\xEA\xBF\x4E\x5D\x7C\x1B\x26\xD3\x75\x79\xED\xE6\x84\xE3\x57\xE5\xAD\x29\xC4\xF4\x3A\x28\xE7\xA5\x7B\x84\x36\x69\xB3\xFD\x5E\x76\xBD\xA3\x2D\x99\xD3\x90\x4E\x23\x28\x7D\x18\x63\xF1\x54\x3B\x26\x9D\x76\x5B\x97\x42\xB2\xFF\xAE\xF0\x4E\xEC\xDD\x39\x95\x4E\x83\x06\x7F\xE7\x49\x40\xC8\xC5\x01\xB2\x54\x5A\x66\x1D\x3D\xFC\xF9\xE9\x3C\x0A\x9E\x81\xB8\x70\xF0\x01\x8B\xE4\x23\x54\x7C\xC8\xAE\xF8\x90\x1E\x00\x96\x72\xD4\x54\xCF\x61\x23\xBC\xEA\xFB\x9D\x02\x95\xD1\xB6\xB9\x71\x3A\x69\x08\x3F\x0F\xB4\xE1\x42\xC7\x88\xF5\x3F\x98\xA8\xA7\xBA\x1C\xE0\x71\x71\xEF\x58\x57\x81\x50\x7A\x5C\x6B\x74\x46\x0E\x83\x03\x98\xC3\x8E\xA8\x6E\xF2\x76\x32\x6E\x27\x83\xC2\x73\xF3\xDC\x18\xE8\xB4\x93\xEA\x75\x44\x6B\x04\x60\x20\x71\x57\x87\x9D\xF3\xBE\xA0\x90\x23\x3D\x8A\x24\xE1\xDA\x21\xDB\xC3\x02\x03\x01\x00\x01\xA3\x33\x30\x31\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x11\x06\x03\x55\x1D\x0E\x04\x0A\x04\x08\x4A\xA0\xAA\x58\x84\xD3\x5E\x3C\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\x06\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x5A\xCE\x87\xF9\x16\x72\x15\x57\x4B\x1D\xD9\x9B\xE7\xA2\x26\x30\xEC\x93\x67\xDF\xD6\x2D\xD2\x34\xAF\xF7\x38\xA5\xCE\xAB\x16\xB9\xAB\x2F\x7C\x35\xCB\xAC\xD0\x0F\xB4\x4C\x2B\xFC\x80\xEF\x6B\x8C\x91\x5F\x36\x76\xF7\xDB\xB3\x1B\x19\xEA\xF4\xB2\x11\xFD\x61\x71\x44\xBF\x28\xB3\x3A\x1D\xBF\xB3\x43\xE8\x9F\xBF\xDC\x31\x08\x71\xB0\x9D\x8D\xD6\x34\x47\x32\x90\xC6\x65\x24\xF7\xA0\x4A\x7C\x04\x73\x8F\x39\x6F\x17\x8C\x72\xB5\xBD\x4B\xC8\x7A\xF8\x7B\x83\xC3\x28\x4E\x9C\x09\xEA\x67\x3F\xB2\x67\x04\x1B\xC3\x14\xDA\xF8\xE7\x49\x24\x91\xD0\x1D\x6A\xFA\x61\x39\xEF\x6B\xE7\x21\x75\x06\x07\xD8\x12\xB4\x21\x20\x70\x42\x71\x81\xDA\x3C\x9A\x36\xBE\xA6\x5B\x0D\x6A\x6C\x9A\x1F\x91\x7B\xF9\xF9\xEF\x42\xBA\x4E\x4E\x9E\xCC\x0C\x8D\x94\xDC\xD9\x45\x9C\x5E\xEC\x42\x50\x63\xAE\xF4\x5D\xC4\xB1\x12\xDC\xCA\x3B\xA8\x2E\x9D\x14\x5A\x05\x75\xB7\xEC\xD7\x63\xE2\xBA\x35\xB6\x04\x08\x91\xE8\xDA\x9D\x9C\xF6\x66\xB5\x18\xAC\x0A\xA6\x54\x26\x34\x33\xD2\x1B\xC1\xD4\x7F\x1A\x3A\x8E\x0B\xAA\x32\x6E\xDB\xFC\x4F\x25\x9F\xD9\x32\xC7\x96\x5A\x70\xAC\xDF\x4C",
 	["CN=Staat der Nederlanden Root CA,O=Staat der Nederlanden,C=NL"] = "\x30\x82\x03\xBA\x30\x82\x02\xA2\xA0\x03\x02\x01\x02\x02\x04\x00\x98\x96\x8A\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x55\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4E\x4C\x31\x1E\x30\x1C\x06\x03\x55\x04\x0A\x13\x15\x53\x74\x61\x61\x74\x20\x64\x65\x72\x20\x4E\x65\x64\x65\x72\x6C\x61\x6E\x64\x65\x6E\x31\x26\x30\x24\x06\x03\x55\x04\x03\x13\x1D\x53\x74\x61\x61\x74\x20\x64\x65\x72\x20\x4E\x65\x64\x65\x72\x6C\x61\x6E\x64\x65\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x30\x32\x31\x32\x31\x37\x30\x39\x32\x33\x34\x39\x5A\x17\x0D\x31\x35\x31\x32\x31\x36\x30\x39\x31\x35\x33\x38\x5A\x30\x55\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4E\x4C\x31\x1E\x30\x1C\x06\x03\x55\x04\x0A\x13\x15\x53\x74\x61\x61\x74\x20\x64\x65\x72\x20\x4E\x65\x64\x65\x72\x6C\x61\x6E\x64\x65\x6E\x31\x26\x30\x24\x06\x03\x55\x04\x03\x13\x1D\x53\x74\x61\x61\x74\x20\x64\x65\x72\x20\x4E\x65\x64\x65\x72\x6C\x61\x6E\x64\x65\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\x98\xD2\xB5\x51\x11\x7A\x81\xA6\x14\x98\x71\x6D\xBE\xCC\xE7\x13\x1B\xD6\x27\x0E\x7A\xB3\x6A\x18\x1C\xB6\x61\x5A\xD5\x61\x09\xBF\xDE\x90\x13\xC7\x67\xEE\xDD\xF3\xDA\xC5\x0C\x12\x9E\x35\x55\x3E\x2C\x27\x88\x40\x6B\xF7\xDC\xDD\x22\x61\xF5\xC2\xC7\x0E\xF5\xF6\xD5\x76\x53\x4D\x8F\x8C\xBC\x18\x76\x37\x85\x9D\xE8\xCA\x49\xC7\xD2\x4F\x98\x13\x09\xA2\x3E\x22\x88\x9C\x7F\xD6\xF2\x10\x65\xB4\xEE\x5F\x18\xD5\x17\xE3\xF8\xC5\xFD\xE2\x9D\xA2\xEF\x53\x0E\x85\x77\xA2\x0F\xE1\x30\x47\xEE\x00\xE7\x33\x7D\x44\x67\x1A\x0B\x51\xE8\x8B\xA0\x9E\x50\x98\x68\x34\x52\x1F\x2E\x6D\x01\xF2\x60\x45\xF2\x31\xEB\xA9\x31\x68\x29\xBB\x7A\x41\x9E\xC6\x19\x7F\x94\xB4\x51\x39\x03\x7F\xB2\xDE\xA7\x32\x9B\xB4\x47\x8E\x6F\xB4\x4A\xAE\xE5\xAF\xB1\xDC\xB0\x1B\x61\xBC\x99\x72\xDE\xE4\x89\xB7\x7A\x26\x5D\xDA\x33\x49\x5B\x52\x9C\x0E\xF5\x8A\xAD\xC3\xB8\x3D\xE8\x06\x6A\xC2\xD5\x2A\x0B\x6C\x7B\x84\xBD\x56\x05\xCB\x86\x65\x92\xEC\x44\x2B\xB0\x8E\xB9\xDC\x70\x0B\x46\xDA\xAD\xBC\x63\x88\x39\xFA\xDB\x6A\xFE\x23\xFA\xBC\xE4\x48\xF4\x67\x2B\x6A\x11\x10\x21\x49\x02\x03\x01\x00\x01\xA3\x81\x91\x30\x81\x8E\x30\x0C\x06\x03\x55\x1D\x13\x04\x05\x30\x03\x01\x01\xFF\x30\x4F\x06\x03\x55\x1D\x20\x04\x48\x30\x46\x30\x44\x06\x04\x55\x1D\x20\x00\x30\x3C\x30\x3A\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x2E\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x70\x6B\x69\x6F\x76\x65\x72\x68\x65\x69\x64\x2E\x6E\x6C\x2F\x70\x6F\x6C\x69\x63\x69\x65\x73\x2F\x72\x6F\x6F\x74\x2D\x70\x6F\x6C\x69\x63\x79\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xA8\x7D\xEB\xBC\x63\xA4\x74\x13\x74\x00\xEC\x96\xE0\xD3\x34\xC1\x2C\xBF\x6C\xF8\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x05\x84\x87\x55\x74\x36\x61\xC1\xBB\xD1\xD4\xC6\x15\xA8\x13\xB4\x9F\xA4\xFE\xBB\xEE\x15\xB4\x2F\x06\x0C\x29\xF2\xA8\x92\xA4\x61\x0D\xFC\xAB\x5C\x08\x5B\x51\x13\x2B\x4D\xC2\x2A\x61\xC8\xF8\x09\x58\xFC\x2D\x02\xB2\x39\x7D\x99\x66\x81\xBF\x6E\x5C\x95\x45\x20\x6C\xE6\x79\xA7\xD1\xD8\x1C\x29\xFC\xC2\x20\x27\x51\xC8\xF1\x7C\x5D\x34\x67\x69\x85\x11\x30\xC6\x00\xD2\xD7\xF3\xD3\x7C\xB6\xF0\x31\x57\x28\x12\x82\x73\xE9\x33\x2F\xA6\x55\xB4\x0B\x91\x94\x47\x9C\xFA\xBB\x7A\x42\x32\xE8\xAE\x7E\x2D\xC8\xBC\xAC\x14\xBF\xD9\x0F\xD9\x5B\xFC\xC1\xF9\x7A\x95\xE1\x7D\x7E\x96\xFC\x71\xB0\xC2\x4C\xC8\xDF\x45\x34\xC9\xCE\x0D\xF2\x9C\x64\x08\xD0\x3B\xC3\x29\xC5\xB2\xED\x90\x04\xC1\xB1\x29\x91\xC5\x30\x6F\xC1\xA9\x72\x33\xCC\xFE\x5D\x16\x17\x2C\x11\x69\xE7\x7E\xFE\xC5\x83\x08\xDF\xBC\xDC\x22\x3A\x2E\x20\x69\x23\x39\x56\x60\x67\x90\x8B\x2E\x76\x39\xFB\x11\x88\x97\xF6\x7C\xBD\x4B\xB8\x20\x16\x67\x05\x8D\xE2\x3B\xC1\x72\x3F\x94\x95\x37\xC7\x5D\xB9\x9E\xD8\x93\xA1\x17\x8F\xFF\x0C\x66\x15\xC1\x24\x7C\x32\x7C\x03\x1D\x3B\xA1\x58\x45\x32\x93",
 	["OU=TDC Internet Root CA,O=TDC Internet,C=DK"] = "\x30\x82\x04\x2B\x30\x82\x03\x13\xA0\x03\x02\x01\x02\x02\x04\x3A\xCC\xA5\x4C\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x43\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x4B\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x54\x44\x43\x20\x49\x6E\x74\x65\x72\x6E\x65\x74\x31\x1D\x30\x1B\x06\x03\x55\x04\x0B\x13\x14\x54\x44\x43\x20\x49\x6E\x74\x65\x72\x6E\x65\x74\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x30\x31\x30\x34\x30\x35\x31\x36\x33\x33\x31\x37\x5A\x17\x0D\x32\x31\x30\x34\x30\x35\x31\x37\x30\x33\x31\x37\x5A\x30\x43\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x4B\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x54\x44\x43\x20\x49\x6E\x74\x65\x72\x6E\x65\x74\x31\x1D\x30\x1B\x06\x03\x55\x04\x0B\x13\x14\x54\x44\x43\x20\x49\x6E\x74\x65\x72\x6E\x65\x74\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xC4\xB8\x40\xBC\x91\xD5\x63\x1F\xD7\x99\xA0\x8B\x0C\x40\x1E\x74\xB7\x48\x9D\x46\x8C\x02\xB2\xE0\x24\x5F\xF0\x19\x13\xA7\x37\x83\x6B\x5D\xC7\x8E\xF9\x84\x30\xCE\x1A\x3B\xFA\xFB\xCE\x8B\x6D\x23\xC6\xC3\x6E\x66\x9F\x89\xA5\xDF\xE0\x42\x50\x67\xFA\x1F\x6C\x1E\xF4\xD0\x05\xD6\xBF\xCA\xD6\x4E\xE4\x68\x60\x6C\x46\xAA\x1C\x5D\x63\xE1\x07\x86\x0E\x65\x00\xA7\x2E\xA6\x71\xC6\xBC\xB9\x81\xA8\x3A\x7D\x1A\xD2\xF9\xD1\xAC\x4B\xCB\xCE\x75\xAF\xDC\x7B\xFA\x81\x73\xD4\xFC\xBA\xBD\x41\x88\xD4\x74\xB3\xF9\x5E\x38\x3A\x3C\x43\xA8\xD2\x95\x4E\x77\x6D\x13\x0C\x9D\x8F\x78\x01\xB7\x5A\x20\x1F\x03\x37\x35\xE2\x2C\xDB\x4B\x2B\x2C\x78\xB9\x49\xDB\xC4\xD0\xC7\x9C\x9C\xE4\x8A\x20\x09\x21\x16\x56\x66\xFF\x05\xEC\x5B\xE3\xF0\xCF\xAB\x24\x24\x5E\xC3\x7F\x70\x7A\x12\xC4\xD2\xB5\x10\xA0\xB6\x21\xE1\x8D\x78\x69\x55\x44\x69\xF5\xCA\x96\x1C\x34\x85\x17\x25\x77\xE2\xF6\x2F\x27\x98\x78\xFD\x79\x06\x3A\xA2\xD6\x5A\x43\xC1\xFF\xEC\x04\x3B\xEE\x13\xEF\xD3\x58\x5A\xFF\x92\xEB\xEC\xAE\xDA\xF2\x37\x03\x47\x41\xB6\x97\xC9\x2D\x0A\x41\x22\xBB\xBB\xE6\xA7\x02\x03\x01\x00\x01\xA3\x82\x01\x25\x30\x82\x01\x21\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x65\x06\x03\x55\x1D\x1F\x04\x5E\x30\x5C\x30\x5A\xA0\x58\xA0\x56\xA4\x54\x30\x52\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x4B\x31\x15\x30\x13\x06\x03\x55\x04\x0A\x13\x0C\x54\x44\x43\x20\x49\x6E\x74\x65\x72\x6E\x65\x74\x31\x1D\x30\x1B\x06\x03\x55\x04\x0B\x13\x14\x54\x44\x43\x20\x49\x6E\x74\x65\x72\x6E\x65\x74\x20\x52\x6F\x6F\x74\x20\x43\x41\x31\x0D\x30\x0B\x06\x03\x55\x04\x03\x13\x04\x43\x52\x4C\x31\x30\x2B\x06\x03\x55\x1D\x10\x04\x24\x30\x22\x80\x0F\x32\x30\x30\x31\x30\x34\x30\x35\x31\x36\x33\x33\x31\x37\x5A\x81\x0F\x32\x30\x32\x31\x30\x34\x30\x35\x31\x37\x30\x33\x31\x37\x5A\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\x06\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x6C\x64\x01\xC7\xFD\x85\x6D\xAC\xC8\xDA\x9E\x50\x08\x85\x08\xB5\x3C\x56\xA8\x50\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x6C\x64\x01\xC7\xFD\x85\x6D\xAC\xC8\xDA\x9E\x50\x08\x85\x08\xB5\x3C\x56\xA8\x50\x30\x0C\x06\x03\x55\x1D\x13\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x09\x2A\x86\x48\x86\xF6\x7D\x07\x41\x00\x04\x10\x30\x0E\x1B\x08\x56\x35\x2E\x30\x3A\x34\x2E\x30\x03\x02\x04\x90\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x4E\x43\xCC\xD1\xDD\x1D\x10\x1B\x06\x7F\xB7\xA4\xFA\xD3\xD9\x4D\xFB\x23\x9F\x23\x54\x5B\xE6\x8B\x2F\x04\x28\x8B\xB5\x27\x6D\x89\xA1\xEC\x98\x69\xDC\xE7\x8D\x26\x83\x05\x79\x74\xEC\xB4\xB9\xA3\x97\xC1\x35\x00\xFD\x15\xDA\x39\x81\x3A\x95\x31\x90\xDE\x97\xE9\x86\xA8\x99\x77\x0C\xE5\x5A\xA0\x84\xFF\x12\x16\xAC\x6E\xB8\x8D\xC3\x7B\x92\xC2\xAC\x2E\xD0\x7D\x28\xEC\xB6\xF3\x60\x38\x69\x6F\x3E\xD8\x04\x55\x3E\x9E\xCC\x55\xD2\xBA\xFE\xBB\x47\x04\xD7\x0A\xD9\x16\x0A\x34\x29\xF5\x58\x13\xD5\x4F\xCF\x8F\x56\x4B\xB3\x1E\xEE\xD3\x98\x79\xDA\x08\x1E\x0C\x6F\xB8\xF8\x16\x27\xEF\xC2\x6F\x3D\xF6\xA3\x4B\x3E\x0E\xE4\x6D\x6C\xDB\x3B\x41\x12\x9B\xBD\x0D\x47\x23\x7F\x3C\x4A\xD0\xAF\xC0\xAF\xF6\xEF\x1B\xB5\x15\xC4\xEB\x83\xC4\x09\x5F\x74\x8B\xD9\x11\xFB\xC2\x56\xB1\x3C\xF8\x70\xCA\x34\x8D\x43\x40\x13\x8C\xFD\x99\x03\x54\x79\xC6\x2E\xEA\x86\xA1\xF6\x3A\xD4\x09\xBC\xF4\xBC\x66\xCC\x3D\x58\xD0\x57\x49\x0A\xEE\x25\xE2\x41\xEE\x13\xF9\x9B\x38\x34\xD1\x00\xF5\x7E\xE7\x94\x1D\xFC\x69\x03\x62\xB8\x99\x05\x05\x3D\x6B\x78\x12\xBD\xB0\x6F\x65",
-	["CN=TDC OCES CA,O=TDC,C=DK"] = "\x30\x82\x05\x19\x30\x82\x04\x01\xA0\x03\x02\x01\x02\x02\x04\x3E\x48\xBD\xC4\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x31\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x4B\x31\x0C\x30\x0A\x06\x03\x55\x04\x0A\x13\x03\x54\x44\x43\x31\x14\x30\x12\x06\x03\x55\x04\x03\x13\x0B\x54\x44\x43\x20\x4F\x43\x45\x53\x20\x43\x41\x30\x1E\x17\x0D\x30\x33\x30\x32\x31\x31\x30\x38\x33\x39\x33\x30\x5A\x17\x0D\x33\x37\x30\x32\x31\x31\x30\x39\x30\x39\x33\x30\x5A\x30\x31\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x4B\x31\x0C\x30\x0A\x06\x03\x55\x04\x0A\x13\x03\x54\x44\x43\x31\x14\x30\x12\x06\x03\x55\x04\x03\x13\x0B\x54\x44\x43\x20\x4F\x43\x45\x53\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xAC\x62\xF6\x61\x20\xB2\xCF\xC0\xC6\x85\xD7\xE3\x79\xE6\xCC\xED\xF2\x39\x92\xA4\x97\x2E\x64\xA3\x84\x5B\x87\x9C\x4C\xFD\xA4\xF3\xC4\x5F\x21\xBD\x56\x10\xEB\xDB\x2E\x61\xEC\x93\x69\xE3\xA3\xCC\xBD\x99\xC3\x05\xFC\x06\xB8\xCA\x36\x1C\xFE\x90\x8E\x49\x4C\xC4\x56\x9A\x2F\x56\xBC\xCF\x7B\x0C\xF1\x6F\x47\xA6\x0D\x43\x4D\xE2\xE9\x1D\x39\x34\xCD\x8D\x2C\xD9\x12\x98\xF9\xE3\xE1\xC1\x4A\x7C\x86\x38\xC4\xA9\xC4\x61\x88\xD2\x5E\xAF\x1A\x26\x4D\xD5\xE4\xA0\x22\x47\x84\xD9\x64\xB7\x19\x96\xFC\xEC\x19\xE4\xB2\x97\x26\x4E\x4A\x4C\xCB\x8F\x24\x8B\x54\x18\x1C\x48\x61\x7B\xD5\x88\x68\xDA\x5D\xB5\xEA\xCD\x1A\x30\xC1\x80\x83\x76\x50\xAA\x4F\xD1\xD4\xDD\x38\xF0\xEF\x16\xF4\xE1\x0C\x50\x06\xBF\xEA\xFB\x7A\x49\xA1\x28\x2B\x1C\xF6\xFC\x15\x32\xA3\x74\x6A\x8F\xA9\xC3\x62\x29\x71\x31\xE5\x3B\xA4\x60\x17\x5E\x74\xE6\xDA\x13\xED\xE9\x1F\x1F\x1B\xD1\xB2\x68\x73\xC6\x10\x34\x75\x46\x10\x10\xE3\x90\x00\x76\x40\xCB\x8B\xB7\x43\x09\x21\xFF\xAB\x4E\x93\xC6\x58\xE9\xA5\x82\xDB\x77\xC4\x3A\x99\xB1\x72\x95\x49\x04\xF0\xB7\x2B\xFA\x7B\x59\x8E\xDD\x02\x03\x01\x00\x01\xA3\x82\x02\x37\x30\x82\x02\x33\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x81\xEC\x06\x03\x55\x1D\x20\x04\x81\xE4\x30\x81\xE1\x30\x81\xDE\x06\x08\x2A\x81\x50\x81\x29\x01\x01\x01\x30\x81\xD1\x30\x2F\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x23\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x65\x72\x74\x69\x66\x69\x6B\x61\x74\x2E\x64\x6B\x2F\x72\x65\x70\x6F\x73\x69\x74\x6F\x72\x79\x30\x81\x9D\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x02\x30\x81\x90\x30\x0A\x16\x03\x54\x44\x43\x30\x03\x02\x01\x01\x1A\x81\x81\x43\x65\x72\x74\x69\x66\x69\x6B\x61\x74\x65\x72\x20\x66\x72\x61\x20\x64\x65\x6E\x6E\x65\x20\x43\x41\x20\x75\x64\x73\x74\x65\x64\x65\x73\x20\x75\x6E\x64\x65\x72\x20\x4F\x49\x44\x20\x31\x2E\x32\x2E\x32\x30\x38\x2E\x31\x36\x39\x2E\x31\x2E\x31\x2E\x31\x2E\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x65\x73\x20\x66\x72\x6F\x6D\x20\x74\x68\x69\x73\x20\x43\x41\x20\x61\x72\x65\x20\x69\x73\x73\x75\x65\x64\x20\x75\x6E\x64\x65\x72\x20\x4F\x49\x44\x20\x31\x2E\x32\x2E\x32\x30\x38\x2E\x31\x36\x39\x2E\x31\x2E\x31\x2E\x31\x2E\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x81\x81\x06\x03\x55\x1D\x1F\x04\x7A\x30\x78\x30\x48\xA0\x46\xA0\x44\xA4\x42\x30\x40\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x44\x4B\x31\x0C\x30\x0A\x06\x03\x55\x04\x0A\x13\x03\x54\x44\x43\x31\x14\x30\x12\x06\x03\x55\x04\x03\x13\x0B\x54\x44\x43\x20\x4F\x43\x45\x53\x20\x43\x41\x31\x0D\x30\x0B\x06\x03\x55\x04\x03\x13\x04\x43\x52\x4C\x31\x30\x2C\xA0\x2A\xA0\x28\x86\x26\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x6F\x63\x65\x73\x2E\x63\x65\x72\x74\x69\x66\x69\x6B\x61\x74\x2E\x64\x6B\x2F\x6F\x63\x65\x73\x2E\x63\x72\x6C\x30\x2B\x06\x03\x55\x1D\x10\x04\x24\x30\x22\x80\x0F\x32\x30\x30\x33\x30\x32\x31\x31\x30\x38\x33\x39\x33\x30\x5A\x81\x0F\x32\x30\x33\x37\x30\x32\x31\x31\x30\x39\x30\x39\x33\x30\x5A\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x60\xB5\x85\xEC\x56\x64\x7E\x12\x19\x27\x67\x1D\x50\x15\x4B\x73\xAE\x3B\xF9\x12\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x60\xB5\x85\xEC\x56\x64\x7E\x12\x19\x27\x67\x1D\x50\x15\x4B\x73\xAE\x3B\xF9\x12\x30\x1D\x06\x09\x2A\x86\x48\x86\xF6\x7D\x07\x41\x00\x04\x10\x30\x0E\x1B\x08\x56\x36\x2E\x30\x3A\x34\x2E\x30\x03\x02\x04\x90\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x0A\xBA\x26\x26\x46\xD3\x73\xA8\x09\xF3\x6B\x0B\x30\x99\xFD\x8A\xE1\x57\x7A\x11\xD3\xB8\x94\xD7\x09\x10\x6E\xA3\xB1\x38\x03\xD1\xB6\xF2\x43\x41\x29\x62\xA7\x72\xD8\xFB\x7C\x05\xE6\x31\x70\x27\x54\x18\x4E\x8A\x7C\x4E\xE5\xD1\xCA\x8C\x78\x88\xCF\x1B\xD3\x90\x8B\xE6\x23\xF8\x0B\x0E\x33\x43\x7D\x9C\xE2\x0A\x19\x8F\xC9\x01\x3E\x74\x5D\x74\xC9\x8B\x1C\x03\xE5\x18\xC8\x01\x4C\x3F\xCB\x97\x05\x5D\x98\x71\xA6\x98\x6F\xB6\x7C\xBD\x37\x7F\xBE\xE1\x93\x25\x6D\x6F\xF0\x0A\xAD\x17\x18\xE1\x03\xBC\x07\x29\xC8\xAD\x26\xE8\xF8\x61\xF0\xFD\x21\x09\x7E\x9A\x8E\xA9\x68\x7D\x48\x62\x72\xBD\x00\xEA\x01\x99\xB8\x06\x82\x51\x81\x4E\xF1\xF5\xB4\x91\x54\xB9\x23\x7A\x00\x9A\x9F\x5D\x8D\xE0\x3C\x64\xB9\x1A\x12\x92\x2A\xC7\x82\x44\x72\x39\xDC\xE2\x3C\xC6\xD8\x55\xF5\x15\x4E\xC8\x05\x0E\xDB\xC6\xD0\x62\xA6\xEC\x15\xB4\xB5\x02\x82\xDB\xAC\x8C\xA2\x81\xF0\x9B\x99\x31\xF5\x20\x20\xA8\x88\x61\x0A\x07\x9F\x94\xFC\xD0\xD7\x1B\xCC\x2E\x17\xF3\x04\x27\x76\x67\xEB\x54\x83\xFD\xA4\x90\x7E\x06\x3D\x04\xA3\x43\x2D\xDA\xFC\x0B\x62\xEA\x2F\x5F\x62\x53",
 	["CN=UTN - DATACorp SGC,OU=http://www.usertrust.com,O=The USERTRUST Network,L=Salt Lake City,ST=UT,C=US"] = "\x30\x82\x04\x5E\x30\x82\x03\x46\xA0\x03\x02\x01\x02\x02\x10\x44\xBE\x0C\x8B\x50\x00\x21\xB4\x11\xD3\x2A\x68\x06\xA9\xAD\x69\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\x93\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x0B\x30\x09\x06\x03\x55\x04\x08\x13\x02\x55\x54\x31\x17\x30\x15\x06\x03\x55\x04\x07\x13\x0E\x53\x61\x6C\x74\x20\x4C\x61\x6B\x65\x20\x43\x69\x74\x79\x31\x1E\x30\x1C\x06\x03\x55\x04\x0A\x13\x15\x54\x68\x65\x20\x55\x53\x45\x52\x54\x52\x55\x53\x54\x20\x4E\x65\x74\x77\x6F\x72\x6B\x31\x21\x30\x1F\x06\x03\x55\x04\x0B\x13\x18\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x75\x73\x65\x72\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x55\x54\x4E\x20\x2D\x20\x44\x41\x54\x41\x43\x6F\x72\x70\x20\x53\x47\x43\x30\x1E\x17\x0D\x39\x39\x30\x36\x32\x34\x31\x38\x35\x37\x32\x31\x5A\x17\x0D\x31\x39\x30\x36\x32\x34\x31\x39\x30\x36\x33\x30\x5A\x30\x81\x93\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x0B\x30\x09\x06\x03\x55\x04\x08\x13\x02\x55\x54\x31\x17\x30\x15\x06\x03\x55\x04\x07\x13\x0E\x53\x61\x6C\x74\x20\x4C\x61\x6B\x65\x20\x43\x69\x74\x79\x31\x1E\x30\x1C\x06\x03\x55\x04\x0A\x13\x15\x54\x68\x65\x20\x55\x53\x45\x52\x54\x52\x55\x53\x54\x20\x4E\x65\x74\x77\x6F\x72\x6B\x31\x21\x30\x1F\x06\x03\x55\x04\x0B\x13\x18\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x75\x73\x65\x72\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x31\x1B\x30\x19\x06\x03\x55\x04\x03\x13\x12\x55\x54\x4E\x20\x2D\x20\x44\x41\x54\x41\x43\x6F\x72\x70\x20\x53\x47\x43\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xDF\xEE\x58\x10\xA2\x2B\x6E\x55\xC4\x8E\xBF\x2E\x46\x09\xE7\xE0\x08\x0F\x2E\x2B\x7A\x13\x94\x1B\xBD\xF6\xB6\x80\x8E\x65\x05\x93\x00\x1E\xBC\xAF\xE2\x0F\x8E\x19\x0D\x12\x47\xEC\xAC\xAD\xA3\xFA\x2E\x70\xF8\xDE\x6E\xFB\x56\x42\x15\x9E\x2E\x5C\xEF\x23\xDE\x21\xB9\x05\x76\x27\x19\x0F\x4F\xD6\xC3\x9C\xB4\xBE\x94\x19\x63\xF2\xA6\x11\x0A\xEB\x53\x48\x9C\xBE\xF2\x29\x3B\x16\xE8\x1A\xA0\x4C\xA6\xC9\xF4\x18\x59\x68\xC0\x70\xF2\x53\x00\xC0\x5E\x50\x82\xA5\x56\x6F\x36\xF9\x4A\xE0\x44\x86\xA0\x4D\x4E\xD6\x47\x6E\x49\x4A\xCB\x67\xD7\xA6\xC4\x05\xB9\x8E\x1E\xF4\xFC\xFF\xCD\xE7\x36\xE0\x9C\x05\x6C\xB2\x33\x22\x15\xD0\xB4\xE0\xCC\x17\xC0\xB2\xC0\xF4\xFE\x32\x3F\x29\x2A\x95\x7B\xD8\xF2\xA7\x4E\x0F\x54\x7C\xA1\x0D\x80\xB3\x09\x03\xC1\xFF\x5C\xDD\x5E\x9A\x3E\xBC\xAE\xBC\x47\x8A\x6A\xAE\x71\xCA\x1F\xB1\x2A\xB8\x5F\x42\x05\x0B\xEC\x46\x30\xD1\x72\x0B\xCA\xE9\x56\x6D\xF5\xEF\xDF\x78\xBE\x61\xBA\xB2\xA5\xAE\x04\x4C\xBC\xA8\xAC\x69\x15\x97\xBD\xEF\xEB\xB4\x8C\xBF\x35\xF8\xD4\xC3\xD1\x28\x0E\x5C\x3A\x9F\x70\x18\x33\x20\x77\xC4\xA2\xAF\x02\x03\x01\x00\x01\xA3\x81\xAB\x30\x81\xA8\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\xC6\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x53\x32\xD1\xB3\xCF\x7F\xFA\xE0\xF1\xA0\x5D\x85\x4E\x92\xD2\x9E\x45\x1D\xB4\x4F\x30\x3D\x06\x03\x55\x1D\x1F\x04\x36\x30\x34\x30\x32\xA0\x30\xA0\x2E\x86\x2C\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x75\x73\x65\x72\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x2F\x55\x54\x4E\x2D\x44\x41\x54\x41\x43\x6F\x72\x70\x53\x47\x43\x2E\x63\x72\x6C\x30\x2A\x06\x03\x55\x1D\x25\x04\x23\x30\x21\x06\x08\x2B\x06\x01\x05\x05\x07\x03\x01\x06\x0A\x2B\x06\x01\x04\x01\x82\x37\x0A\x03\x03\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x04\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x27\x35\x97\x00\x8A\x8B\x28\xBD\xC6\x33\x30\x1E\x29\xFC\xE2\xF7\xD5\x98\xD4\x40\xBB\x60\xCA\xBF\xAB\x17\x2C\x09\x36\x7F\x50\xFA\x41\xDC\xAE\x96\x3A\x0A\x23\x3E\x89\x59\xC9\xA3\x07\xED\x1B\x37\xAD\xFC\x7C\xBE\x51\x49\x5A\xDE\x3A\x0A\x54\x08\x16\x45\xC2\x99\xB1\x87\xCD\x8C\x68\xE0\x69\x03\xE9\xC4\x4E\x98\xB2\x3B\x8C\x16\xB3\x0E\xA0\x0C\x98\x50\x9B\x93\xA9\x70\x09\xC8\x2C\xA3\x8F\xDF\x02\xE4\xE0\x71\x3A\xF1\xB4\x23\x72\xA0\xAA\x01\xDF\xDF\x98\x3E\x14\x50\xA0\x31\x26\xBD\x28\xE9\x5A\x30\x26\x75\xF9\x7B\x60\x1C\x8D\xF3\xCD\x50\x26\x6D\x04\x27\x9A\xDF\xD5\x0D\x45\x47\x29\x6B\x2C\xE6\x76\xD9\xA9\x29\x7D\x32\xDD\xC9\x36\x3C\xBD\xAE\x35\xF1\x11\x9E\x1D\xBB\x90\x3F\x12\x47\x4E\x8E\xD7\x7E\x0F\x62\x73\x1D\x52\x26\x38\x1C\x18\x49\xFD\x30\x74\x9A\xC4\xE5\x22\x2F\xD8\xC0\x8D\xED\x91\x7A\x4C\x00\x8F\x72\x7F\x5D\xDA\xDD\x1B\x8B\x45\x6B\xE7\xDD\x69\x97\xA8\xC5\x56\x4C\x0F\x0C\xF6\x9F\x7A\x91\x37\xF6\x97\x82\xE0\xDD\x71\x69\xFF\x76\x3F\x60\x4D\x3C\xCF\xF7\x99\xF9\xC6\x57\xF4\xC9\x55\x39\x78\xBA\x2C\x79\xC9\xA6\x88\x2B\xF4\x08",
 	["CN=UTN-USERFirst-Hardware,OU=http://www.usertrust.com,O=The USERTRUST Network,L=Salt Lake City,ST=UT,C=US"] = "\x30\x82\x04\x74\x30\x82\x03\x5C\xA0\x03\x02\x01\x02\x02\x10\x44\xBE\x0C\x8B\x50\x00\x24\xB4\x11\xD3\x36\x2A\xFE\x65\x0A\xFD\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\x97\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x0B\x30\x09\x06\x03\x55\x04\x08\x13\x02\x55\x54\x31\x17\x30\x15\x06\x03\x55\x04\x07\x13\x0E\x53\x61\x6C\x74\x20\x4C\x61\x6B\x65\x20\x43\x69\x74\x79\x31\x1E\x30\x1C\x06\x03\x55\x04\x0A\x13\x15\x54\x68\x65\x20\x55\x53\x45\x52\x54\x52\x55\x53\x54\x20\x4E\x65\x74\x77\x6F\x72\x6B\x31\x21\x30\x1F\x06\x03\x55\x04\x0B\x13\x18\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x75\x73\x65\x72\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x31\x1F\x30\x1D\x06\x03\x55\x04\x03\x13\x16\x55\x54\x4E\x2D\x55\x53\x45\x52\x46\x69\x72\x73\x74\x2D\x48\x61\x72\x64\x77\x61\x72\x65\x30\x1E\x17\x0D\x39\x39\x30\x37\x30\x39\x31\x38\x31\x30\x34\x32\x5A\x17\x0D\x31\x39\x30\x37\x30\x39\x31\x38\x31\x39\x32\x32\x5A\x30\x81\x97\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x0B\x30\x09\x06\x03\x55\x04\x08\x13\x02\x55\x54\x31\x17\x30\x15\x06\x03\x55\x04\x07\x13\x0E\x53\x61\x6C\x74\x20\x4C\x61\x6B\x65\x20\x43\x69\x74\x79\x31\x1E\x30\x1C\x06\x03\x55\x04\x0A\x13\x15\x54\x68\x65\x20\x55\x53\x45\x52\x54\x52\x55\x53\x54\x20\x4E\x65\x74\x77\x6F\x72\x6B\x31\x21\x30\x1F\x06\x03\x55\x04\x0B\x13\x18\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x75\x73\x65\x72\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x31\x1F\x30\x1D\x06\x03\x55\x04\x03\x13\x16\x55\x54\x4E\x2D\x55\x53\x45\x52\x46\x69\x72\x73\x74\x2D\x48\x61\x72\x64\x77\x61\x72\x65\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xB1\xF7\xC3\x38\x3F\xB4\xA8\x7F\xCF\x39\x82\x51\x67\xD0\x6D\x9F\xD2\xFF\x58\xF3\xE7\x9F\x2B\xEC\x0D\x89\x54\x99\xB9\x38\x99\x16\xF7\xE0\x21\x79\x48\xC2\xBB\x61\x74\x12\x96\x1D\x3C\x6A\x72\xD5\x3C\x10\x67\x3A\x39\xED\x2B\x13\xCD\x66\xEB\x95\x09\x33\xA4\x6C\x97\xB1\xE8\xC6\xEC\xC1\x75\x79\x9C\x46\x5E\x8D\xAB\xD0\x6A\xFD\xB9\x2A\x55\x17\x10\x54\xB3\x19\xF0\x9A\xF6\xF1\xB1\x5D\xB6\xA7\x6D\xFB\xE0\x71\x17\x6B\xA2\x88\xFB\x00\xDF\xFE\x1A\x31\x77\x0C\x9A\x01\x7A\xB1\x32\xE3\x2B\x01\x07\x38\x6E\xC3\xA5\x5E\x23\xBC\x45\x9B\x7B\x50\xC1\xC9\x30\x8F\xDB\xE5\x2B\x7A\xD3\x5B\xFB\x33\x40\x1E\xA0\xD5\x98\x17\xBC\x8B\x87\xC3\x89\xD3\x5D\xA0\x8E\xB2\xAA\xAA\xF6\x8E\x69\x88\x06\xC5\xFA\x89\x21\xF3\x08\x9D\x69\x2E\x09\x33\x9B\x29\x0D\x46\x0F\x8C\xCC\x49\x34\xB0\x69\x51\xBD\xF9\x06\xCD\x68\xAD\x66\x4C\xBC\x3E\xAC\x61\xBD\x0A\x88\x0E\xC8\xDF\x3D\xEE\x7C\x04\x4C\x9D\x0A\x5E\x6B\x91\xD6\xEE\xC7\xED\x28\x8D\xAB\x4D\x87\x89\x73\xD0\x6E\xA4\xD0\x1E\x16\x8B\x14\xE1\x76\x44\x03\x7F\x63\xAC\xE4\xCD\x49\x9C\xC5\x92\xF4\xAB\x32\xA1\x48\x5B\x02\x03\x01\x00\x01\xA3\x81\xB9\x30\x81\xB6\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\xC6\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xA1\x72\x5F\x26\x1B\x28\x98\x43\x95\x5D\x07\x37\xD5\x85\x96\x9D\x4B\xD2\xC3\x45\x30\x44\x06\x03\x55\x1D\x1F\x04\x3D\x30\x3B\x30\x39\xA0\x37\xA0\x35\x86\x33\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x75\x73\x65\x72\x74\x72\x75\x73\x74\x2E\x63\x6F\x6D\x2F\x55\x54\x4E\x2D\x55\x53\x45\x52\x46\x69\x72\x73\x74\x2D\x48\x61\x72\x64\x77\x61\x72\x65\x2E\x63\x72\x6C\x30\x31\x06\x03\x55\x1D\x25\x04\x2A\x30\x28\x06\x08\x2B\x06\x01\x05\x05\x07\x03\x01\x06\x08\x2B\x06\x01\x05\x05\x07\x03\x05\x06\x08\x2B\x06\x01\x05\x05\x07\x03\x06\x06\x08\x2B\x06\x01\x05\x05\x07\x03\x07\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x47\x19\x0F\xDE\x74\xC6\x99\x97\xAF\xFC\xAD\x28\x5E\x75\x8E\xEB\x2D\x67\xEE\x4E\x7B\x2B\xD7\x0C\xFF\xF6\xDE\xCB\x55\xA2\x0A\xE1\x4C\x54\x65\x93\x60\x6B\x9F\x12\x9C\xAD\x5E\x83\x2C\xEB\x5A\xAE\xC0\xE4\x2D\xF4\x00\x63\x1D\xB8\xC0\x6C\xF2\xCF\x49\xBB\x4D\x93\x6F\x06\xA6\x0A\x22\xB2\x49\x62\x08\x4E\xFF\xC8\xC8\x14\xB2\x88\x16\x5D\xE7\x01\xE4\x12\x95\xE5\x45\x34\xB3\x8B\x69\xBD\xCF\xB4\x85\x8F\x75\x51\x9E\x7D\x3A\x38\x3A\x14\x48\x12\xC6\xFB\xA7\x3B\x1A\x8D\x0D\x82\x40\x07\xE8\x04\x08\x90\xA1\x89\xCB\x19\x50\xDF\xCA\x1C\x01\xBC\x1D\x04\x19\x7B\x10\x76\x97\x3B\xEE\x90\x90\xCA\xC4\x0E\x1F\x16\x6E\x75\xEF\x33\xF8\xD3\x6F\x5B\x1E\x96\xE3\xE0\x74\x77\x74\x7B\x8A\xA2\x6E\x2D\xDD\x76\xD6\x39\x30\x82\xF0\xAB\x9C\x52\xF2\x2A\xC7\xAF\x49\x5E\x7E\xC7\x68\xE5\x82\x81\xC8\x6A\x27\xF9\x27\x88\x2A\xD5\x58\x50\x95\x1F\xF0\x3B\x1C\x57\xBB\x7D\x14\x39\x62\x2B\x9A\xC9\x94\x92\x2A\xA3\x22\x0C\xFF\x89\x26\x7D\x5F\x23\x2B\x47\xD7\x15\x1D\xA9\x6A\x9E\x51\x0D\x2A\x51\x9E\x81\xF9\xD4\x3B\x5E\x70\x12\x7F\x10\x32\x9C\x1E\xBB\x9D\xF8\x66\xA8",
 	["CN=Chambers of Commerce Root,OU=http://www.chambersign.org,O=AC Camerfirma SA CIF A82743287,C=EU"] = "\x30\x82\x04\xBD\x30\x82\x03\xA5\xA0\x03\x02\x01\x02\x02\x01\x00\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x7F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x55\x31\x27\x30\x25\x06\x03\x55\x04\x0A\x13\x1E\x41\x43\x20\x43\x61\x6D\x65\x72\x66\x69\x72\x6D\x61\x20\x53\x41\x20\x43\x49\x46\x20\x41\x38\x32\x37\x34\x33\x32\x38\x37\x31\x23\x30\x21\x06\x03\x55\x04\x0B\x13\x1A\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x68\x61\x6D\x62\x65\x72\x73\x69\x67\x6E\x2E\x6F\x72\x67\x31\x22\x30\x20\x06\x03\x55\x04\x03\x13\x19\x43\x68\x61\x6D\x62\x65\x72\x73\x20\x6F\x66\x20\x43\x6F\x6D\x6D\x65\x72\x63\x65\x20\x52\x6F\x6F\x74\x30\x1E\x17\x0D\x30\x33\x30\x39\x33\x30\x31\x36\x31\x33\x34\x33\x5A\x17\x0D\x33\x37\x30\x39\x33\x30\x31\x36\x31\x33\x34\x34\x5A\x30\x7F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x55\x31\x27\x30\x25\x06\x03\x55\x04\x0A\x13\x1E\x41\x43\x20\x43\x61\x6D\x65\x72\x66\x69\x72\x6D\x61\x20\x53\x41\x20\x43\x49\x46\x20\x41\x38\x32\x37\x34\x33\x32\x38\x37\x31\x23\x30\x21\x06\x03\x55\x04\x0B\x13\x1A\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x68\x61\x6D\x62\x65\x72\x73\x69\x67\x6E\x2E\x6F\x72\x67\x31\x22\x30\x20\x06\x03\x55\x04\x03\x13\x19\x43\x68\x61\x6D\x62\x65\x72\x73\x20\x6F\x66\x20\x43\x6F\x6D\x6D\x65\x72\x63\x65\x20\x52\x6F\x6F\x74\x30\x82\x01\x20\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0D\x00\x30\x82\x01\x08\x02\x82\x01\x01\x00\xB7\x36\x55\xE5\xA5\x5D\x18\x30\xE0\xDA\x89\x54\x91\xFC\xC8\xC7\x52\xF8\x2F\x50\xD9\xEF\xB1\x75\x73\x65\x47\x7D\x1B\x5B\xBA\x75\xC5\xFC\xA1\x88\x24\xFA\x2F\xED\xCA\x08\x4A\x39\x54\xC4\x51\x7A\xB5\xDA\x60\xEA\x38\x3C\x81\xB2\xCB\xF1\xBB\xD9\x91\x23\x3F\x48\x01\x70\x75\xA9\x05\x2A\xAD\x1F\x71\xF3\xC9\x54\x3D\x1D\x06\x6A\x40\x3E\xB3\x0C\x85\xEE\x5C\x1B\x79\xC2\x62\xC4\xB8\x36\x8E\x35\x5D\x01\x0C\x23\x04\x47\x35\xAA\x9B\x60\x4E\xA0\x66\x3D\xCB\x26\x0A\x9C\x40\xA1\xF4\x5D\x98\xBF\x71\xAB\xA5\x00\x68\x2A\xED\x83\x7A\x0F\xA2\x14\xB5\xD4\x22\xB3\x80\xB0\x3C\x0C\x5A\x51\x69\x2D\x58\x18\x8F\xED\x99\x9E\xF1\xAE\xE2\x95\xE6\xF6\x47\xA8\xD6\x0C\x0F\xB0\x58\x58\xDB\xC3\x66\x37\x9E\x9B\x91\x54\x33\x37\xD2\x94\x1C\x6A\x48\xC9\xC9\xF2\xA5\xDA\xA5\x0C\x23\xF7\x23\x0E\x9C\x32\x55\x5E\x71\x9C\x84\x05\x51\x9A\x2D\xFD\xE6\x4E\x2A\x34\x5A\xDE\xCA\x40\x37\x67\x0C\x54\x21\x55\x77\xDA\x0A\x0C\xCC\x97\xAE\x80\xDC\x94\x36\x4A\xF4\x3E\xCE\x36\x13\x1E\x53\xE4\xAC\x4E\x3A\x05\xEC\xDB\xAE\x72\x9C\x38\x8B\xD0\x39\x3B\x89\x0A\x3E\x77\xFE\x75\x02\x01\x03\xA3\x82\x01\x44\x30\x82\x01\x40\x30\x12\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x08\x30\x06\x01\x01\xFF\x02\x01\x0C\x30\x3C\x06\x03\x55\x1D\x1F\x04\x35\x30\x33\x30\x31\xA0\x2F\xA0\x2D\x86\x2B\x68\x74\x74\x70\x3A\x2F\x2F\x63\x72\x6C\x2E\x63\x68\x61\x6D\x62\x65\x72\x73\x69\x67\x6E\x2E\x6F\x72\x67\x2F\x63\x68\x61\x6D\x62\x65\x72\x73\x72\x6F\x6F\x74\x2E\x63\x72\x6C\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xE3\x94\xF5\xB1\x4D\xE9\xDB\xA1\x29\x5B\x57\x8B\x4D\x76\x06\x76\xE1\xD1\xA2\x8A\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x11\x06\x09\x60\x86\x48\x01\x86\xF8\x42\x01\x01\x04\x04\x03\x02\x00\x07\x30\x27\x06\x03\x55\x1D\x11\x04\x20\x30\x1E\x81\x1C\x63\x68\x61\x6D\x62\x65\x72\x73\x72\x6F\x6F\x74\x40\x63\x68\x61\x6D\x62\x65\x72\x73\x69\x67\x6E\x2E\x6F\x72\x67\x30\x27\x06\x03\x55\x1D\x12\x04\x20\x30\x1E\x81\x1C\x63\x68\x61\x6D\x62\x65\x72\x73\x72\x6F\x6F\x74\x40\x63\x68\x61\x6D\x62\x65\x72\x73\x69\x67\x6E\x2E\x6F\x72\x67\x30\x58\x06\x03\x55\x1D\x20\x04\x51\x30\x4F\x30\x4D\x06\x0B\x2B\x06\x01\x04\x01\x81\x87\x2E\x0A\x03\x01\x30\x3E\x30\x3C\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x30\x68\x74\x74\x70\x3A\x2F\x2F\x63\x70\x73\x2E\x63\x68\x61\x6D\x62\x65\x72\x73\x69\x67\x6E\x2E\x6F\x72\x67\x2F\x63\x70\x73\x2F\x63\x68\x61\x6D\x62\x65\x72\x73\x72\x6F\x6F\x74\x2E\x68\x74\x6D\x6C\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x0C\x41\x97\xC2\x1A\x86\xC0\x22\x7C\x9F\xFB\x90\xF3\x1A\xD1\x03\xB1\xEF\x13\xF9\x21\x5F\x04\x9C\xDA\xC9\xA5\x8D\x27\x6C\x96\x87\x91\xBE\x41\x90\x01\x72\x93\xE7\x1E\x7D\x5F\xF6\x89\xC6\x5D\xA7\x40\x09\x3D\xAC\x49\x45\x45\xDC\x2E\x8D\x30\x68\xB2\x09\xBA\xFB\xC3\x2F\xCC\xBA\x0B\xDF\x3F\x77\x7B\x46\x7D\x3A\x12\x24\x8E\x96\x8F\x3C\x05\x0A\x6F\xD2\x94\x28\x1D\x6D\x0C\xC0\x2E\x88\x22\xD5\xD8\xCF\x1D\x13\xC7\xF0\x48\xD7\xD7\x05\xA7\xCF\xC7\x47\x9E\x3B\x3C\x34\xC8\x80\x4F\xD4\x14\xBB\xFC\x0D\x50\xF7\xFA\xB3\xEC\x42\x5F\xA9\xDD\x6D\xC8\xF4\x75\xCF\x7B\xC1\x72\x26\xB1\x01\x1C\x5C\x2C\xFD\x7A\x4E\xB4\x01\xC5\x05\x57\xB9\xE7\x3C\xAA\x05\xD9\x88\xE9\x07\x46\x41\xCE\xEF\x41\x81\xAE\x58\xDF\x83\xA2\xAE\xCA\xD7\x77\x1F\xE7\x00\x3C\x9D\x6F\x8E\xE4\x32\x09\x1D\x4D\x78\x34\x78\x34\x3C\x94\x9B\x26\xED\x4F\x71\xC6\x19\x7A\xBD\x20\x22\x48\x5A\xFE\x4B\x7D\x03\xB7\xE7\x58\xBE\xC6\x32\x4E\x74\x1E\x68\xDD\xA8\x68\x5B\xB3\x3E\xEE\x62\x7D\xD9\x80\xE8\x0A\x75\x7A\xB7\xEE\xB4\x65\x9A\x21\x90\xE0\xAA\xD0\x98\xBC\x38\xB5\x73\x3C\x8B\xF8\xDC",
@@ -139,4 +135,12 @@ redef root_certs += {
 	["CN=Root CA Generalitat Valenciana,OU=PKIGVA,O=Generalitat Valenciana,C=ES"] = "\x30\x82\x06\x8B\x30\x82\x05\x73\xA0\x03\x02\x01\x02\x02\x04\x3B\x45\xE5\x68\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x68\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x53\x31\x1F\x30\x1D\x06\x03\x55\x04\x0A\x13\x16\x47\x65\x6E\x65\x72\x61\x6C\x69\x74\x61\x74\x20\x56\x61\x6C\x65\x6E\x63\x69\x61\x6E\x61\x31\x0F\x30\x0D\x06\x03\x55\x04\x0B\x13\x06\x50\x4B\x49\x47\x56\x41\x31\x27\x30\x25\x06\x03\x55\x04\x03\x13\x1E\x52\x6F\x6F\x74\x20\x43\x41\x20\x47\x65\x6E\x65\x72\x61\x6C\x69\x74\x61\x74\x20\x56\x61\x6C\x65\x6E\x63\x69\x61\x6E\x61\x30\x1E\x17\x0D\x30\x31\x30\x37\x30\x36\x31\x36\x32\x32\x34\x37\x5A\x17\x0D\x32\x31\x30\x37\x30\x31\x31\x35\x32\x32\x34\x37\x5A\x30\x68\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x53\x31\x1F\x30\x1D\x06\x03\x55\x04\x0A\x13\x16\x47\x65\x6E\x65\x72\x61\x6C\x69\x74\x61\x74\x20\x56\x61\x6C\x65\x6E\x63\x69\x61\x6E\x61\x31\x0F\x30\x0D\x06\x03\x55\x04\x0B\x13\x06\x50\x4B\x49\x47\x56\x41\x31\x27\x30\x25\x06\x03\x55\x04\x03\x13\x1E\x52\x6F\x6F\x74\x20\x43\x41\x20\x47\x65\x6E\x65\x72\x61\x6C\x69\x74\x61\x74\x20\x56\x61\x6C\x65\x6E\x63\x69\x61\x6E\x61\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xC6\x2A\xAB\x57\x11\x37\x2F\x22\x8A\xCA\x03\x74\x1D\xCA\xED\x2D\xA2\x0B\xBC\x33\x52\x40\x26\x47\xBE\x5A\x69\xA6\x3B\x72\x36\x17\x4C\xE8\xDF\xB8\xBB\x2F\x76\xE1\x40\x46\x74\x65\x02\x90\x52\x08\xB4\xFF\xA8\x8C\xC1\xE0\xC7\x89\x56\x10\x39\x33\xEF\x68\xB4\x5F\x5F\xDA\x6D\x23\xA1\x89\x5E\x22\xA3\x4A\x06\xF0\x27\xF0\x57\xB9\xF8\xE9\x4E\x32\x77\x0A\x3F\x41\x64\xF3\xEB\x65\xEE\x76\xFE\x54\xAA\x7D\x1D\x20\xAE\xF3\xD7\x74\xC2\x0A\x5F\xF5\x08\x28\x52\x08\xCC\x55\x5D\xD2\x0F\xDB\x9A\x81\xA5\xBB\xA1\xB3\xC1\x94\xCD\x54\xE0\x32\x75\x31\x91\x1A\x62\xB2\xDE\x75\xE2\xCF\x4F\x89\xD9\x91\x90\x0F\x41\x1B\xB4\x5A\x4A\x77\xBD\x67\x83\xE0\x93\xE7\x5E\xA7\x0C\xE7\x81\xD3\xF4\x52\xAC\x53\xB2\x03\xC7\x44\x26\xFB\x79\xE5\xCB\x34\x60\x50\x10\x7B\x1B\xDB\x6B\xD7\x47\xAB\x5F\x7C\x68\xCA\x6E\x9D\x41\x03\x10\xEE\x6B\x99\x7B\x5E\x25\xA8\xC2\xAB\xE4\xC0\xF3\x5C\x9C\xE3\xBE\xCE\x31\x4C\x64\x1E\x5E\x80\xA2\xF5\x83\x7E\x0C\xD6\xCA\x8C\x55\x8E\xBE\xE0\xBE\x49\x07\x0F\xA3\x24\x41\x7A\x58\x1D\x84\xEA\x58\x12\xC8\xE1\xB7\xED\xEF\x93\xDE\x94\x08\x31\x02\x03\x01\x00\x01\xA3\x82\x03\x3B\x30\x82\x03\x37\x30\x32\x06\x08\x2B\x06\x01\x05\x05\x07\x01\x01\x04\x26\x30\x24\x30\x22\x06\x08\x2B\x06\x01\x05\x05\x07\x30\x01\x86\x16\x68\x74\x74\x70\x3A\x2F\x2F\x6F\x63\x73\x70\x2E\x70\x6B\x69\x2E\x67\x76\x61\x2E\x65\x73\x30\x12\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x08\x30\x06\x01\x01\xFF\x02\x01\x02\x30\x82\x02\x34\x06\x03\x55\x1D\x20\x04\x82\x02\x2B\x30\x82\x02\x27\x30\x82\x02\x23\x06\x0A\x2B\x06\x01\x04\x01\xBF\x55\x02\x01\x00\x30\x82\x02\x13\x30\x82\x01\xE8\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x02\x30\x82\x01\xDA\x1E\x82\x01\xD6\x00\x41\x00\x75\x00\x74\x00\x6F\x00\x72\x00\x69\x00\x64\x00\x61\x00\x64\x00\x20\x00\x64\x00\x65\x00\x20\x00\x43\x00\x65\x00\x72\x00\x74\x00\x69\x00\x66\x00\x69\x00\x63\x00\x61\x00\x63\x00\x69\x00\xF3\x00\x6E\x00\x20\x00\x52\x00\x61\x00\xED\x00\x7A\x00\x20\x00\x64\x00\x65\x00\x20\x00\x6C\x00\x61\x00\x20\x00\x47\x00\x65\x00\x6E\x00\x65\x00\x72\x00\x61\x00\x6C\x00\x69\x00\x74\x00\x61\x00\x74\x00\x20\x00\x56\x00\x61\x00\x6C\x00\x65\x00\x6E\x00\x63\x00\x69\x00\x61\x00\x6E\x00\x61\x00\x2E\x00\x0D\x00\x0A\x00\x4C\x00\x61\x00\x20\x00\x44\x00\x65\x00\x63\x00\x6C\x00\x61\x00\x72\x00\x61\x00\x63\x00\x69\x00\xF3\x00\x6E\x00\x20\x00\x64\x00\x65\x00\x20\x00\x50\x00\x72\x00\xE1\x00\x63\x00\x74\x00\x69\x00\x63\x00\x61\x00\x73\x00\x20\x00\x64\x00\x65\x00\x20\x00\x43\x00\x65\x00\x72\x00\x74\x00\x69\x00\x66\x00\x69\x00\x63\x00\x61\x00\x63\x00\x69\x00\xF3\x00\x6E\x00\x20\x00\x71\x00\x75\x00\x65\x00\x20\x00\x72\x00\x69\x00\x67\x00\x65\x00\x20\x00\x65\x00\x6C\x00\x20\x00\x66\x00\x75\x00\x6E\x00\x63\x00\x69\x00\x6F\x00\x6E\x00\x61\x00\x6D\x00\x69\x00\x65\x00\x6E\x00\x74\x00\x6F\x00\x20\x00\x64\x00\x65\x00\x20\x00\x6C\x00\x61\x00\x20\x00\x70\x00\x72\x00\x65\x00\x73\x00\x65\x00\x6E\x00\x74\x00\x65\x00\x20\x00\x41\x00\x75\x00\x74\x00\x6F\x00\x72\x00\x69\x00\x64\x00\x61\x00\x64\x00\x20\x00\x64\x00\x65\x00\x20\x00\x43\x00\x65\x00\x72\x00\x74\x00\x69\x00\x66\x00\x69\x00\x63\x00\x61\x00\x63\x00\x69\x00\xF3\x00\x6E\x00\x20\x00\x73\x00\x65\x00\x20\x00\x65\x00\x6E\x00\x63\x00\x75\x00\x65\x00\x6E\x00\x74\x00\x72\x00\x61\x00\x20\x00\x65\x00\x6E\x00\x20\x00\x6C\x00\x61\x00\x20\x00\x64\x00\x69\x00\x72\x00\x65\x00\x63\x00\x63\x00\x69\x00\xF3\x00\x6E\x00\x20\x00\x77\x00\x65\x00\x62\x00\x20\x00\x68\x00\x74\x00\x74\x00\x70\x00\x3A\x00\x2F\x00\x2F\x00\x77\x00\x77\x00\x77\x00\x2E\x00\x70\x00\x6B\x00\x69\x00\x2E\x00\x67\x00\x76\x00\x61\x00\x2E\x00\x65\x00\x73\x00\x2F\x00\x63\x00\x70\x00\x73\x30\x25\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x19\x68\x74\x74\x70\x3A\x2F\x2F\x77\x77\x77\x2E\x70\x6B\x69\x2E\x67\x76\x61\x2E\x65\x73\x2F\x63\x70\x73\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x7B\x35\xD3\x40\xD2\x1C\x78\x19\x66\xEF\x74\x10\x28\xDC\x3E\x4F\xB2\x78\x04\xFC\x30\x81\x95\x06\x03\x55\x1D\x23\x04\x81\x8D\x30\x81\x8A\x80\x14\x7B\x35\xD3\x40\xD2\x1C\x78\x19\x66\xEF\x74\x10\x28\xDC\x3E\x4F\xB2\x78\x04\xFC\xA1\x6C\xA4\x6A\x30\x68\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x53\x31\x1F\x30\x1D\x06\x03\x55\x04\x0A\x13\x16\x47\x65\x6E\x65\x72\x61\x6C\x69\x74\x61\x74\x20\x56\x61\x6C\x65\x6E\x63\x69\x61\x6E\x61\x31\x0F\x30\x0D\x06\x03\x55\x04\x0B\x13\x06\x50\x4B\x49\x47\x56\x41\x31\x27\x30\x25\x06\x03\x55\x04\x03\x13\x1E\x52\x6F\x6F\x74\x20\x43\x41\x20\x47\x65\x6E\x65\x72\x61\x6C\x69\x74\x61\x74\x20\x56\x61\x6C\x65\x6E\x63\x69\x61\x6E\x61\x82\x04\x3B\x45\xE5\x68\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x24\x61\x4E\xF5\xB5\xC8\x42\x02\x2A\xB3\x5C\x75\xAD\xC5\x6D\xCA\xE7\x94\x3F\xA5\x68\x95\x88\xC1\x54\xC0\x10\x69\xA2\x12\x2F\x18\x3F\x25\x50\xA8\x7C\x4A\xEA\xC6\x09\xD9\xF4\x75\xC6\x40\xDA\xAF\x50\x9D\x3D\xA5\x16\xBB\x6D\x31\xC6\xC7\x73\x0A\x48\xFE\x20\x72\xED\x6F\xCC\xE8\x83\x61\x16\x46\x90\x01\x95\x4B\x7D\x8E\x9A\x52\x09\x2F\xF6\x6F\x1C\xE4\xA1\x71\xCF\x8C\x2A\x5A\x17\x73\x83\x47\x4D\x0F\x36\xFB\x04\x4D\x49\x51\xE2\x14\xC9\x64\x61\xFB\xD4\x14\xE0\xF4\x9E\xB7\x34\x8F\x0A\x26\xBD\x97\x5C\xF4\x79\x3A\x4A\x30\x19\xCC\xAD\x4F\xA0\x98\x8A\xB4\x31\x97\x2A\xE2\x73\x6D\x7E\x78\xB8\xF8\x88\x89\x4F\xB1\x22\x91\x64\x4B\xF5\x50\xDE\x03\xDB\xE5\xC5\x76\xE7\x13\x66\x75\x7E\x65\xFB\x01\x9F\x93\x87\x88\x9D\xF9\x46\x57\x7C\x4D\x60\xAF\x98\x73\x13\x23\xA4\x20\x91\x81\xFA\xD0\x61\x66\xB8\x7D\xD1\xAF\xD6\x6F\x1E\x6C\x3D\xE9\x11\xFD\xA9\xF9\x82\x22\x86\x99\x33\x71\x5A\xEA\x19\x57\x3D\x91\xCD\xA9\xC0\xA3\x6E\x07\x13\xA6\xC9\xED\xF8\x68\xA3\x9E\xC3\x5A\x72\x09\x87\x28\xD1\xC4\x73\xC4\x73\x18\x5F\x50\x75\x16\x31\x9F\xB7\xE8\x7C\xC3",
 	["CN=A-Trust-nQual-03,OU=A-Trust-nQual-03,O=A-Trust Ges. f. Sicherheitssysteme im elektr. Datenverkehr GmbH,C=AT"] = "\x30\x82\x03\xCF\x30\x82\x02\xB7\xA0\x03\x02\x01\x02\x02\x03\x01\x6C\x1E\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\x8D\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x41\x54\x31\x48\x30\x46\x06\x03\x55\x04\x0A\x0C\x3F\x41\x2D\x54\x72\x75\x73\x74\x20\x47\x65\x73\x2E\x20\x66\x2E\x20\x53\x69\x63\x68\x65\x72\x68\x65\x69\x74\x73\x73\x79\x73\x74\x65\x6D\x65\x20\x69\x6D\x20\x65\x6C\x65\x6B\x74\x72\x2E\x20\x44\x61\x74\x65\x6E\x76\x65\x72\x6B\x65\x68\x72\x20\x47\x6D\x62\x48\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x0C\x10\x41\x2D\x54\x72\x75\x73\x74\x2D\x6E\x51\x75\x61\x6C\x2D\x30\x33\x31\x19\x30\x17\x06\x03\x55\x04\x03\x0C\x10\x41\x2D\x54\x72\x75\x73\x74\x2D\x6E\x51\x75\x61\x6C\x2D\x30\x33\x30\x1E\x17\x0D\x30\x35\x30\x38\x31\x37\x32\x32\x30\x30\x30\x30\x5A\x17\x0D\x31\x35\x30\x38\x31\x37\x32\x32\x30\x30\x30\x30\x5A\x30\x81\x8D\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x41\x54\x31\x48\x30\x46\x06\x03\x55\x04\x0A\x0C\x3F\x41\x2D\x54\x72\x75\x73\x74\x20\x47\x65\x73\x2E\x20\x66\x2E\x20\x53\x69\x63\x68\x65\x72\x68\x65\x69\x74\x73\x73\x79\x73\x74\x65\x6D\x65\x20\x69\x6D\x20\x65\x6C\x65\x6B\x74\x72\x2E\x20\x44\x61\x74\x65\x6E\x76\x65\x72\x6B\x65\x68\x72\x20\x47\x6D\x62\x48\x31\x19\x30\x17\x06\x03\x55\x04\x0B\x0C\x10\x41\x2D\x54\x72\x75\x73\x74\x2D\x6E\x51\x75\x61\x6C\x2D\x30\x33\x31\x19\x30\x17\x06\x03\x55\x04\x03\x0C\x10\x41\x2D\x54\x72\x75\x73\x74\x2D\x6E\x51\x75\x61\x6C\x2D\x30\x33\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xAD\x3D\x61\x6E\x03\xF3\x90\x3B\xC0\x41\x0B\x84\x80\xCD\xEC\x2A\xA3\x9D\x6B\xBB\x6E\xC2\x42\x84\xF7\x51\x14\xE1\xA0\xA8\x2D\x51\xA3\x51\xF2\xDE\x23\xF0\x34\x44\xFF\x94\xEB\xCC\x05\x23\x95\x40\xB9\x07\x78\xA5\x25\xF6\x0A\xBD\x45\x86\xE8\xD9\xBD\xC0\x04\x8E\x85\x44\x61\xEF\x7F\xA7\xC9\xFA\xC1\x25\xCC\x85\x2C\x63\x3F\x05\x60\x73\x49\x05\xE0\x60\x78\x95\x10\x4B\xDC\xF9\x11\x59\xCE\x71\x7F\x40\x9B\x8A\xAA\x24\xDF\x0B\x42\xE2\xDB\x56\xBC\x4A\xD2\xA5\x0C\x9B\xB7\x43\x3E\xDD\x83\xD3\x26\x10\x02\xCF\xEA\x23\xC4\x49\x4E\xE5\xD3\xE9\xB4\x88\xAB\x0C\xAE\x62\x92\xD4\x65\x87\xD9\x6A\xD7\xF4\x85\x9F\xE4\x33\x22\x25\xA5\xE5\xC8\x33\xBA\xC3\xC7\x41\xDC\x5F\xC6\x6A\xCC\x00\x0E\x6D\x32\xA8\xB6\x87\x36\x00\x62\x77\x9B\x1E\x1F\x34\xCB\x90\x3C\x78\x88\x74\x05\xEB\x79\xF5\x93\x71\x65\xCA\x9D\xC7\x6B\x18\x2D\x3D\x5C\x4E\xE7\xD5\xF8\x3F\x31\x7D\x8F\x87\xEC\x0A\x22\x2F\x23\xE9\xFE\xBB\x7D\xC9\xE0\xF4\xEC\xEB\x7C\xC4\xB0\xC3\x2D\x62\xB5\x9A\x71\xD6\xB1\x6A\xE8\xEC\xD9\xED\xD5\x72\xEC\xBE\x57\x01\xCE\x05\x55\x9F\xDE\xD1\x60\x88\x10\xB3\x02\x03\x01\x00\x01\xA3\x36\x30\x34\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x11\x06\x03\x55\x1D\x0E\x04\x0A\x04\x08\x44\x6A\x95\x67\x55\x79\x11\x4F\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x55\xD4\x54\xD1\x59\x48\x5C\xB3\x93\x85\xAA\xBF\x63\x2F\xE4\x80\xCE\x34\xA3\x34\x62\x3E\xF6\xD8\xEE\x67\x88\x31\x04\x03\x6F\x0B\xD4\x07\xFB\x4E\x75\x0F\xD3\x2E\xD3\xC0\x17\xC7\xC6\x28\xEC\x06\x0D\x11\x24\x0E\x0E\xA5\x5D\xBF\x8C\xB2\x13\x96\x71\xDC\xD4\xCE\x0E\x0D\x0A\x68\x32\x6C\xB9\x41\x31\x19\xAB\xB1\x07\x7B\x4D\x98\xD3\x5C\xB0\xD1\xF0\xA7\x42\xA0\xB5\xC4\x8E\xAF\xFE\xF1\x3F\xF4\xEF\x4F\x46\x00\x76\xEB\x02\xFB\xF9\x9D\xD2\x40\x96\xC7\x88\x3A\xB8\x9F\x11\x79\xF3\x80\x65\xA8\xBD\x1F\xD3\x78\x81\xA0\x51\x4C\x37\xB4\xA6\x5D\x25\x70\xD1\x66\xC9\x68\xF9\x2E\x11\x14\x68\xF1\x54\x98\x08\xAC\x26\x92\x0F\xDE\x89\x9E\xD4\xFA\xB3\x79\x2B\xD2\xA3\x79\xD4\xEC\x8B\xAC\x87\x53\x68\x42\x4C\x51\x51\x74\x1E\x1B\x27\x2E\xE3\xF5\x1F\x29\x74\x4D\xED\xAF\xF7\xE1\x92\x99\x81\xE8\xBE\x3A\xC7\x17\x50\xF6\xB7\xC6\xFC\x9B\xB0\x8A\x6B\xD6\x88\x03\x91\x8F\x06\x77\x3A\x85\x02\xDD\x98\xD5\x43\x78\x3F\xC6\x30\x15\xAC\x9B\x6B\xCB\x57\xB7\x89\x51\x8B\x3A\xE8\xC9\x84\x0C\xDB\xB1\x50\x20\x0A\x1A\x4A\xBA\x6A\x1A\xBD\xEC\x1B\xC8\xC5\x84\x9A\xCD",
 	["CN=TWCA Root Certification Authority,OU=Root CA,O=TAIWAN-CA,C=TW"] = "\x30\x82\x03\x7B\x30\x82\x02\x63\xA0\x03\x02\x01\x02\x02\x01\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x54\x57\x31\x12\x30\x10\x06\x03\x55\x04\x0A\x0C\x09\x54\x41\x49\x57\x41\x4E\x2D\x43\x41\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x0C\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x2A\x30\x28\x06\x03\x55\x04\x03\x0C\x21\x54\x57\x43\x41\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x1E\x17\x0D\x30\x38\x30\x38\x32\x38\x30\x37\x32\x34\x33\x33\x5A\x17\x0D\x33\x30\x31\x32\x33\x31\x31\x35\x35\x39\x35\x39\x5A\x30\x5F\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x54\x57\x31\x12\x30\x10\x06\x03\x55\x04\x0A\x0C\x09\x54\x41\x49\x57\x41\x4E\x2D\x43\x41\x31\x10\x30\x0E\x06\x03\x55\x04\x0B\x0C\x07\x52\x6F\x6F\x74\x20\x43\x41\x31\x2A\x30\x28\x06\x03\x55\x04\x03\x0C\x21\x54\x57\x43\x41\x20\x52\x6F\x6F\x74\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xB0\x7E\x72\xB8\xA4\x03\x94\xE6\xA7\xDE\x09\x38\x91\x4A\x11\x40\x87\xA7\x7C\x59\x64\x14\x7B\xB5\x11\x10\xDD\xFE\xBF\xD5\xC0\xBB\x56\xE2\x85\x25\xF4\x35\x72\x0F\xF8\x53\xD0\x41\xE1\x44\x01\xC2\xB4\x1C\xC3\x31\x42\x16\x47\x85\x33\x22\x76\xB2\x0A\x6F\x0F\xE5\x25\x50\x4F\x85\x86\xBE\xBF\x98\x2E\x10\x67\x1E\xBE\x11\x05\x86\x05\x90\xC4\x59\xD0\x7C\x78\x10\xB0\x80\x5C\xB7\xE1\xC7\x2B\x75\xCB\x7C\x9F\xAE\xB5\xD1\x9D\x23\x37\x63\xA7\xDC\x42\xA2\x2D\x92\x04\x1B\x50\xC1\x7B\xB8\x3E\x1B\xC9\x56\x04\x8B\x2F\x52\x9B\xAD\xA9\x56\xE9\xC1\xFF\xAD\xA9\x58\x87\x30\xB6\x81\xF7\x97\x45\xFC\x19\x57\x3B\x2B\x6F\xE4\x47\xF4\x99\x45\xFE\x1D\xF1\xF8\x97\xA3\x88\x1D\x37\x1C\x5C\x8F\xE0\x76\x25\x9A\x50\xF8\xA0\x54\xFF\x44\x90\x76\x23\xD2\x32\xC6\xC3\xAB\x06\xBF\xFC\xFB\xBF\xF3\xAD\x7D\x92\x62\x02\x5B\x29\xD3\x35\xA3\x93\x9A\x43\x64\x60\x5D\xB2\xFA\x32\xFF\x3B\x04\xAF\x4D\x40\x6A\xF9\xC7\xE3\xEF\x23\xFD\x6B\xCB\xE5\x0F\x8B\x38\x0D\xEE\x0A\xFC\xFE\x0F\x98\x9F\x30\x31\xDD\x6C\x52\x65\xF9\x8B\x81\xBE\x22\xE1\x1C\x58\x03\xBA\x91\x1B\x89\x07\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x6A\x38\x5B\x26\x8D\xDE\x8B\x5A\xF2\x4F\x7A\x54\x83\x19\x18\xE3\x08\x35\xA6\xBA\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x3C\xD5\x77\x3D\xDA\xDF\x89\xBA\x87\x0C\x08\x54\x6A\x20\x50\x92\xBE\xB0\x41\x3D\xB9\x26\x64\x83\x0A\x2F\xE8\x40\xC0\x97\x28\x27\x82\x30\x4A\xC9\x93\xFF\x6A\xE7\xA6\x00\x7F\x89\x42\x9A\xD6\x11\xE5\x53\xCE\x2F\xCC\xF2\xDA\x05\xC4\xFE\xE2\x50\xC4\x3A\x86\x7D\xCC\xDA\x7E\x10\x09\x3B\x92\x35\x2A\x53\xB2\xFE\xEB\x2B\x05\xD9\x6C\x5D\xE6\xD0\xEF\xD3\x6A\x66\x9E\x15\x28\x85\x7A\xE8\x82\x00\xAC\x1E\xA7\x09\x69\x56\x42\xD3\x68\x51\x18\xBE\x54\x9A\xBF\x44\x41\xBA\x49\xBE\x20\xBA\x69\x5C\xEE\xB8\x77\xCD\xCE\x6C\x1F\xAD\x83\x96\x18\x7D\x0E\xB5\x14\x39\x84\xF1\x28\xE9\x2D\xA3\x9E\x7B\x1E\x7A\x72\x5A\x83\xB3\x79\x6F\xEF\xB4\xFC\xD0\x0A\xA5\x58\x4F\x46\xDF\xFB\x6D\x79\x59\xF2\x84\x22\x52\xAE\x0F\xCC\xFB\x7C\x3B\xE7\x6A\xCA\x47\x61\xC3\x7A\xF8\xD3\x92\x04\x1F\xB8\x20\x84\xE1\x36\x54\x16\xC7\x40\xDE\x3B\x8A\x73\xDC\xDF\xC6\x09\x4C\xDF\xEC\xDA\xFF\xD4\x53\x42\xA1\xC9\xF2\x62\x1D\x22\x83\x3C\x97\xC5\xF9\x19\x62\x27\xAC\x65\x22\xD7\xD3\x3C\xC6\xE5\x8E\xB2\x53\xCC\x49\xCE\xBC\x30\xFE\x7B\x0E\x33\x90\xFB\xED\xD2\x14\x91\x1F\x07\xAF",
+	["OU=Security Communication RootCA2,O=SECOM Trust Systems CO.\,LTD.,C=JP"] = "\x30\x82\x03\x77\x30\x82\x02\x5F\xA0\x03\x02\x01\x02\x02\x01\x00\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x5D\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4A\x50\x31\x25\x30\x23\x06\x03\x55\x04\x0A\x13\x1C\x53\x45\x43\x4F\x4D\x20\x54\x72\x75\x73\x74\x20\x53\x79\x73\x74\x65\x6D\x73\x20\x43\x4F\x2E\x2C\x4C\x54\x44\x2E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x53\x65\x63\x75\x72\x69\x74\x79\x20\x43\x6F\x6D\x6D\x75\x6E\x69\x63\x61\x74\x69\x6F\x6E\x20\x52\x6F\x6F\x74\x43\x41\x32\x30\x1E\x17\x0D\x30\x39\x30\x35\x32\x39\x30\x35\x30\x30\x33\x39\x5A\x17\x0D\x32\x39\x30\x35\x32\x39\x30\x35\x30\x30\x33\x39\x5A\x30\x5D\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4A\x50\x31\x25\x30\x23\x06\x03\x55\x04\x0A\x13\x1C\x53\x45\x43\x4F\x4D\x20\x54\x72\x75\x73\x74\x20\x53\x79\x73\x74\x65\x6D\x73\x20\x43\x4F\x2E\x2C\x4C\x54\x44\x2E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x53\x65\x63\x75\x72\x69\x74\x79\x20\x43\x6F\x6D\x6D\x75\x6E\x69\x63\x61\x74\x69\x6F\x6E\x20\x52\x6F\x6F\x74\x43\x41\x32\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xD0\x15\x39\x52\xB1\x52\xB3\xBA\xC5\x59\x82\xC4\x5D\x52\xAE\x3A\x43\x65\x80\x4B\xC7\xF2\x96\xBC\xDB\x36\x97\xD6\xA6\x64\x8C\xA8\x5E\xF0\xE3\x0A\x1C\xF7\xDF\x97\x3D\x4B\xAE\xF6\x5D\xEC\x21\xB5\x41\xAB\xCD\xB9\x7E\x76\x9F\xBE\xF9\x3E\x36\x34\xA0\x3B\xC1\xF6\x31\x11\x45\x74\x93\x3D\x57\x80\xC5\xF9\x89\x99\xCA\xE5\xAB\x6A\xD4\xB5\xDA\x41\x90\x10\xC1\xD6\xD6\x42\x89\xC2\xBF\xF4\x38\x12\x95\x4C\x54\x05\xF7\x36\xE4\x45\x83\x7B\x14\x65\xD6\xDC\x0C\x4D\xD1\xDE\x7E\x0C\xAB\x3B\xC4\x15\xBE\x3A\x56\xA6\x5A\x6F\x76\x69\x52\xA9\x7A\xB9\xC8\xEB\x6A\x9A\x5D\x52\xD0\x2D\x0A\x6B\x35\x16\x09\x10\x84\xD0\x6A\xCA\x3A\x06\x00\x37\x47\xE4\x7E\x57\x4F\x3F\x8B\xEB\x67\xB8\x88\xAA\xC5\xBE\x53\x55\xB2\x91\xC4\x7D\xB9\xB0\x85\x19\x06\x78\x2E\xDB\x61\x1A\xFA\x85\xF5\x4A\x91\xA1\xE7\x16\xD5\x8E\xA2\x39\xDF\x94\xB8\x70\x1F\x28\x3F\x8B\xFC\x40\x5E\x63\x83\x3C\x83\x2A\x1A\x99\x6B\xCF\xDE\x59\x6A\x3B\xFC\x6F\x16\xD7\x1F\xFD\x4A\x10\xEB\x4E\x82\x16\x3A\xAC\x27\x0C\x53\xF1\xAD\xD5\x24\xB0\x6B\x03\x50\xC1\x2D\x3C\x16\xDD\x44\x34\x27\x1A\x75\xFB\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x0A\x85\xA9\x77\x65\x05\x98\x7C\x40\x81\xF8\x0F\x97\x2C\x38\xF1\x0A\xEC\x3C\xCF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x01\x01\x00\x4C\x3A\xA3\x44\xAC\xB9\x45\xB1\xC7\x93\x7E\xC8\x0B\x0A\x42\xDF\x64\xEA\x1C\xEE\x59\x6C\x08\xBA\x89\x5F\x6A\xCA\x4A\x95\x9E\x7A\x8F\x07\xC5\xDA\x45\x72\x82\x71\x0E\x3A\xD2\xCC\x6F\xA7\xB4\xA1\x23\xBB\xF6\x24\x9F\xCB\x17\xFE\x8C\xA6\xCE\xC2\xD2\xDB\xCC\x8D\xFC\x71\xFC\x03\x29\xC1\x6C\x5D\x33\x5F\x64\xB6\x65\x3B\x89\x6F\x18\x76\x78\xF5\xDC\xA2\x48\x1F\x19\x3F\x8E\x93\xEB\xF1\xFA\x17\xEE\xCD\x4E\xE3\x04\x12\x55\xD6\xE5\xE4\xDD\xFB\x3E\x05\x7C\xE2\x1D\x5E\xC6\xA7\xBC\x97\x4F\x68\x3A\xF5\xE9\x2E\x0A\x43\xB6\xAF\x57\x5C\x62\x68\x7C\xB7\xFD\xA3\x8A\x84\xA0\xAC\x62\xBE\x2B\x09\x87\x34\xF0\x6A\x01\xBB\x9B\x29\x56\x3C\xFE\x00\x37\xCF\x23\x6C\xF1\x4E\xAA\xB6\x74\x46\x12\x6C\x91\xEE\x34\xD5\xEC\x9A\x91\xE7\x44\xBE\x90\x31\x72\xD5\x49\x02\xF6\x02\xE5\xF4\x1F\xEB\x7C\xD9\x96\x55\xA9\xFF\xEC\x8A\xF9\x99\x47\xFF\x35\x5A\x02\xAA\x04\xCB\x8A\x5B\x87\x71\x29\x91\xBD\xA4\xB4\x7A\x0D\xBD\x9A\xF5\x57\x23\x00\x07\x21\x17\x3F\x4A\x39\xD1\x05\x49\x0B\xA7\xB6\x37\x81\xA5\x5D\x8C\xAA\x33\x5E\x81\x28\x7C\xA7\x7D\x27\xEB\x00\xAE\x8D\x37",
+	["CN=EC-ACC,OU=Jerarquia Entitats de Certificacio Catalanes,OU=Vegeu https://www.catcert.net/verarrel (c)03,OU=Serveis Publics de Certificacio,O=Agencia Catalana de Certificacio (NIF Q-0801176-I),C=ES"] = "\x30\x82\x05\x56\x30\x82\x04\x3E\xA0\x03\x02\x01\x02\x02\x10\xEE\x2B\x3D\xEB\xD4\x21\xDE\x14\xA8\x62\xAC\x04\xF3\xDD\xC4\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\xF3\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x53\x31\x3B\x30\x39\x06\x03\x55\x04\x0A\x13\x32\x41\x67\x65\x6E\x63\x69\x61\x20\x43\x61\x74\x61\x6C\x61\x6E\x61\x20\x64\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x63\x69\x6F\x20\x28\x4E\x49\x46\x20\x51\x2D\x30\x38\x30\x31\x31\x37\x36\x2D\x49\x29\x31\x28\x30\x26\x06\x03\x55\x04\x0B\x13\x1F\x53\x65\x72\x76\x65\x69\x73\x20\x50\x75\x62\x6C\x69\x63\x73\x20\x64\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x63\x69\x6F\x31\x35\x30\x33\x06\x03\x55\x04\x0B\x13\x2C\x56\x65\x67\x65\x75\x20\x68\x74\x74\x70\x73\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x61\x74\x63\x65\x72\x74\x2E\x6E\x65\x74\x2F\x76\x65\x72\x61\x72\x72\x65\x6C\x20\x28\x63\x29\x30\x33\x31\x35\x30\x33\x06\x03\x55\x04\x0B\x13\x2C\x4A\x65\x72\x61\x72\x71\x75\x69\x61\x20\x45\x6E\x74\x69\x74\x61\x74\x73\x20\x64\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x63\x69\x6F\x20\x43\x61\x74\x61\x6C\x61\x6E\x65\x73\x31\x0F\x30\x0D\x06\x03\x55\x04\x03\x13\x06\x45\x43\x2D\x41\x43\x43\x30\x1E\x17\x0D\x30\x33\x30\x31\x30\x37\x32\x33\x30\x30\x30\x30\x5A\x17\x0D\x33\x31\x30\x31\x30\x37\x32\x32\x35\x39\x35\x39\x5A\x30\x81\xF3\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x45\x53\x31\x3B\x30\x39\x06\x03\x55\x04\x0A\x13\x32\x41\x67\x65\x6E\x63\x69\x61\x20\x43\x61\x74\x61\x6C\x61\x6E\x61\x20\x64\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x63\x69\x6F\x20\x28\x4E\x49\x46\x20\x51\x2D\x30\x38\x30\x31\x31\x37\x36\x2D\x49\x29\x31\x28\x30\x26\x06\x03\x55\x04\x0B\x13\x1F\x53\x65\x72\x76\x65\x69\x73\x20\x50\x75\x62\x6C\x69\x63\x73\x20\x64\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x63\x69\x6F\x31\x35\x30\x33\x06\x03\x55\x04\x0B\x13\x2C\x56\x65\x67\x65\x75\x20\x68\x74\x74\x70\x73\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x61\x74\x63\x65\x72\x74\x2E\x6E\x65\x74\x2F\x76\x65\x72\x61\x72\x72\x65\x6C\x20\x28\x63\x29\x30\x33\x31\x35\x30\x33\x06\x03\x55\x04\x0B\x13\x2C\x4A\x65\x72\x61\x72\x71\x75\x69\x61\x20\x45\x6E\x74\x69\x74\x61\x74\x73\x20\x64\x65\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x63\x69\x6F\x20\x43\x61\x74\x61\x6C\x61\x6E\x65\x73\x31\x0F\x30\x0D\x06\x03\x55\x04\x03\x13\x06\x45\x43\x2D\x41\x43\x43\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xB3\x22\xC7\x4F\xE2\x97\x42\x95\x88\x47\x83\x40\xF6\x1D\x17\xF3\x83\x73\x24\x1E\x51\xF3\x98\x8A\xC3\x92\xB8\xFF\x40\x90\x05\x70\x87\x60\xC9\x00\xA9\xB5\x94\x65\x19\x22\x15\x17\xC2\x43\x6C\x66\x44\x9A\x0D\x04\x3E\x39\x6F\xA5\x4B\x7A\xAA\x63\xB7\x8A\x44\x9D\xD9\x63\x91\x84\x66\xE0\x28\x0F\xBA\x42\xE3\x6E\x8E\xF7\x14\x27\x93\x69\xEE\x91\x0E\xA3\x5F\x0E\xB1\xEB\x66\xA2\x72\x4F\x12\x13\x86\x65\x7A\x3E\xDB\x4F\x07\xF4\xA7\x09\x60\xDA\x3A\x42\x99\xC7\xB2\x7F\xB3\x16\x95\x1C\xC7\xF9\x34\xB5\x94\x85\xD5\x99\x5E\xA0\x48\xA0\x7E\xE7\x17\x65\xB8\xA2\x75\xB8\x1E\xF3\xE5\x42\x7D\xAF\xED\xF3\x8A\x48\x64\x5D\x82\x14\x93\xD8\xC0\xE4\xFF\xB3\x50\x72\xF2\x76\xF6\xB3\x5D\x42\x50\x79\xD0\x94\x3E\x6B\x0C\x00\xBE\xD8\x6B\x0E\x4E\x2A\xEC\x3E\xD2\xCC\x82\xA2\x18\x65\x33\x13\x77\x9E\x9A\x5D\x1A\x13\xD8\xC3\xDB\x3D\xC8\x97\x7A\xEE\x70\xED\xA7\xE6\x7C\xDB\x71\xCF\x2D\x94\x62\xDF\x6D\xD6\xF5\x38\xBE\x3F\xA5\x85\x0A\x19\xB8\xA8\xD8\x09\x75\x42\x70\xC4\xEA\xEF\xCB\x0E\xC8\x34\xA8\x12\x22\x98\x0C\xB8\x13\x94\xB6\x4B\xEC\xF0\xD0\x90\xE7\x27\x02\x03\x01\x00\x01\xA3\x81\xE3\x30\x81\xE0\x30\x1D\x06\x03\x55\x1D\x11\x04\x16\x30\x14\x81\x12\x65\x63\x5F\x61\x63\x63\x40\x63\x61\x74\x63\x65\x72\x74\x2E\x6E\x65\x74\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xA0\xC3\x8B\x44\xAA\x37\xA5\x45\xBF\x97\x80\x5A\xD1\xF1\x78\xA2\x9B\xE9\x5D\x8D\x30\x7F\x06\x03\x55\x1D\x20\x04\x78\x30\x76\x30\x74\x06\x0B\x2B\x06\x01\x04\x01\xF5\x78\x01\x03\x01\x0A\x30\x65\x30\x2C\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x01\x16\x20\x68\x74\x74\x70\x73\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x61\x74\x63\x65\x72\x74\x2E\x6E\x65\x74\x2F\x76\x65\x72\x61\x72\x72\x65\x6C\x30\x35\x06\x08\x2B\x06\x01\x05\x05\x07\x02\x02\x30\x29\x1A\x27\x56\x65\x67\x65\x75\x20\x68\x74\x74\x70\x73\x3A\x2F\x2F\x77\x77\x77\x2E\x63\x61\x74\x63\x65\x72\x74\x2E\x6E\x65\x74\x2F\x76\x65\x72\x61\x72\x72\x65\x6C\x20\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\xA0\x48\x5B\x82\x01\xF6\x4D\x48\xB8\x39\x55\x35\x9C\x80\x7A\x53\x99\xD5\x5A\xFF\xB1\x71\x3B\xCC\x39\x09\x94\x5E\xD6\xDA\xEF\xBE\x01\x5B\x5D\xD3\x1E\xD8\xFD\x7D\x4F\xCD\xA0\x41\xE0\x34\x93\xBF\xCB\xE2\x86\x9C\x37\x92\x90\x56\x1C\xDC\xEB\x29\x05\xE5\xC4\x9E\xC7\x35\xDF\x8A\x0C\xCD\xC5\x21\x43\xE9\xAA\x88\xE5\x35\xC0\x19\x42\x63\x5A\x02\x5E\xA4\x48\x18\x3A\x85\x6F\xDC\x9D\xBC\x3F\x9D\x9C\xC1\x87\xB8\x7A\x61\x08\xE9\x77\x0B\x7F\x70\xAB\x7A\xDD\xD9\x97\x2C\x64\x1E\x85\xBF\xBC\x74\x96\xA1\xC3\x7A\x12\xEC\x0C\x1A\x6E\x83\x0C\x3C\xE8\x72\x46\x9F\xFB\x48\xD5\x5E\x97\xE6\xB1\xA1\xF8\xE4\xEF\x46\x25\x94\x9C\x89\xDB\x69\x38\xBE\xEC\x5C\x0E\x56\xC7\x65\x51\xE5\x50\x88\x88\xBF\x42\xD5\x2B\x3D\xE5\xF9\xBA\x9E\x2E\xB3\xCA\xF4\x73\x92\x02\x0B\xBE\x4C\x66\xEB\x20\xFE\xB9\xCB\xB5\x99\x7F\xE6\xB6\x13\xFA\xCA\x4B\x4D\xD9\xEE\x53\x46\x06\x3B\xC6\x4E\xAD\x93\x5A\x81\x7E\x6C\x2A\x4B\x6A\x05\x45\x8C\xF2\x21\xA4\x31\x90\x87\x6C\x65\x9C\x9D\xA5\x60\x95\x3A\x52\x7F\xF5\xD1\xAB\x08\x6E\xF3\xEE\x5B\xF9\x88\x3D\x7E\xB8\x6F\x6E\x03\xE4\x42",
+	["CN=Hellenic Academic and Research Institutions RootCA 2011,O=Hellenic Academic and Research Institutions Cert. Authority,C=GR"] = "\x30\x82\x04\x31\x30\x82\x03\x19\xA0\x03\x02\x01\x02\x02\x01\x00\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x81\x95\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x52\x31\x44\x30\x42\x06\x03\x55\x04\x0A\x13\x3B\x48\x65\x6C\x6C\x65\x6E\x69\x63\x20\x41\x63\x61\x64\x65\x6D\x69\x63\x20\x61\x6E\x64\x20\x52\x65\x73\x65\x61\x72\x63\x68\x20\x49\x6E\x73\x74\x69\x74\x75\x74\x69\x6F\x6E\x73\x20\x43\x65\x72\x74\x2E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x31\x40\x30\x3E\x06\x03\x55\x04\x03\x13\x37\x48\x65\x6C\x6C\x65\x6E\x69\x63\x20\x41\x63\x61\x64\x65\x6D\x69\x63\x20\x61\x6E\x64\x20\x52\x65\x73\x65\x61\x72\x63\x68\x20\x49\x6E\x73\x74\x69\x74\x75\x74\x69\x6F\x6E\x73\x20\x52\x6F\x6F\x74\x43\x41\x20\x32\x30\x31\x31\x30\x1E\x17\x0D\x31\x31\x31\x32\x30\x36\x31\x33\x34\x39\x35\x32\x5A\x17\x0D\x33\x31\x31\x32\x30\x31\x31\x33\x34\x39\x35\x32\x5A\x30\x81\x95\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x52\x31\x44\x30\x42\x06\x03\x55\x04\x0A\x13\x3B\x48\x65\x6C\x6C\x65\x6E\x69\x63\x20\x41\x63\x61\x64\x65\x6D\x69\x63\x20\x61\x6E\x64\x20\x52\x65\x73\x65\x61\x72\x63\x68\x20\x49\x6E\x73\x74\x69\x74\x75\x74\x69\x6F\x6E\x73\x20\x43\x65\x72\x74\x2E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x31\x40\x30\x3E\x06\x03\x55\x04\x03\x13\x37\x48\x65\x6C\x6C\x65\x6E\x69\x63\x20\x41\x63\x61\x64\x65\x6D\x69\x63\x20\x61\x6E\x64\x20\x52\x65\x73\x65\x61\x72\x63\x68\x20\x49\x6E\x73\x74\x69\x74\x75\x74\x69\x6F\x6E\x73\x20\x52\x6F\x6F\x74\x43\x41\x20\x32\x30\x31\x31\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xA9\x53\x00\xE3\x2E\xA6\xF6\x8E\xFA\x60\xD8\x2D\x95\x3E\xF8\x2C\x2A\x54\x4E\xCD\xB9\x84\x61\x94\x58\x4F\x8F\x3D\x8B\xE4\x43\xF3\x75\x89\x8D\x51\xE4\xC3\x37\xD2\x8A\x88\x4D\x79\x1E\xB7\x12\xDD\x43\x78\x4A\x8A\x92\xE6\xD7\x48\xD5\x0F\xA4\x3A\x29\x44\x35\xB8\x07\xF6\x68\x1D\x55\xCD\x38\x51\xF0\x8C\x24\x31\x85\xAF\x83\xC9\x7D\xE9\x77\xAF\xED\x1A\x7B\x9D\x17\xF9\xB3\x9D\x38\x50\x0F\xA6\x5A\x79\x91\x80\xAF\x37\xAE\xA6\xD3\x31\xFB\xB5\x26\x09\x9D\x3C\x5A\xEF\x51\xC5\x2B\xDF\x96\x5D\xEB\x32\x1E\x02\xDA\x70\x49\xEC\x6E\x0C\xC8\x9A\x37\x8D\xF7\xF1\x36\x60\x4B\x26\x2C\x82\x9E\xD0\x78\xF3\x0D\x0F\x63\xA4\x51\x30\xE1\xF9\x2B\x27\x12\x07\xD8\xEA\xBD\x18\x62\x98\xB0\x59\x37\x7D\xBE\xEE\xF3\x20\x51\x42\x5A\x83\xEF\x93\xBA\x69\x15\xF1\x62\x9D\x9F\x99\x39\x82\xA1\xB7\x74\x2E\x8B\xD4\xC5\x0B\x7B\x2F\xF0\xC8\x0A\xDA\x3D\x79\x0A\x9A\x93\x1C\xA5\x28\x72\x73\x91\x43\x9A\xA7\xD1\x4D\x85\x84\xB9\xA9\x74\x8F\x14\x40\xC7\xDC\xDE\xAC\x41\x64\x6C\xB4\x19\x9B\x02\x63\x6D\x24\x64\x8F\x44\xB2\x25\xEA\xCE\x5D\x74\x0C\x63\x32\x5C\x8D\x87\xE5\x02\x03\x01\x00\x01\xA3\x81\x89\x30\x81\x86\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0B\x06\x03\x55\x1D\x0F\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xA6\x91\x42\xFD\x13\x61\x4A\x23\x9E\x08\xA4\x29\xE5\xD8\x13\x04\x23\xEE\x41\x25\x30\x47\x06\x03\x55\x1D\x1E\x04\x40\x30\x3E\xA0\x3C\x30\x05\x82\x03\x2E\x67\x72\x30\x05\x82\x03\x2E\x65\x75\x30\x06\x82\x04\x2E\x65\x64\x75\x30\x06\x82\x04\x2E\x6F\x72\x67\x30\x05\x81\x03\x2E\x67\x72\x30\x05\x81\x03\x2E\x65\x75\x30\x06\x81\x04\x2E\x65\x64\x75\x30\x06\x81\x04\x2E\x6F\x72\x67\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x1F\xEF\x79\x41\xE1\x7B\x6E\x3F\xB2\x8C\x86\x37\x42\x4A\x4E\x1C\x37\x1E\x8D\x66\xBA\x24\x81\xC9\x4F\x12\x0F\x21\xC0\x03\x97\x86\x25\x6D\x5D\xD3\x22\x29\xA8\x6C\xA2\x0D\xA9\xEB\x3D\x06\x5B\x99\x3A\xC7\xCC\xC3\x9A\x34\x7F\xAB\x0E\xC8\x4E\x1C\xE1\xFA\xE4\xDC\xCD\x0D\xBE\xBF\x24\xFE\x6C\xE7\x6B\xC2\x0D\xC8\x06\x9E\x4E\x8D\x61\x28\xA6\x6A\xFD\xE5\xF6\x62\xEA\x18\x3C\x4E\xA0\x53\x9D\xB2\x3A\x9C\xEB\xA5\x9C\x91\x16\xB6\x4D\x82\xE0\x0C\x05\x48\xA9\x6C\xF5\xCC\xF8\xCB\x9D\x49\xB4\xF0\x02\xA5\xFD\x70\x03\xED\x8A\x21\xA5\xAE\x13\x86\x49\xC3\x33\x73\xBE\x87\x3B\x74\x8B\x17\x45\x26\x4C\x16\x91\x83\xFE\x67\x7D\xCD\x4D\x63\x67\xFA\xF3\x03\x12\x96\x78\x06\x8D\xB1\x67\xED\x8E\x3F\xBE\x9F\x4F\x02\xF5\xB3\x09\x2F\xF3\x4C\x87\xDF\x2A\xCB\x95\x7C\x01\xCC\xAC\x36\x7A\xBF\xA2\x73\x7A\xF7\x8F\xC1\xB5\x9A\xA1\x14\xB2\x8F\x33\x9F\x0D\xEF\x22\xDC\x66\x7B\x84\xBD\x45\x17\x06\x3D\x3C\xCA\xB9\x77\x34\x8F\xCA\xEA\xCF\x3F\x31\x3E\xE3\x88\xE3\x80\x49\x25\xC8\x97\xB5\x9D\x9A\x99\x4D\xB0\x3C\xF8\x4A\x00\x9B\x64\xDD\x9F\x39\x4B\xD1\x27\xD7\xB8",
+	["CN=Actalis Authentication Root CA,O=Actalis S.p.A./03358520967,L=Milan,C=IT"] = "\x30\x82\x05\xBB\x30\x82\x03\xA3\xA0\x03\x02\x01\x02\x02\x08\x57\x0A\x11\x97\x42\xC4\xE3\xCC\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x6B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x49\x54\x31\x0E\x30\x0C\x06\x03\x55\x04\x07\x0C\x05\x4D\x69\x6C\x61\x6E\x31\x23\x30\x21\x06\x03\x55\x04\x0A\x0C\x1A\x41\x63\x74\x61\x6C\x69\x73\x20\x53\x2E\x70\x2E\x41\x2E\x2F\x30\x33\x33\x35\x38\x35\x32\x30\x39\x36\x37\x31\x27\x30\x25\x06\x03\x55\x04\x03\x0C\x1E\x41\x63\x74\x61\x6C\x69\x73\x20\x41\x75\x74\x68\x65\x6E\x74\x69\x63\x61\x74\x69\x6F\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x31\x31\x30\x39\x32\x32\x31\x31\x32\x32\x30\x32\x5A\x17\x0D\x33\x30\x30\x39\x32\x32\x31\x31\x32\x32\x30\x32\x5A\x30\x6B\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x49\x54\x31\x0E\x30\x0C\x06\x03\x55\x04\x07\x0C\x05\x4D\x69\x6C\x61\x6E\x31\x23\x30\x21\x06\x03\x55\x04\x0A\x0C\x1A\x41\x63\x74\x61\x6C\x69\x73\x20\x53\x2E\x70\x2E\x41\x2E\x2F\x30\x33\x33\x35\x38\x35\x32\x30\x39\x36\x37\x31\x27\x30\x25\x06\x03\x55\x04\x03\x0C\x1E\x41\x63\x74\x61\x6C\x69\x73\x20\x41\x75\x74\x68\x65\x6E\x74\x69\x63\x61\x74\x69\x6F\x6E\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xA7\xC6\xC4\xA5\x29\xA4\x2C\xEF\xE5\x18\xC5\xB0\x50\xA3\x6F\x51\x3B\x9F\x0A\x5A\xC9\xC2\x48\x38\x0A\xC2\x1C\xA0\x18\x7F\x91\xB5\x87\xB9\x40\x3F\xDD\x1D\x68\x1F\x08\x83\xD5\x2D\x1E\x88\xA0\xF8\x8F\x56\x8F\x6D\x99\x02\x92\x90\x16\xD5\x5F\x08\x6C\x89\xD7\xE1\xAC\xBC\x20\xC2\xB1\xE0\x83\x51\x8A\x69\x4D\x00\x96\x5A\x6F\x2F\xC0\x44\x7E\xA3\x0E\xE4\x91\xCD\x58\xEE\xDC\xFB\xC7\x1E\x45\x47\xDD\x27\xB9\x08\x01\x9F\xA6\x21\x1D\xF5\x41\x2D\x2F\x4C\xFD\x28\xAD\xE0\x8A\xAD\x22\xB4\x56\x65\x8E\x86\x54\x8F\x93\x43\x29\xDE\x39\x46\x78\xA3\x30\x23\xBA\xCD\xF0\x7D\x13\x57\xC0\x5D\xD2\x83\x6B\x48\x4C\xC4\xAB\x9F\x80\x5A\x5B\x3A\xBD\xC9\xA7\x22\x3F\x80\x27\x33\x5B\x0E\xB7\x8A\x0C\x5D\x07\x37\x08\xCB\x6C\xD2\x7A\x47\x22\x44\x35\xC5\xCC\xCC\x2E\x8E\xDD\x2A\xED\xB7\x7D\x66\x0D\x5F\x61\x51\x22\x55\x1B\xE3\x46\xE3\xE3\x3D\xD0\x35\x62\x9A\xDB\xAF\x14\xC8\x5B\xA1\xCC\x89\x1B\xE1\x30\x26\xFC\xA0\x9B\x1F\x81\xA7\x47\x1F\x04\xEB\xA3\x39\x92\x06\x9F\x99\xD3\xBF\xD3\xEA\x4F\x50\x9C\x19\xFE\x96\x87\x1E\x3C\x65\xF6\xA3\x18\x24\x83\x86\x10\xE7\x54\x3E\xA8\x3A\x76\x24\x4F\x81\x21\xC5\xE3\x0F\x02\xF8\x93\x94\x47\x20\xBB\xFE\xD4\x0E\xD3\x68\xB9\xDD\xC4\x7A\x84\x82\xE3\x53\x54\x79\xDD\xDB\x9C\xD2\xF2\x07\x9B\x2E\xB6\xBC\x3E\xED\x85\x6D\xEF\x25\x11\xF2\x97\x1A\x42\x61\xF7\x4A\x97\xE8\x8B\xB1\x10\x07\xFA\x65\x81\xB2\xA2\x39\xCF\xF7\x3C\xFF\x18\xFB\xC6\xF1\x5A\x8B\x59\xE2\x02\xAC\x7B\x92\xD0\x4E\x14\x4F\x59\x45\xF6\x0C\x5E\x28\x5F\xB0\xE8\x3F\x45\xCF\xCF\xAF\x9B\x6F\xFB\x84\xD3\x77\x5A\x95\x6F\xAC\x94\x84\x9E\xEE\xBC\xC0\x4A\x8F\x4A\x93\xF8\x44\x21\xE2\x31\x45\x61\x50\x4E\x10\xD8\xE3\x35\x7C\x4C\x19\xB4\xDE\x05\xBF\xA3\x06\x9F\xC8\xB5\xCD\xE4\x1F\xD7\x17\x06\x0D\x7A\x95\x74\x55\x0D\x68\x1A\xFC\x10\x1B\x62\x64\x9D\x6D\xE0\x95\xA0\xC3\x94\x07\x57\x0D\x14\xE6\xBD\x05\xFB\xB8\x9F\xE6\xDF\x8B\xE2\xC6\xE7\x7E\x96\xF6\x53\xC5\x80\x34\x50\x28\x58\xF0\x12\x50\x71\x17\x30\xBA\xE6\x78\x63\xBC\xF4\xB2\xAD\x9B\x2B\xB2\xFE\xE1\x39\x8C\x5E\xBA\x0B\x20\x94\xDE\x7B\x83\xB8\xFF\xE3\x56\x8D\xB7\x11\xE9\x3B\x8C\xF2\xB1\xC1\x5D\x9D\xA4\x0B\x4C\x2B\xD9\xB2\x18\xF5\xB5\x9F\x4B\x02\x03\x01\x00\x01\xA3\x63\x30\x61\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x52\xD8\x88\x3A\xC8\x9F\x78\x66\xED\x89\xF3\x7B\x38\x70\x94\xC9\x02\x02\x36\xD0\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\x52\xD8\x88\x3A\xC8\x9F\x78\x66\xED\x89\xF3\x7B\x38\x70\x94\xC9\x02\x02\x36\xD0\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x02\x01\x00\x0B\x7B\x72\x87\xC0\x60\xA6\x49\x4C\x88\x58\xE6\x1D\x88\xF7\x14\x64\x48\xA6\xD8\x58\x0A\x0E\x4F\x13\x35\xDF\x35\x1D\xD4\xED\x06\x31\xC8\x81\x3E\x6A\xD5\xDD\x3B\x1A\x32\xEE\x90\x3D\x11\xD2\x2E\xF4\x8E\xC3\x63\x2E\x23\x66\xB0\x67\xBE\x6F\xB6\xC0\x13\x39\x60\xAA\xA2\x34\x25\x93\x75\x52\xDE\xA7\x9D\xAD\x0E\x87\x89\x52\x71\x6A\x16\x3C\x19\x1D\x83\xF8\x9A\x29\x65\xBE\xF4\x3F\x9A\xD9\xF0\xF3\x5A\x87\x21\x71\x80\x4D\xCB\xE0\x38\x9B\x3F\xBB\xFA\xE0\x30\x4D\xCF\x86\xD3\x65\x10\x19\x18\xD1\x97\x02\xB1\x2B\x72\x42\x68\xAC\xA0\xBD\x4E\x5A\xDA\x18\xBF\x6B\x98\x81\xD0\xFD\x9A\xBE\x5E\x15\x48\xCD\x11\x15\xB9\xC0\x29\x5C\xB4\xE8\x88\xF7\x3E\x36\xAE\xB7\x62\xFD\x1E\x62\xDE\x70\x78\x10\x1C\x48\x5B\xDA\xBC\xA4\x38\xBA\x67\xED\x55\x3E\x5E\x57\xDF\xD4\x03\x40\x4C\x81\xA4\xD2\x4F\x63\xA7\x09\x42\x09\x14\xFC\x00\xA9\xC2\x80\x73\x4F\x2E\xC0\x40\xD9\x11\x7B\x48\xEA\x7A\x02\xC0\xD3\xEB\x28\x01\x26\x58\x74\xC1\xC0\x73\x22\x6D\x93\x95\xFD\x39\x7D\xBB\x2A\xE3\xF6\x82\xE3\x2C\x97\x5F\x4E\x1F\x91\x94\xFA\xFE\x2C\xA3\xD8\x76\x1A\xB8\x4D\xB2\x38\x4F\x9B\xFA\x1D\x48\x60\x79\x26\xE2\xF3\xFD\xA9\xD0\x9A\xE8\x70\x8F\x49\x7A\xD6\xE5\xBD\x0A\x0E\xDB\x2D\xF3\x8D\xBF\xEB\xE3\xA4\x7D\xCB\xC7\x95\x71\xE8\xDA\xA3\x7C\xC5\xC2\xF8\x74\x92\x04\x1B\x86\xAC\xA4\x22\x53\x40\xB6\xAC\xFE\x4C\x76\xCF\xFB\x94\x32\xC0\x35\x9F\x76\x3F\x6E\xE5\x90\x6E\xA0\xA6\x26\xA2\xB8\x2C\xBE\xD1\x2B\x85\xFD\xA7\x68\xC8\xBA\x01\x2B\xB1\x6C\x74\x1D\xB8\x73\x95\xE7\xEE\xB7\xC7\x25\xF0\x00\x4C\x00\xB2\x7E\xB6\x0B\x8B\x1C\xF3\xC0\x50\x9E\x25\xB9\xE0\x08\xDE\x36\x66\xFF\x37\xA5\xD1\xBB\x54\x64\x2C\xC9\x27\xB5\x4B\x92\x7E\x65\xFF\xD3\x2D\xE1\xB9\x4E\xBC\x7F\xA4\x41\x21\x90\x41\x77\xA6\x39\x1F\xEA\x9E\xE3\x9F\xD0\x66\x6F\x05\xEC\xAA\x76\x7E\xBF\x6B\x16\xA0\xEB\xB5\xC7\xFC\x92\x54\x2F\x2B\x11\x27\x25\x37\x78\x4C\x51\x6A\xB0\xF3\xCC\x58\x5D\x14\xF1\x6A\x48\x15\xFF\xC2\x07\xB6\xB1\x8D\x0F\x8E\x5C\x50\x46\xB3\x3D\xBF\x01\x98\x4F\xB2\x59\x54\x47\x3E\x34\x7B\x78\x6D\x56\x93\x2E\x73\xEA\x66\x28\x78\xCD\x1D\x14\xBF\xA0\x8F\x2F\x2E\xB8\x2E\x8E\xF2\x14\x8A\xCC\xE9\xB5\x7C\xFB\x6C\x9D\x0C\xA5\xE1\x96",
+	["OU=Trustis FPS Root CA,O=Trustis Limited,C=GB"] = "\x30\x82\x03\x67\x30\x82\x02\x4F\xA0\x03\x02\x01\x02\x02\x10\x1B\x1F\xAD\xB6\x20\xF9\x24\xD3\x36\x6B\xF7\xC7\xF1\x8C\xA0\x59\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x30\x45\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x54\x72\x75\x73\x74\x69\x73\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x1C\x30\x1A\x06\x03\x55\x04\x0B\x13\x13\x54\x72\x75\x73\x74\x69\x73\x20\x46\x50\x53\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x30\x33\x31\x32\x32\x33\x31\x32\x31\x34\x30\x36\x5A\x17\x0D\x32\x34\x30\x31\x32\x31\x31\x31\x33\x36\x35\x34\x5A\x30\x45\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x47\x42\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x54\x72\x75\x73\x74\x69\x73\x20\x4C\x69\x6D\x69\x74\x65\x64\x31\x1C\x30\x1A\x06\x03\x55\x04\x0B\x13\x13\x54\x72\x75\x73\x74\x69\x73\x20\x46\x50\x53\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x01\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x01\x0F\x00\x30\x82\x01\x0A\x02\x82\x01\x01\x00\xC5\x50\x7B\x9E\x3B\x35\xD0\xDF\xC4\x8C\xCD\x8E\x9B\xED\xA3\xC0\x36\x99\xF4\x42\xEA\xA7\x3E\x80\x83\x0F\xA6\xA7\x59\x87\xC9\x90\x45\x43\x7E\x00\xEA\x86\x79\x2A\x03\xBD\x3D\x37\x99\x89\x66\xB7\xE5\x8A\x56\x86\x93\x9C\x68\x4B\x68\x04\x8C\x93\x93\x02\x3E\x30\xD2\x37\x3A\x22\x61\x89\x1C\x85\x4E\x7D\x8F\xD5\xAF\x7B\x35\xF6\x7E\x28\x47\x89\x31\xDC\x0E\x79\x64\x1F\x99\xD2\x5B\xBA\xFE\x7F\x60\xBF\xAD\xEB\xE7\x3C\x38\x29\x6A\x2F\xE5\x91\x0B\x55\xFF\xEC\x6F\x58\xD5\x2D\xC9\xDE\x4C\x66\x71\x8F\x0C\xD7\x04\xDA\x07\xE6\x1E\x18\xE3\xBD\x29\x02\xA8\xFA\x1C\xE1\x5B\xB9\x83\xA8\x41\x48\xBC\x1A\x71\x8D\xE7\x62\xE5\x2D\xB2\xEB\xDF\x7C\xCF\xDB\xAB\x5A\xCA\x31\xF1\x4C\x22\xF3\x05\x13\xF7\x82\xF9\x73\x79\x0C\xBE\xD7\x4B\x1C\xC0\xD1\x15\x3C\x93\x41\x64\xD1\xE6\xBE\x23\x17\x22\x00\x89\x5E\x1F\x6B\xA5\xAC\x6E\xA7\x4B\x8C\xED\xA3\x72\xE6\xAF\x63\x4D\x2F\x85\xD2\x14\x35\x9A\x2E\x4E\x8C\xEA\x32\x98\x28\x86\xA1\x91\x09\x41\x3A\xB4\xE1\xE3\xF2\xFA\xF0\xC9\x0A\xA2\x41\xDD\xA9\xE3\x03\xC7\x88\x15\x3B\x1C\xD4\x1A\x94\xD7\x9F\x64\x59\x12\x6D\x02\x03\x01\x00\x01\xA3\x53\x30\x51\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1F\x06\x03\x55\x1D\x23\x04\x18\x30\x16\x80\x14\xBA\xFA\x71\x25\x79\x8B\x57\x41\x25\x21\x86\x0B\x71\xEB\xB2\x64\x0E\x8B\x21\x67\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xBA\xFA\x71\x25\x79\x8B\x57\x41\x25\x21\x86\x0B\x71\xEB\xB2\x64\x0E\x8B\x21\x67\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x05\x05\x00\x03\x82\x01\x01\x00\x7E\x58\xFF\xFD\x35\x19\x7D\x9C\x18\x4F\x9E\xB0\x2B\xBC\x8E\x8C\x14\xFF\x2C\xA0\xDA\x47\x5B\xC3\xEF\x81\x2D\xAF\x05\xEA\x74\x48\x5B\xF3\x3E\x4E\x07\xC7\x6D\xC5\xB3\x93\xCF\x22\x35\x5C\xB6\x3F\x75\x27\x5F\x09\x96\xCD\xA0\xFE\xBE\x40\x0C\x5C\x12\x55\xF8\x93\x82\xCA\x29\xE9\x5E\x3F\x56\x57\x8B\x38\x36\xF7\x45\x1A\x4C\x28\xCD\x9E\x41\xB8\xED\x56\x4C\x84\xA4\x40\xC8\xB8\xB0\xA5\x2B\x69\x70\x04\x6A\xC3\xF8\xD4\x12\x32\xF9\x0E\xC3\xB1\xDC\x32\x84\x44\x2C\x6F\xCB\x46\x0F\xEA\x66\x41\x0F\x4F\xF1\x58\xA5\xA6\x0D\x0D\x0F\x61\xDE\xA5\x9E\x5D\x7D\x65\xA1\x3C\x17\xE7\xA8\x55\x4E\xEF\xA0\xC7\xED\xC6\x44\x7F\x54\xF5\xA3\xE0\x8F\xF0\x7C\x55\x22\x8F\x29\xB6\x81\xA3\xE1\x6D\x4E\x2C\x1B\x80\x67\xEC\xAD\x20\x9F\x0C\x62\x61\xD5\x97\xFF\x43\xED\x2D\xC1\xDA\x5D\x29\x2A\x85\x3F\xAC\x65\xEE\x86\x0F\x05\x8D\x90\x5F\xDF\xEE\x9F\xF4\xBF\xEE\x1D\xFB\x98\xE4\x7F\x90\x2B\x84\x78\x10\x0E\x6C\x49\x53\xEF\x15\x5B\x65\x46\x4A\x5D\xAF\xBA\xFB\x3A\x72\x1D\xCD\xF6\x25\x88\x1E\x97\xCC\x21\x9C\x29\x01\x0D\x65\xEB\x57\xD9\xF3\x57\x96\xBB\x48\xCD\x81",
+	["CN=StartCom Certification Authority G2,O=StartCom Ltd.,C=IL"] = "\x30\x82\x05\x63\x30\x82\x03\x4B\xA0\x03\x02\x01\x02\x02\x01\x3B\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x53\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x49\x4C\x31\x16\x30\x14\x06\x03\x55\x04\x0A\x13\x0D\x53\x74\x61\x72\x74\x43\x6F\x6D\x20\x4C\x74\x64\x2E\x31\x2C\x30\x2A\x06\x03\x55\x04\x03\x13\x23\x53\x74\x61\x72\x74\x43\x6F\x6D\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x47\x32\x30\x1E\x17\x0D\x31\x30\x30\x31\x30\x31\x30\x31\x30\x30\x30\x31\x5A\x17\x0D\x33\x39\x31\x32\x33\x31\x32\x33\x35\x39\x30\x31\x5A\x30\x53\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x49\x4C\x31\x16\x30\x14\x06\x03\x55\x04\x0A\x13\x0D\x53\x74\x61\x72\x74\x43\x6F\x6D\x20\x4C\x74\x64\x2E\x31\x2C\x30\x2A\x06\x03\x55\x04\x03\x13\x23\x53\x74\x61\x72\x74\x43\x6F\x6D\x20\x43\x65\x72\x74\x69\x66\x69\x63\x61\x74\x69\x6F\x6E\x20\x41\x75\x74\x68\x6F\x72\x69\x74\x79\x20\x47\x32\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xB6\x89\x36\x5B\x07\xB7\x20\x36\xBD\x82\xBB\xE1\x16\x20\x03\x95\x7A\xAF\x0E\xA3\x55\xC9\x25\x99\x4A\xC5\xD0\x56\x41\x87\x90\x4D\x21\x60\xA4\x14\x87\x3B\xCD\xFD\xB2\x3E\xB4\x67\x03\x6A\xED\xE1\x0F\x4B\xC0\x91\x85\x70\x45\xE0\x42\x9E\xDE\x29\x23\xD4\x01\x0D\xA0\x10\x79\xB8\xDB\x03\xBD\xF3\xA9\x2F\xD1\xC6\xE0\x0F\xCB\x9E\x8A\x14\x0A\xB8\xBD\xF6\x56\x62\xF1\xC5\x72\xB6\x32\x25\xD9\xB2\xF3\xBD\x65\xC5\x0D\x2C\x6E\xD5\x92\x6F\x18\x8B\x00\x41\x14\x82\x6F\x40\x20\x26\x7A\x28\x0F\xF5\x1E\x7F\x27\xF7\x94\xB1\x37\x3D\xB7\xC7\x91\xF7\xE2\x01\xEC\xFD\x94\x89\xE1\xCC\x6E\xD3\x36\xD6\x0A\x19\x79\xAE\xD7\x34\x82\x65\xFF\x7C\x42\xBB\xB6\xDD\x0B\xA6\x34\xAF\x4B\x60\xFE\x7F\x43\x49\x06\x8B\x8C\x43\xB8\x56\xF2\xD9\x7F\x21\x43\x17\xEA\xA7\x48\x95\x01\x75\x75\xEA\x2B\xA5\x43\x95\xEA\x15\x84\x9D\x08\x8D\x26\x6E\x55\x9B\xAB\xDC\xD2\x39\xD2\x31\x1D\x60\xE2\xAC\xCC\x56\x45\x24\xF5\x1C\x54\xAB\xEE\x86\xDD\x96\x32\x85\xF8\x4C\x4F\xE8\x95\x76\xB6\x05\xDD\x36\x23\x67\xBC\xFF\x15\xE2\xCA\x3B\xE6\xA6\xEC\x3B\xEC\x26\x11\x34\x48\x8D\xF6\x80\x2B\x1A\x23\x02\xEB\x8A\x1C\x3A\x76\x2A\x7B\x56\x16\x1C\x72\x2A\xB3\xAA\xE3\x60\xA5\x00\x9F\x04\x9B\xE2\x6F\x1E\x14\x58\x5B\xA5\x6C\x8B\x58\x3C\xC3\xBA\x4E\x3A\x5C\xF7\xE1\x96\x2B\x3E\xEF\x07\xBC\xA4\xE5\x5D\xCC\x4D\x9F\x0D\xE1\xDC\xAA\xBB\xE1\x6E\x1A\xEC\x8F\xE1\xB6\x4C\x4D\x79\x72\x5D\x17\x35\x0B\x1D\xD7\xC1\x47\xDA\x96\x24\xE0\xD0\x72\xA8\x5A\x5F\x66\x2D\x10\xDC\x2F\x2A\x13\xAE\x26\xFE\x0A\x1C\x19\xCC\xD0\x3E\x0B\x9C\xC8\x09\x2E\xF9\x5B\x96\x7A\x47\x9C\xE9\x7A\xF3\x05\x50\x74\x95\x73\x9E\x30\x09\xF3\x97\x82\x5E\xE6\x8F\x39\x08\x1E\x59\xE5\x35\x14\x42\x13\xFF\x00\x9C\xF7\xBE\xAA\x50\xCF\xE2\x51\x48\xD7\xB8\x6F\xAF\xF8\x4E\x7E\x33\x98\x92\x14\x62\x3A\x75\x63\xCF\x7B\xFA\xDE\x82\x3B\xA9\xBB\x39\xE2\xC4\xBD\x2C\x00\x0E\xC8\x17\xAC\x13\xEF\x4D\x25\x8E\xD8\xB3\x90\x2F\xA9\xDA\x29\x7D\x1D\xAF\x74\x3A\xB2\x27\xC0\xC1\x1E\x3E\x75\xA3\x16\xA9\xAF\x7A\x22\x5D\x9F\x13\x1A\xCF\xA7\xA0\xEB\xE3\x86\x0A\xD3\xFD\xE6\x96\x95\xD7\x23\xC8\x37\xDD\xC4\x7C\xAA\x36\xAC\x98\x1A\x12\xB1\xE0\x4E\xE8\xB1\x3B\xF5\xD6\x6F\xF1\x30\xD7\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x4B\xC5\xB4\x40\x6B\xAD\x1C\xB3\xA5\x1C\x65\x6E\x46\x36\x89\x87\x05\x0C\x0E\xB6\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x02\x01\x00\x73\x57\x3F\x2C\xD5\x95\x32\x7E\x37\xDB\x96\x92\xEB\x19\x5E\x7E\x53\xE7\x41\xEC\x11\xB6\x47\xEF\xB5\xDE\xED\x74\x5C\xC5\xF1\x8E\x49\xE0\xFC\x6E\x99\x13\xCD\x9F\x8A\xDA\xCD\x3A\x0A\xD8\x3A\x5A\x09\x3F\x5F\x34\xD0\x2F\x03\xD2\x66\x1D\x1A\xBD\x9C\x90\x37\xC8\x0C\x8E\x07\x5A\x94\x45\x46\x2A\xE6\xBE\x7A\xDA\xA1\xA9\xA4\x69\x12\x92\xB0\x7D\x36\xD4\x44\x87\xD7\x51\xF1\x29\x63\xD6\x75\xCD\x16\xE4\x27\x89\x1D\xF8\xC2\x32\x48\xFD\xDB\x99\xD0\x8F\x5F\x54\x74\xCC\xAC\x67\x34\x11\x62\xD9\x0C\x0A\x37\x87\xD1\xA3\x17\x48\x8E\xD2\x17\x1D\xF6\xD7\xFD\xDB\x65\xEB\xFD\xA8\xD4\xF5\xD6\x4F\xA4\x5B\x75\xE8\xC5\xD2\x60\xB2\xDB\x09\x7E\x25\x8B\x7B\xBA\x52\x92\x9E\x3E\xE8\xC5\x77\xA1\x3C\xE0\x4A\x73\x6B\x61\xCF\x86\xDC\x43\xFF\xFF\x21\xFE\x23\x5D\x24\x4A\xF5\xD3\x6D\x0F\x62\x04\x05\x57\x82\xDA\x6E\xA4\x33\x25\x79\x4B\x2E\x54\x19\x8B\xCC\x2C\x3D\x30\xE9\xD1\x06\xFF\xE8\x32\x46\xBE\xB5\x33\x76\x77\xA8\x01\x5D\x96\xC1\xC1\xD5\xBE\xAE\x25\xC0\xC9\x1E\x0A\x09\x20\x88\xA1\x0E\xC9\xF3\x6F\x4D\x82\x54\x00\x20\xA7\xD2\x8F\xE4\x39\x54\x17\x2E\x8D\x1E\xB8\x1B\xBB\x1B\xBD\x9A\x4E\x3B\x10\x34\xDC\x9C\x88\x53\xEF\xA2\x31\x5B\x58\x4F\x91\x62\xC8\xC2\x9A\x9A\xCD\x15\x5D\x38\xA9\xD6\xBE\xF8\x13\xB5\x9F\x12\x69\xF2\x50\x62\xAC\xFB\x17\x37\xF4\xEE\xB8\x75\x67\x60\x10\xFB\x83\x50\xF9\x44\xB5\x75\x9C\x40\x17\xB2\xFE\xFD\x79\x5D\x6E\x58\x58\x5F\x30\xFC\x00\xAE\xAF\x33\xC1\x0E\x4E\x6C\xBA\xA7\xA6\xA1\x7F\x32\xDB\x38\xE0\xB1\x72\x17\x0A\x2B\x91\xEC\x6A\x63\x26\xED\x89\xD4\x78\xCC\x74\x1E\x05\xF8\x6B\xFE\x8C\x6A\x76\x39\x29\xAE\x65\x23\x12\x95\x08\x22\x1C\x97\xCE\x5B\x06\xEE\x0C\xE2\xBB\xBC\x1F\x44\x93\xF6\xD8\x38\x45\x05\x21\xED\xE4\xAD\xAB\x12\xB6\x03\xA4\x42\x2E\x2D\xC4\x09\x3A\x03\x67\x69\x84\x9A\xE1\x59\x90\x8A\x28\x85\xD5\x5D\x74\xB1\xD1\x0E\x20\x58\x9B\x13\xA5\xB0\x63\xA6\xED\x7B\x47\xFD\x45\x55\x30\xA4\xEE\x9A\xD4\xE6\xE2\x87\xEF\x98\xC9\x32\x82\x11\x29\x22\xBC\x00\x0A\x31\x5E\x2D\x0F\xC0\x8E\xE9\x6B\xB2\x8F\x2E\x06\xD8\xD1\x91\xC7\xC6\x12\xF4\x4C\xFD\x30\x17\xC3\xC1\xDA\x38\x5B\xE3\xA9\xEA\xE6\xA1\xBA\x79\xEF\x73\xD8\xB6\x53\x57\x2D\xF6\xD0\xE1\xD7\x48",
+	["CN=Buypass Class 2 Root CA,O=Buypass AS-983163327,C=NO"] = "\x30\x82\x05\x59\x30\x82\x03\x41\xA0\x03\x02\x01\x02\x02\x01\x02\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x4E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4E\x4F\x31\x1D\x30\x1B\x06\x03\x55\x04\x0A\x0C\x14\x42\x75\x79\x70\x61\x73\x73\x20\x41\x53\x2D\x39\x38\x33\x31\x36\x33\x33\x32\x37\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x0C\x17\x42\x75\x79\x70\x61\x73\x73\x20\x43\x6C\x61\x73\x73\x20\x32\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x31\x30\x31\x30\x32\x36\x30\x38\x33\x38\x30\x33\x5A\x17\x0D\x34\x30\x31\x30\x32\x36\x30\x38\x33\x38\x30\x33\x5A\x30\x4E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4E\x4F\x31\x1D\x30\x1B\x06\x03\x55\x04\x0A\x0C\x14\x42\x75\x79\x70\x61\x73\x73\x20\x41\x53\x2D\x39\x38\x33\x31\x36\x33\x33\x32\x37\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x0C\x17\x42\x75\x79\x70\x61\x73\x73\x20\x43\x6C\x61\x73\x73\x20\x32\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xD7\xC7\x5E\xF7\xC1\x07\xD4\x77\xFB\x43\x21\xF4\xF4\xF5\x69\xE4\xEE\x32\x01\xDB\xA3\x86\x1F\xE4\x59\x0D\xBA\xE7\x75\x83\x52\xEB\xEA\x1C\x61\x15\x48\xBB\x1D\x07\xCA\x8C\xAE\xB0\xDC\x96\x9D\xEA\xC3\x60\x92\x86\x82\x28\x73\x9C\x56\x06\xFF\x4B\x64\xF0\x0C\x2A\x37\x49\xB5\xE5\xCF\x0C\x7C\xEE\xF1\x4A\xBB\x73\x30\x65\xF3\xD5\x2F\x83\xB6\x7E\xE3\xE7\xF5\x9E\xAB\x60\xF9\xD3\xF1\x9D\x92\x74\x8A\xE4\x1C\x96\xAC\x5B\x80\xE9\xB5\xF4\x31\x87\xA3\x51\xFC\xC7\x7E\xA1\x6F\x8E\x53\x77\xD4\x97\xC1\x55\x33\x92\x3E\x18\x2F\x75\xD4\xAD\x86\x49\xCB\x95\xAF\x54\x06\x6C\xD8\x06\x13\x8D\x5B\xFF\xE1\x26\x19\x59\xC0\x24\xBA\x81\x71\x79\x90\x44\x50\x68\x24\x94\x5F\xB8\xB3\x11\xF1\x29\x41\x61\xA3\x41\xCB\x23\x36\xD5\xC1\xF1\x32\x50\x10\x4E\x7F\xF4\x86\x93\xEC\x84\xD3\x8E\xBC\x4B\xBF\x5C\x01\x4E\x07\x3D\xDC\x14\x8A\x94\x0A\xA4\xEA\x73\xFB\x0B\x51\xE8\x13\x07\x18\xFA\x0E\xF1\x2B\xD1\x54\x15\x7D\x3C\xE1\xF7\xB4\x19\x42\x67\x62\x5E\x77\xE0\xA2\x55\xEC\xB6\xD9\x69\x17\xD5\x3A\xAF\x44\xED\x4A\xC5\x9E\xE4\x7A\x27\x7C\xE5\x75\xD7\xAA\xCB\x25\xE7\xDF\x6B\x0A\xDB\x0F\x4D\x93\x4E\xA8\xA0\xCD\x7B\x2E\xF2\x59\x01\x6A\xB7\x0D\xB8\x07\x81\x7E\x8B\x38\x1B\x38\xE6\x0A\x57\x99\x3D\xEE\x21\xE8\xA3\xF5\x0C\x16\xDD\x8B\xEC\x34\x8E\x9C\x2A\x1C\x00\x15\x17\x8D\x68\x83\xD2\x70\x9F\x18\x08\xCD\x11\x68\xD5\xC9\x6B\x52\xCD\xC4\x46\x8F\xDC\xB5\xF3\xD8\x57\x73\x1E\xE9\x94\x39\x04\xBF\xD3\xDE\x38\xDE\xB4\x53\xEC\x69\x1C\xA2\x7E\xC4\x8F\xE4\x1B\x70\xAD\xF2\xA2\xF9\xFB\xF7\x16\x64\x66\x69\x9F\x49\x51\xA2\xE2\x15\x18\x67\x06\x4A\x7F\xD5\x6C\xB5\x4D\xB3\x33\xE0\x61\xEB\x5D\xBE\xE9\x98\x0F\x32\xD7\x1D\x4B\x3C\x2E\x5A\x01\x52\x91\x09\xF2\xDF\xEA\x8D\xD8\x06\x40\x63\xAA\x11\xE4\xFE\xC3\x37\x9E\x14\x52\x3F\xF4\xE2\xCC\xF2\x61\x93\xD1\xFD\x67\x6B\xD7\x52\xAE\xBF\x68\xAB\x40\x43\xA0\x57\x35\x53\x78\xF0\x53\xF8\x61\x42\x07\x64\xC6\xD7\x6F\x9B\x4C\x38\x0D\x63\xAC\x62\xAF\x36\x8B\xA2\x73\x0A\x0D\xF5\x21\xBD\x74\xAA\x4D\xEA\x72\x03\x49\xDB\xC7\x5F\x1D\x62\x63\xC7\xFD\xDD\x91\xEC\x33\xEE\xF5\x6D\xB4\x6E\x30\x68\xDE\xC8\xD6\x26\xB0\x75\x5E\x7B\xB4\x07\x20\x98\xA1\x76\x32\xB8\x4D\x6C\x4F\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\xC9\x80\x77\xE0\x62\x92\x82\xF5\x46\x9C\xF3\xBA\xF7\x4C\xC3\xDE\xB8\xA3\xAD\x39\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x02\x01\x00\x53\x5F\x21\xF5\xBA\xB0\x3A\x52\x39\x2C\x92\xB0\x6C\x00\xC9\xEF\xCE\x20\xEF\x06\xF2\x96\x9E\xE9\xA4\x74\x7F\x7A\x16\xFC\xB7\xF5\xB6\xFB\x15\x1B\x3F\xAB\xA6\xC0\x72\x5D\x10\xB1\x71\xEE\xBC\x4F\xE3\xAD\xAC\x03\x6D\x2E\x71\x2E\xAF\xC4\xE3\xAD\xA3\xBD\x0C\x11\xA7\xB4\xFF\x4A\xB2\x7B\x10\x10\x1F\xA7\x57\x41\xB2\xC0\xAE\xF4\x2C\x59\xD6\x47\x10\x88\xF3\x21\x51\x29\x30\xCA\x60\x86\xAF\x46\xAB\x1D\xED\x3A\x5B\xB0\x94\xDE\x44\xE3\x41\x08\xA2\xC1\xEC\x1D\xD6\xFD\x4F\xB6\xD6\x47\xD0\x14\x0B\xCA\xE6\xCA\xB5\x7B\x77\x7E\x41\x1F\x5E\x83\xC7\xB6\x8C\x39\x96\xB0\x3F\x96\x81\x41\x6F\x60\x90\xE2\xE8\xF9\xFB\x22\x71\xD9\x7D\xB3\x3D\x46\xBF\xB4\x84\xAF\x90\x1C\x0F\x8F\x12\x6A\xAF\xEF\xEE\x1E\x7A\xAE\x02\x4A\x8A\x17\x2B\x76\xFE\xAC\x54\x89\x24\x2C\x4F\x3F\xB6\xB2\xA7\x4E\x8C\xA8\x91\x97\xFB\x29\xC6\x7B\x5C\x2D\xB9\xCB\x66\xB6\xB7\xA8\x5B\x12\x51\x85\xB5\x09\x7E\x62\x78\x70\xFE\xA9\x6A\x60\xB6\x1D\x0E\x79\x0C\xFD\xCA\xEA\x24\x80\x72\xC3\x97\x3F\xF2\x77\xAB\x43\x22\x0A\xC7\xEB\xB6\x0C\x84\x82\x2C\x80\x6B\x41\x8A\x08\xC0\xEB\xA5\x6B\xDF\x99\x12\xCB\x8A\xD5\x5E\x80\x0C\x91\xE0\x26\x08\x36\x48\xC5\xFA\x38\x11\x35\xFF\x25\x83\x2D\xF2\x7A\xBF\xDA\xFD\x8E\xFE\xA5\xCB\x45\x2C\x1F\xC4\x88\x53\xAE\x77\x0E\xD9\x9A\x76\xC5\x8E\x2C\x1D\xA3\xBA\xD5\xEC\x32\xAE\xC0\xAA\xAC\xF7\xD1\x7A\x4D\xEB\xD4\x07\xE2\x48\xF7\x22\x8E\xB0\xA4\x9F\x6A\xCE\x8E\xB2\xB2\x60\xF4\xA3\x22\xD0\x23\xEB\x94\x5A\x7A\x69\xDD\x0F\xBF\x40\x57\xAC\x6B\x59\x50\xD9\xA3\x99\xE1\x6E\xFE\x8D\x01\x79\x27\x23\x15\xDE\x92\x9D\x7B\x09\x4D\x5A\xE7\x4B\x48\x30\x5A\x18\xE6\x0A\x6D\xE6\x8F\xE0\xD2\xBB\xE6\xDF\x7C\x6E\x21\x82\xC1\x68\x39\x4D\xB4\x98\x58\x66\x62\xCC\x4A\x90\x5E\xC3\xFA\x27\x04\xB1\x79\x15\x74\x99\xCC\xBE\xAD\x20\xDE\x26\x60\x1C\xEB\x56\x51\xA6\xA3\xEA\xE4\xA3\x3F\xA7\xFF\x61\xDC\xF1\x5A\x4D\x6C\x32\x23\x43\xEE\xAC\xA8\xEE\xEE\x4A\x12\x09\x3C\x5D\x71\xC2\xBE\x79\xFA\xC2\x87\x68\x1D\x0B\xFD\x5C\x69\xCC\x06\xD0\x9A\x7D\x54\x99\x2A\xC9\x39\x1A\x19\xAF\x4B\x2A\x43\xF3\x63\x5D\x5A\x58\xE2\x2F\xE3\x1D\xE4\xA9\xD6\xD0\x0A\xD0\x9E\xBF\xD7\x81\x09\xF1\xC9\xC7\x26\x0D\xAC\x98\x16\x56\xA0",
+	["CN=Buypass Class 3 Root CA,O=Buypass AS-983163327,C=NO"] = "\x30\x82\x05\x59\x30\x82\x03\x41\xA0\x03\x02\x01\x02\x02\x01\x02\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x30\x4E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4E\x4F\x31\x1D\x30\x1B\x06\x03\x55\x04\x0A\x0C\x14\x42\x75\x79\x70\x61\x73\x73\x20\x41\x53\x2D\x39\x38\x33\x31\x36\x33\x33\x32\x37\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x0C\x17\x42\x75\x79\x70\x61\x73\x73\x20\x43\x6C\x61\x73\x73\x20\x33\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x1E\x17\x0D\x31\x30\x31\x30\x32\x36\x30\x38\x32\x38\x35\x38\x5A\x17\x0D\x34\x30\x31\x30\x32\x36\x30\x38\x32\x38\x35\x38\x5A\x30\x4E\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x4E\x4F\x31\x1D\x30\x1B\x06\x03\x55\x04\x0A\x0C\x14\x42\x75\x79\x70\x61\x73\x73\x20\x41\x53\x2D\x39\x38\x33\x31\x36\x33\x33\x32\x37\x31\x20\x30\x1E\x06\x03\x55\x04\x03\x0C\x17\x42\x75\x79\x70\x61\x73\x73\x20\x43\x6C\x61\x73\x73\x20\x33\x20\x52\x6F\x6F\x74\x20\x43\x41\x30\x82\x02\x22\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x82\x02\x0F\x00\x30\x82\x02\x0A\x02\x82\x02\x01\x00\xA5\xDA\x0A\x95\x16\x50\xE3\x95\xF2\x5E\x9D\x76\x31\x06\x32\x7A\x9B\xF1\x10\x76\xB8\x00\x9A\xB5\x52\x36\xCD\x24\x47\xB0\x9F\x18\x64\xBC\x9A\xF6\xFA\xD5\x79\xD8\x90\x62\x4C\x22\x2F\xDE\x38\x3D\xD6\xE0\xA8\xE9\x1C\x2C\xDB\x78\x11\xE9\x8E\x68\x51\x15\x72\xC7\xF3\x33\x87\xE4\xA0\x5D\x0B\x5C\xE0\x57\x07\x2A\x30\xF5\xCD\xC4\x37\x77\x28\x4D\x18\x91\xE6\xBF\xD5\x52\xFD\x71\x2D\x70\x3E\xE7\xC6\xC4\x8A\xE3\xF0\x28\x0B\xF4\x76\x98\xA1\x8B\x87\x55\xB2\x3A\x13\xFC\xB7\x3E\x27\x37\x8E\x22\xE3\xA8\x4F\x2A\xEF\x60\xBB\x3D\xB7\x39\xC3\x0E\x01\x47\x99\x5D\x12\x4F\xDB\x43\xFA\x57\xA1\xED\xF9\x9D\xBE\x11\x47\x26\x5B\x13\x98\xAB\x5D\x16\x8A\xB0\x37\x1C\x57\x9D\x45\xFF\x88\x96\x36\xBF\xBB\xCA\x07\x7B\x6F\x87\x63\xD7\xD0\x32\x6A\xD6\x5D\x6C\x0C\xF1\xB3\x6E\x39\xE2\x6B\x31\x2E\x39\x00\x27\x14\xDE\x38\xC0\xEC\x19\x66\x86\x12\xE8\x9D\x72\x16\x13\x64\x52\xC7\xA9\x37\x1C\xFD\x82\x30\xED\x84\x18\x1D\xF4\xAE\x5C\xFF\x70\x13\x00\xEB\xB1\xF5\x33\x7A\x4B\xD6\x55\xF8\x05\x8D\x4B\x69\xB0\xF5\xB3\x28\x36\x5C\x14\xC4\x51\x73\x4D\x6B\x0B\xF1\x34\x07\xDB\x17\x39\xD7\xDC\x28\x7B\x6B\xF5\x9F\xF3\x2E\xC1\x4F\x17\x2A\x10\xF3\xCC\xCA\xE8\xEB\xFD\x6B\xAB\x2E\x9A\x9F\x2D\x82\x6E\x04\xD4\x52\x01\x93\x2D\x3D\x86\xFC\x7E\xFC\xDF\xEF\x42\x1D\xA6\x6B\xEF\xB9\x20\xC6\xF7\xBD\xA0\xA7\x95\xFD\xA7\xE6\x89\x24\xD8\xCC\x8C\x34\x6C\xE2\x23\x2F\xD9\x12\x1A\x21\xB9\x55\x91\x6F\x0B\x91\x79\x19\x0C\xAD\x40\x88\x0B\x70\xE2\x7A\xD2\x0E\xD8\x68\x48\xBB\x82\x13\x39\x10\x58\xE9\xD8\x2A\x07\xC6\x12\xDB\x58\xDB\xD2\x3B\x55\x10\x47\x05\x15\x67\x62\x7E\x18\x63\xA6\x46\x3F\x09\x0E\x54\x32\x5E\xBF\x0D\x62\x7A\x27\xEF\x80\xE8\xDB\xD9\x4B\x06\x5A\x37\x5A\x25\xD0\x08\x12\x77\xD4\x6F\x09\x50\x97\x3D\xC8\x1D\xC3\xDF\x8C\x45\x30\x56\xC6\xD3\x64\xAB\x66\xF3\xC0\x5E\x96\x9C\xC3\xC4\xEF\xC3\x7C\x6B\x8B\x3A\x79\x7F\xB3\x49\xCF\x3D\xE2\x89\x9F\xA0\x30\x4B\x85\xB9\x9C\x94\x24\x79\x8F\x7D\x6B\xA9\x45\x68\x0F\x2B\xD0\xF1\xDA\x1C\xCB\x69\xB8\xCA\x49\x62\x6D\xC8\xD0\x63\x62\xDD\x60\x0F\x58\xAA\x8F\xA1\xBC\x05\xA5\x66\xA2\xCF\x1B\x76\xB2\x84\x64\xB1\x4C\x39\x52\xC0\x30\xBA\xF0\x8C\x4B\x02\xB0\xB6\xB7\x02\x03\x01\x00\x01\xA3\x42\x30\x40\x30\x0F\x06\x03\x55\x1D\x13\x01\x01\xFF\x04\x05\x30\x03\x01\x01\xFF\x30\x1D\x06\x03\x55\x1D\x0E\x04\x16\x04\x14\x47\xB8\xCD\xFF\xE5\x6F\xEE\xF8\xB2\xEC\x2F\x4E\x0E\xF9\x25\xB0\x8E\x3C\x6B\xC3\x30\x0E\x06\x03\x55\x1D\x0F\x01\x01\xFF\x04\x04\x03\x02\x01\x06\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x0B\x05\x00\x03\x82\x02\x01\x00\x00\x20\x23\x41\x35\x04\x90\xC2\x40\x62\x60\xEF\xE2\x35\x4C\xD7\x3F\xAC\xE2\x34\x90\xB8\xA1\x6F\x76\xFA\x16\x16\xA4\x48\x37\x2C\xE9\x90\xC2\xF2\x3C\xF8\x0A\x9F\xD8\x81\xE5\xBB\x5B\xDA\x25\x2C\xA4\xA7\x55\x71\x24\x32\xF6\xC8\x0B\xF2\xBC\x6A\xF8\x93\xAC\xB2\x07\xC2\x5F\x9F\xDB\xCC\xC8\x8A\xAA\xBE\x6A\x6F\xE1\x49\x10\xCC\x31\xD7\x80\xBB\xBB\xC8\xD8\xA2\x0E\x64\x57\xEA\xA2\xF5\xC2\xA9\x31\x15\xD2\x20\x6A\xEC\xFC\x22\x01\x28\xCF\x86\xB8\x80\x1E\xA9\xCC\x11\xA5\x3C\xF2\x16\xB3\x47\x9D\xFC\xD2\x80\x21\xC4\xCB\xD0\x47\x70\x41\xA1\xCA\x83\x19\x08\x2C\x6D\xF2\x5D\x77\x9C\x8A\x14\x13\xD4\x36\x1C\x92\xF0\xE5\x06\x37\xDC\xA6\xE6\x90\x9B\x38\x8F\x5C\x6B\x1B\x46\x86\x43\x42\x5F\x3E\x01\x07\x53\x54\x5D\x65\x7D\xF7\x8A\x73\xA1\x9A\x54\x5A\x1F\x29\x43\x14\x27\xC2\x85\x0F\xB5\x88\x7B\x1A\x3B\x94\xB7\x1D\x60\xA7\xB5\x9C\xE7\x29\x69\x57\x5A\x9B\x93\x7A\x43\x30\x1B\x03\xD7\x62\xC8\x40\xA6\xAA\xFC\x64\xE4\x4A\xD7\x91\x53\x01\xA8\x20\x88\x6E\x9C\x5F\x44\xB9\xCB\x60\x81\x34\xEC\x6F\xD3\x7D\xDA\x48\x5F\xEB\xB4\x90\xBC\x2D\xA9\x1C\x0B\xAC\x1C\xD5\xA2\x68\x20\x80\x04\xD6\xFC\xB1\x8F\x2F\xBB\x4A\x31\x0D\x4A\x86\x1C\xEB\xE2\x36\x29\x26\xF5\xDA\xD8\xC4\xF2\x75\x61\xCF\x7E\xAE\x76\x63\x4A\x7A\x40\x65\x93\x87\xF8\x1E\x80\x8C\x86\xE5\x86\xD6\x8F\x0E\xFC\x53\x2C\x60\xE8\x16\x61\x1A\xA2\x3E\x43\x7B\xCD\x39\x60\x54\x6A\xF5\xF2\x89\x26\x01\x68\x83\x48\xA2\x33\xE8\xC9\x04\x91\xB2\x11\x34\x11\x3E\xEA\xD0\x43\x19\x1F\x03\x93\x90\x0C\xFF\x51\x3D\x57\xF4\x41\x6E\xE1\xCB\xA0\xBE\xEB\xC9\x63\xCD\x6D\xCC\xE4\xF8\x36\xAA\x68\x9D\xED\xBD\x5D\x97\x70\x44\x0D\xB6\x0E\x35\xDC\xE1\x0C\x5D\xBB\xA0\x51\x94\xCB\x7E\x16\xEB\x11\x2F\xA3\x92\x45\xC8\x4C\x71\xD9\xBC\xC9\x99\x52\x57\x46\x2F\x50\xCF\xBD\x35\x69\xF4\x3D\x15\xCE\x06\xA5\x2C\x0F\x3E\xF6\x81\xBA\x94\xBB\xC3\xBB\xBF\x65\x78\xD2\x86\x79\xFF\x49\x3B\x1A\x83\x0C\xF0\xDE\x78\xEC\xC8\xF2\x4D\x4C\x1A\xDE\x82\x29\xF8\xC1\x5A\xDA\xED\xEE\xE6\x27\x5E\xE8\x45\xD0\x9D\x1C\x51\xA8\x68\xAB\x44\xE3\xD0\x8B\x6A\xE3\xF8\x3B\xBB\xDC\x4D\xD7\x64\xF2\x51\xBE\xE6\xAA\xAB\x5A\xE9\x31\xEE\x06\xBC\x73\xBF\x13\x62\x0A\x9F\xC7\xB9\x97",
 };

From 750e1ddf69d9f3375801615e872ec42b8a8d5a6d Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 16 Jul 2012 15:51:47 -0400
Subject: [PATCH 038/238] Fixed a couple of init-time mem leaks.

---
 src/DPM.cc   |  1 +
 src/Scope.cc | 11 +++++------
 src/scan.l   |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/DPM.cc b/src/DPM.cc
index d7e5cd25ef..6ecf3b1336 100644
--- a/src/DPM.cc
+++ b/src/DPM.cc
@@ -117,6 +117,7 @@ void DPM::AddConfig(const Analyzer::Config& cfg)
 				desc.SP();
 #endif
 				}
+			Unref(plist);
 			}
 		}
 
diff --git a/src/Scope.cc b/src/Scope.cc
index 4916cdbfce..731ced93b7 100644
--- a/src/Scope.cc
+++ b/src/Scope.cc
@@ -50,7 +50,7 @@ Scope::~Scope()
 
 ID* Scope::GenerateTemporary(const char* name)
 	{
-	return new ID(copy_string(name), SCOPE_FUNCTION, false);
+	return new ID(name, SCOPE_FUNCTION, false);
 	}
 
 id_list* Scope::GetInits()
@@ -166,16 +166,15 @@ ID* install_ID(const char* name, const char* module_name,
 	else
 		scope = SCOPE_FUNCTION;
 
-	string full_name_str = make_full_var_name(module_name, name);
-	char* full_name = copy_string(full_name_str.c_str());
+	string full_name = make_full_var_name(module_name, name);
 
-	ID* id = new ID(full_name, scope, is_export);
+	ID* id = new ID(full_name.c_str(), scope, is_export);
 	if ( SCOPE_FUNCTION != scope )
-		global_scope()->Insert(full_name, id);
+		global_scope()->Insert(full_name.c_str(), id);
 	else
 		{
 		id->SetOffset(top_scope->Length());
-		top_scope->Insert(full_name, id);
+		top_scope->Insert(full_name.c_str(), id);
 		}
 
 	return id;
diff --git a/src/scan.l b/src/scan.l
index 645ce659cd..d90501dd55 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -776,7 +776,7 @@ void add_input_file(const char* file)
 	if ( ! filename )
 		(void) load_files(file);
 	else
-		input_files.append(copy_string(file));
+		input_files.append((char*) file);
 	}
 
 void add_to_name_list(char* s, char delim, name_list& nl)

From 5d04d583854efd592495d6678895a042cf03e698 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 17 Jul 2012 13:57:23 -0400
Subject: [PATCH 039/238] Fixed small elasticsearch problem in configure
 output.

---
 CMakeLists.txt | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 14cf66ac19..bd6bf95737 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -122,13 +122,15 @@ if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
 	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
 endif()
 
+set(USE_ELASTICSEARCH false)
 set(USE_CURL false)
 find_package(CURL)
+
 if (CURL_FOUND)
+  set(USE_ELASTICSEARCH true)
   set(USE_CURL true)
   include_directories(BEFORE ${CURL_INCLUDE_DIR})
   list(APPEND OPTLIBS ${CURL_LIBRARIES})
-  set(USE_ELASTICSEARCH true)
 endif()
 
 if (ENABLE_PERFTOOLS_DEBUG)

From 81edec8b2eeef682c4bb2639a0b191e12bc2f561 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 17 Jul 2012 14:16:15 -0700
Subject: [PATCH 040/238] Fix bug, where in dns.log rcode always was set to
 0/NOERROR when no reply package was seen.

In the fixed version rcode is only set when a reply packet was seen.

Updates for the baseline have been commited separately in the
topic/bernhard/dns-fix branch.
---
 scripts/base/protocols/dns/main.bro | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index 600de4beaf..c951ff4fd2 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -162,11 +162,11 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
 
 	c$dns = c$dns_state$pending[msg$id];
 
-	c$dns$rcode = msg$rcode;
-	c$dns$rcode_name = base_errors[msg$rcode];
-
 	if ( ! is_query )
 		{
+		c$dns$rcode = msg$rcode;
+		c$dns$rcode_name = base_errors[msg$rcode];
+
 		if ( ! c$dns?$total_answers )
 			c$dns$total_answers = msg$num_answers;
 

From 1fa182c16918d258cbda6bfc69b3394103d4313f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 18 Jul 2012 00:00:31 -0400
Subject: [PATCH 041/238] Some better elasticsearch reliability.

 - Added a configurable option for timing out ES HTTP requests.

 - Stop sending reporter messages after one message for one failure.
---
 .../logging/writers/elasticsearch.bro         |  3 ++
 src/logging.bif                               |  1 +
 src/logging/writers/ElasticSearch.cc          | 38 ++++++++++++++-----
 src/logging/writers/ElasticSearch.h           |  2 +
 4 files changed, 35 insertions(+), 9 deletions(-)

diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index 93c6c98705..adc675e487 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -17,6 +17,9 @@ export {
 	## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.
 	const type_prefix = "" &redef;
 
+	## The time before an ElasticSearch transfer will timeout.
+	const transfer_timeout = 2secs;
+
 	## The batch size is the number of messages that will be queued up before 
 	## they are sent to be bulk indexed.
 	## Note: this is mainly a memory usage parameter.
diff --git a/src/logging.bif b/src/logging.bif
index 3cdb414d80..7e50a9d285 100644
--- a/src/logging.bif
+++ b/src/logging.bif
@@ -91,6 +91,7 @@ const server_host: string;
 const server_port: count;
 const index_prefix: string;
 const type_prefix: string;
+const transfer_timeout: interval;
 const max_batch_size: count;
 const max_batch_interval: interval;
 const max_byte_size: count;
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 1b8dfa495d..71be036a72 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -42,7 +42,10 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	current_index = string();
 	prev_index = string();
 	last_send = current_time();
+	failing = false;
 	
+	transfer_timeout = BifConst::LogElasticSearch::transfer_timeout * 1000;
+
 	curl_handle = HTTPSetup();
 }
 
@@ -77,12 +80,13 @@ bool ElasticSearch::BatchIndex()
 	curl_easy_setopt(curl_handle, CURLOPT_POST, 1);
 	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)buffer.Len());
 	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
-	HTTPSend(curl_handle);
-	
+	failing = ! HTTPSend(curl_handle);
+
+	// We are currently throwing the data out regardless of if the send failed.  Fire and forget!
 	buffer.Clear();
 	counter = 0;
 	last_send = current_time();
-	
+
 	return true;
 	}
 
@@ -347,6 +351,8 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed. 
 	// The best (only?) way to disable that is to just use HTTP 1.0
 	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+
+	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
 	
 	CURLcode return_code = curl_easy_perform(handle);
 	
@@ -355,21 +361,35 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 		case CURLE_COULDNT_CONNECT:
 		case CURLE_COULDNT_RESOLVE_HOST:
 		case CURLE_WRITE_ERROR:
-			return false;
+		case CURLE_RECV_ERROR:
+			{
+			if ( ! failing )
+				Error(Fmt("ElasticSearch server may not be accessible."));
+			}
+
+		case CURLE_OPERATION_TIMEDOUT:
+			{
+			if ( ! failing )
+				Warning(Fmt("HTTP operation with elasticsearch server timed out at %" PRIu64 " msecs.", transfer_timeout));
+			}
 		
 		case CURLE_OK:
 			{
 			uint http_code = 0;
 			curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
-			if ( http_code != 200 )
-				Error(Fmt("Received a non-successful status code back from ElasticSearch server."));
-			
-			return true;
+			if ( http_code == 200 )
+				// Hopefully everything goes through here.
+				return true;
+			else if ( ! failing )
+				Error(Fmt("Received a non-successful status code back from ElasticSearch server, check the elasticsearch server log."));
 			}
 		
 		default:
-			return true;
+			{
+			}
 		}
+		// The "successful" return happens above
+		return false;
 	}
 
 #endif
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index 375845b002..60977f7737 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -65,6 +65,8 @@ private:
 	
 	string path;
 	string index_prefix;
+	uint64 transfer_timeout;
+	bool failing;
 	
 	uint64 batch_size;
 };

From 50f5f8131df7691643209ccf2d058ab98a4ba6ad Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 18 Jul 2012 07:29:01 -0700
Subject: [PATCH 042/238] Revert "Fixed a couple of init-time mem leaks."

This reverts commit 750e1ddf69d9f3375801615e872ec42b8a8d5a6d.
---
 src/DPM.cc   |  1 -
 src/Scope.cc | 11 ++++++-----
 src/scan.l   |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/DPM.cc b/src/DPM.cc
index 6ecf3b1336..d7e5cd25ef 100644
--- a/src/DPM.cc
+++ b/src/DPM.cc
@@ -117,7 +117,6 @@ void DPM::AddConfig(const Analyzer::Config& cfg)
 				desc.SP();
 #endif
 				}
-			Unref(plist);
 			}
 		}
 
diff --git a/src/Scope.cc b/src/Scope.cc
index 731ced93b7..4916cdbfce 100644
--- a/src/Scope.cc
+++ b/src/Scope.cc
@@ -50,7 +50,7 @@ Scope::~Scope()
 
 ID* Scope::GenerateTemporary(const char* name)
 	{
-	return new ID(name, SCOPE_FUNCTION, false);
+	return new ID(copy_string(name), SCOPE_FUNCTION, false);
 	}
 
 id_list* Scope::GetInits()
@@ -166,15 +166,16 @@ ID* install_ID(const char* name, const char* module_name,
 	else
 		scope = SCOPE_FUNCTION;
 
-	string full_name = make_full_var_name(module_name, name);
+	string full_name_str = make_full_var_name(module_name, name);
+	char* full_name = copy_string(full_name_str.c_str());
 
-	ID* id = new ID(full_name.c_str(), scope, is_export);
+	ID* id = new ID(full_name, scope, is_export);
 	if ( SCOPE_FUNCTION != scope )
-		global_scope()->Insert(full_name.c_str(), id);
+		global_scope()->Insert(full_name, id);
 	else
 		{
 		id->SetOffset(top_scope->Length());
-		top_scope->Insert(full_name.c_str(), id);
+		top_scope->Insert(full_name, id);
 		}
 
 	return id;
diff --git a/src/scan.l b/src/scan.l
index d90501dd55..645ce659cd 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -776,7 +776,7 @@ void add_input_file(const char* file)
 	if ( ! filename )
 		(void) load_files(file);
 	else
-		input_files.append((char*) file);
+		input_files.append(copy_string(file));
 	}
 
 void add_to_name_list(char* s, char delim, name_list& nl)

From 43507b1bb9b2ff484716e2e8f151a5fdc8974951 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 18 Jul 2012 11:28:41 -0400
Subject: [PATCH 043/238] New script for easily duplicating logs to
 ElasticSearch.

---
 .../policy/tuning/logs-to-elasticsearch.bro   | 45 +++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 scripts/policy/tuning/logs-to-elasticsearch.bro

diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro
new file mode 100644
index 0000000000..c3cc9d5002
--- /dev/null
+++ b/scripts/policy/tuning/logs-to-elasticsearch.bro
@@ -0,0 +1,45 @@
+##! Load this script to enable global log output to an ElasticSearch database.
+
+module LogElasticSearch;
+
+export {
+	## An elasticsearch specific rotation interval.
+	const rotation_interval = 24hr &redef;
+
+	## Optionally ignore any :bro:enum:`Log::ID` from being sent to
+	## ElasticSearch with this script.
+	const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
+
+	## If you want to explicitly only send certain :bro:enum:`Log::ID` 
+	## streams, add them to this set.  If the set remains empty, all will 
+	## be sent.  The :bro:id:`excluded_log_ids` option will remain in 
+	## effect as well.
+	const send_logs: set[string] = set() &redef;
+}
+
+module Log;
+
+event bro_init() &priority=-5
+	{
+	local my_filters: table[ID, string] of Filter = table();
+
+	for ( [id, name] in filters )
+		{
+		local filter = filters[id, name];
+		if ( fmt("%s", id) in LogElasticSearch::excluded_log_ids ||
+		     (|LogElasticSearch::send_logs| > 0 && fmt("%s", id) !in LogElasticSearch::send_logs) )
+			next;
+
+		filter$name = cat(name, "-es");
+		filter$writer = Log::WRITER_ELASTICSEARCH;
+		filter$interv = LogElasticSearch::rotation_interval;
+		my_filters[id, name] = filter;
+		}
+
+	# This had to be done separately to avoid an ever growing filters list
+	# where the for loop would never end.
+	for ( [id, name] in my_filters )
+		{
+		Log::add_filter(id, filter);
+		}
+	}
\ No newline at end of file

From 6335dbb5e1cf694afea3c306012a258614d13880 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 18 Jul 2012 11:32:14 -0400
Subject: [PATCH 044/238] Fixing calc_next_rotate to use UTC based time
 functions.

---
 src/util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util.cc b/src/util.cc
index 3cfa5fca1c..abbea3e906 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1112,9 +1112,9 @@ double calc_next_rotate(double current, double interval, double base)
 	time_t teatime = time_t(current);
 
 	struct tm t;
-	t = *localtime_r(&teatime, &t);
+	t = *gmtime_r(&teatime, &t);
 	t.tm_hour = t.tm_min = t.tm_sec = 0;
-	double startofday = mktime(&t);
+	double startofday = timegm(&t);
 
 	if ( base < 0 )
 		// No base time given. To get nice timestamps, we round

From 18268273594900cdeabc811d1a9cf6562caf2687 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 18 Jul 2012 15:42:23 -0400
Subject: [PATCH 045/238] Changed ES index names to localtime and added a meta
 index.

---
 src/logging/writers/ElasticSearch.cc | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 71be036a72..2095ed62df 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -263,11 +263,28 @@ bool ElasticSearch::UpdateIndex(double now, double rinterval, double rbase)
 		struct tm tm;
 		char buf[128];
 		time_t teatime = (time_t)interval_beginning;
-		gmtime_r(&teatime, &tm);
+		localtime_r(&teatime, &tm);
 		strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm);
 		
 		prev_index = current_index;
 		current_index = index_prefix + "-" + buf;
+
+		// Send some metadata about this index.
+		buffer.AddRaw("{\"index\":{\"_index\":\"@", 21);
+		buffer.Add(index_prefix);
+		buffer.AddRaw("-meta\",\"_type\":\"index\",\"_id\":\"", 30);
+		buffer.Add(current_index);
+		buffer.AddRaw("-", 1);
+		buffer.Add(Info().rotation_base);
+		buffer.AddRaw("-", 1);
+		buffer.Add(Info().rotation_interval);
+		buffer.AddRaw("\"}}\n{\"name\":\"", 13);
+		buffer.Add(current_index);
+		buffer.AddRaw("\",\"start\":", 10);
+		buffer.Add(interval_beginning);
+		buffer.AddRaw(",\"end\":", 7);
+		buffer.Add(interval_beginning+rinterval);
+		buffer.AddRaw("}\n", 2);
 		}
 	
 	//printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str());

From 0c4c5ff33571c9f5cec67d432dd401fc1770e0d4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 19 Jul 2012 12:14:13 -0400
Subject: [PATCH 046/238] Temporarily removing the ES timeout because it works
 with signals and is incompatible with Bro threads.

---
 src/logging/writers/ElasticSearch.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 2095ed62df..1ae81dfde8 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -369,7 +369,7 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 	// The best (only?) way to disable that is to just use HTTP 1.0
 	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 
-	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
+	//curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
 	
 	CURLcode return_code = curl_easy_perform(handle);
 	

From f73eb3b086c1ae88c122434613501af950a9dba0 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 12 Jul 2012 13:44:24 -0700
Subject: [PATCH 047/238] Reworking thread termination logic.

Turns out the finish methods weren't called correctly, caused by a
mess up with method names which all sounded too similar and the wrong
one ended up being called. I've reworked this by changing the
thread/writer/reader interfaces, which actually also simplifies them
by getting rid of the requirement for writer backends to call their
parent methods (i.e., less opportunity for errors).

This commit also includes the following (because I noticed the problem
above when working on some of these):

     - The ASCII log writer now includes "#start <timestamp>" and
      "#end <timestamp> lines in the each file. The latter supersedes
      Bernhard's "EOF" patch.

      This required a number of tests updates. The standard canonifier
      removes the timestamps, but some tests compare files directly,
      which doesn't work if they aren't printing out the same
      timestamps (like the comm tests).

     - The above required yet another change to the writer API to
       network_time to methods.

     - Renamed ASCII logger "header" options to "meta".

     - Fixes #763 "Escape # when first character in log file line".

All btests pass for me on Linux FC15. Will try MacOS next.
---
 NEWS                                          |  5 +
 .../base/frameworks/logging/writers/ascii.bro | 11 ++-
 src/input/Manager.cc                          |  2 -
 src/input/ReaderBackend.cc                    |  9 +-
 src/input/ReaderBackend.h                     | 25 ++---
 src/input/ReaderFrontend.cc                   | 26 -----
 src/input/readers/Ascii.cc                    |  2 -
 src/input/readers/Benchmark.cc                |  1 -
 src/input/readers/Raw.cc                      |  2 -
 src/logging.bif                               |  4 +-
 src/logging/Manager.cc                        |  3 +-
 src/logging/WriterBackend.cc                  | 16 +--
 src/logging/WriterBackend.h                   | 32 ++++--
 src/logging/WriterFrontend.cc                 | 33 ++-----
 src/logging/WriterFrontend.h                  |  8 +-
 src/logging/writers/Ascii.cc                  | 97 ++++++++++++++-----
 src/logging/writers/Ascii.h                   | 13 ++-
 src/logging/writers/DataSeries.cc             | 14 ++-
 src/logging/writers/DataSeries.h              |  5 +-
 src/logging/writers/None.h                    |  5 +-
 src/threading/BasicThread.cc                  | 35 ++-----
 src/threading/BasicThread.h                   |  8 ++
 src/threading/MsgThread.cc                    | 31 ++++--
 src/threading/MsgThread.h                     | 34 ++++---
 .../ssh-filtered.log                          | 12 +++
 .../ssh.log                                   | 12 ---
 .../test.log                                  | 12 +++
 testing/btest/core/expr-exception.bro         |  2 +-
 testing/btest/istate/events-ssl.bro           |  9 +-
 testing/btest/istate/events.bro               |  9 +-
 .../base/frameworks/logging/ascii-empty.bro   |  5 +-
 .../logging/ascii-line-like-comment.bro       | 23 +++++
 .../base/frameworks/logging/ascii-options.bro |  2 +-
 .../base/frameworks/logging/remote-types.bro  |  8 +-
 .../base/frameworks/logging/remote.bro        |  8 +-
 .../notice/default-policy-order.test          |  6 +-
 testing/scripts/diff-remove-timestamps        |  7 +-
 37 files changed, 313 insertions(+), 223 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log
 delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
 create mode 100644 testing/btest/scripts/base/frameworks/logging/ascii-line-like-comment.bro

diff --git a/NEWS b/NEWS
index d9410e1c7c..0798920d8a 100644
--- a/NEWS
+++ b/NEWS
@@ -140,6 +140,11 @@ the full set.
   Bro now supports decapsulating tunnels directly for protocols it
   understands.
 
+- ASCII logs now record the time when they were opened/closed at the
+  beginning and end of the file, respectively. The options
+  LogAscii::header_prefix and LogAscii::include_header have been
+  renamed to LogAscii::meta_prefix and LogAscii::include_meta,
+  respectively.
 
 Bro 2.0
 -------
diff --git a/scripts/base/frameworks/logging/writers/ascii.bro b/scripts/base/frameworks/logging/writers/ascii.bro
index fa1fcd6797..bacb0996d0 100644
--- a/scripts/base/frameworks/logging/writers/ascii.bro
+++ b/scripts/base/frameworks/logging/writers/ascii.bro
@@ -8,12 +8,13 @@ export {
 	## into files. This is primarily for debugging purposes.
 	const output_to_stdout = F &redef;
 
-	## If true, include a header line with column names and description
-	## of the other ASCII logging options that were used.
-	const include_header = T &redef;
+	## If true, include lines with log meta information such as column names with
+	## types, the values of ASCII logging options that in use, and the time when the
+	## file was opened and closes (the latter at the end). 
+	const include_meta = T &redef;
 
-	## Prefix for the header line if included.
-	const header_prefix = "#" &redef;
+	## Prefix for lines with meta information.
+	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
 	const separator = "\t" &redef;
diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index fc68343813..1c6b69e8ec 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -726,8 +726,6 @@ bool Manager::RemoveStream(Stream *i)
 
 	i->removed = true;
 
-	i->reader->Close();
-
 	DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s",
 		i->name.c_str());
 
diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc
index dea554251e..84106a3c94 100644
--- a/src/input/ReaderBackend.cc
+++ b/src/input/ReaderBackend.cc
@@ -207,7 +207,7 @@ bool ReaderBackend::Init(const ReaderInfo& arg_info, const int arg_num_fields,
 	return success;
 	}
 
-void ReaderBackend::Close()
+bool ReaderBackend::OnFinish(double network_time)
 	{
 	DoClose();
 	disabled = true; // frontend disables itself when it gets the Close-message.
@@ -221,6 +221,8 @@ void ReaderBackend::Close()
 		delete [] (fields);
 		fields = 0;
 		}
+
+	return true;
 	}
 
 bool ReaderBackend::Update()
@@ -243,10 +245,9 @@ void ReaderBackend::DisableFrontend()
 	SendOut(new DisableMessage(frontend));
 	}
 
-bool ReaderBackend::DoHeartbeat(double network_time, double current_time)
+bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
 	{
-	MsgThread::DoHeartbeat(network_time, current_time);
-	return true;
+	return DoHeartbeat(network_time, current_time);
 	}
 
 TransportProto ReaderBackend::StringToProto(const string &proto)
diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h
index 820633254a..1e77a61f37 100644
--- a/src/input/ReaderBackend.h
+++ b/src/input/ReaderBackend.h
@@ -108,15 +108,6 @@ public:
 	 */
 	bool Init(const ReaderInfo& info, int num_fields, const threading::Field* const* fields);
 
-	/**
-	 * Finishes reading from this input stream in a regular fashion. Must
-	 * not be called if an error has been indicated earlier. After
-	 * calling this, no further reading from the stream can be performed.
-	 *
-	 * @return False if an error occured.
-	 */
-	void Close();
-
 	/**
 	 * Force trigger an update of the input stream. The action that will
 	 * be taken depends on the current read mode and the individual input
@@ -149,6 +140,9 @@ public:
 	 */
 	int NumFields() const	{ return num_fields; }
 
+	// Overridden from MsgThread.
+	virtual bool OnHeartbeat(double network_time, double current_time);
+	virtual bool OnFinish(double network_time);
 
 protected:
 	// Methods that have to be overwritten by the individual readers
@@ -200,6 +194,11 @@ protected:
 	 */
 	virtual bool DoUpdate() = 0;
 
+	/**
+	 * Triggered by regular heartbeat messages from the main thread.
+	 */
+	virtual bool DoHeartbeat(double network_time, double current_time) = 0;
+
 	/**
 	 * Method allowing a reader to send a specified Bro event. Vals must
 	 * match the values expected by the bro event.
@@ -271,14 +270,6 @@ protected:
 	 */
 	void EndCurrentSend();
 
-	/**
-	 * Triggered by regular heartbeat messages from the main thread.
-	 *
-	 * This method can be overridden but once must call
-	 * ReaderBackend::DoHeartbeat().
-	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
-
 	/**
 	 *  Convert a string into a TransportProto. This is just a utility
 	 *  function for Readers.
diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc
index e489147d36..7e4ef201b1 100644
--- a/src/input/ReaderFrontend.cc
+++ b/src/input/ReaderFrontend.cc
@@ -37,17 +37,6 @@ public:
 	virtual bool Process() { return Object()->Update(); }
 };
 
-class CloseMessage : public threading::InputMessage<ReaderBackend>
-{
-public:
-	CloseMessage(ReaderBackend* backend)
-		: threading::InputMessage<ReaderBackend>("Close", backend)
-		 { }
-
-	virtual bool Process() { Object()->Close(); return true; }
-};
-
-
 ReaderFrontend::ReaderFrontend(bro_int_t type)
 	{
 	disabled = initialized = false;
@@ -93,21 +82,6 @@ void ReaderFrontend::Update()
 	backend->SendIn(new UpdateMessage(backend));
 	}
 
-void ReaderFrontend::Close()
-	{
-	if ( disabled )
-		return;
-
-	if ( ! initialized )
-		{
-		reporter->Error("Tried to call finish on uninitialized reader");
-		return;
-		}
-
-	disabled = true;
-	backend->SendIn(new CloseMessage(backend));
-	}
-
 string ReaderFrontend::Name() const
 	{
 	if ( ! info.source.size() )
diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index dd1e742e5e..7f93a3138c 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -506,8 +506,6 @@ bool Ascii::DoUpdate()
 
 bool Ascii::DoHeartbeat(double network_time, double current_time)
 {
-	ReaderBackend::DoHeartbeat(network_time, current_time);
-
 	switch ( Info().mode  ) {
 		case MODE_MANUAL:
 			// yay, we do nothing :)
diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc
index d71901fa66..28afdc1c89 100644
--- a/src/input/readers/Benchmark.cc
+++ b/src/input/readers/Benchmark.cc
@@ -222,7 +222,6 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 
 bool Benchmark::DoHeartbeat(double network_time, double current_time)
 {
-	ReaderBackend::DoHeartbeat(network_time, current_time);
 	num_lines = (int) ( (double) num_lines*multiplication_factor);
 	num_lines += add;
 	heartbeatstarttime = CurrTime();
diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc
index 1bae6cfa0c..f62e966883 100644
--- a/src/input/readers/Raw.cc
+++ b/src/input/readers/Raw.cc
@@ -252,8 +252,6 @@ bool Raw::DoUpdate()
 
 bool Raw::DoHeartbeat(double network_time, double current_time)
 	{
-	ReaderBackend::DoHeartbeat(network_time, current_time);
-
 	switch ( Info().mode ) {
 		case MODE_MANUAL:
 			// yay, we do nothing :)
diff --git a/src/logging.bif b/src/logging.bif
index d25e89c33c..48e0edbb06 100644
--- a/src/logging.bif
+++ b/src/logging.bif
@@ -65,8 +65,8 @@ function Log::__flush%(id: Log::ID%): bool
 module LogAscii;
 
 const output_to_stdout: bool;
-const include_header: bool;
-const header_prefix: string;
+const include_meta: bool;
+const meta_prefix: string;
 const separator: string;
 const set_separator: string;
 const empty_field: string;
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 0fea3d577d..1808b83738 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -771,6 +771,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 			WriterBackend::WriterInfo info;
 			info.path = path;
+			info.network_time = network_time;
 
 			HashKey* k;
 			IterCookie* c = filter->config->AsTable()->InitForIteration();
@@ -1156,7 +1157,7 @@ bool Manager::Flush(EnumVal* id)
 
 	for ( Stream::WriterMap::iterator i = stream->writers.begin();
 	      i != stream->writers.end(); i++ )
-		i->second->writer->Flush();
+		i->second->writer->Flush(network_time);
 
 	RemoveDisabledWriters(stream);
 
diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index 00590208d5..68b0b506a1 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -18,7 +18,7 @@ namespace logging  {
 class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
 {
 public:
-        RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
+	RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
 				double open, double close, bool terminating)
 		: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
 		new_name(new_name), old_name(old_name), open(open),
@@ -260,9 +260,9 @@ bool WriterBackend::Rotate(string rotated_path, double open,
 	return true;
 	}
 
-bool WriterBackend::Flush()
+bool WriterBackend::Flush(double network_time)
 	{
-	if ( ! DoFlush() )
+	if ( ! DoFlush(network_time) )
 		{
 		DisableFrontend();
 		return false;
@@ -271,13 +271,15 @@ bool WriterBackend::Flush()
 	return true;
 	}
 
-bool WriterBackend::DoHeartbeat(double network_time, double current_time)
+bool WriterBackend::OnFinish(double network_time)
 	{
-	MsgThread::DoHeartbeat(network_time, current_time);
+	return DoFinish(network_time);
+	}
 
+bool WriterBackend::OnHeartbeat(double network_time, double current_time)
+	{
 	SendOut(new FlushWriteBufferMessage(frontend));
-
-	return true;
+	return DoHeartbeat(network_time, current_time);
 	}
 
 string WriterBackend::Render(const threading::Value::addr_t& addr) const
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index 6e65a8151a..33cde8679e 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -67,6 +67,11 @@ public:
 		 */
 		double rotation_base;
 
+		/**
+		 * The network time when the writer is created.
+		 */
+		double network_time;
+
 		/**
 		 * A map of key/value pairs corresponding to the relevant
 		 * filter's "config" table.
@@ -129,9 +134,11 @@ public:
 	 * Flushes any currently buffered output, assuming the writer
 	 * supports that. (If not, it will be ignored).
 	 *
+	 * @param network_time The network time when the flush was triggered.
+	 *
 	 * @return False if an error occured.
 	 */
-	bool Flush();
+	bool Flush(double network_time);
 
 	/**
 	 * Triggers rotation, if the writer supports that. (If not, it will
@@ -213,6 +220,10 @@ public:
 	  */
 	string Render(double d) const;
 
+	// Overridden from MsgThread.
+	virtual bool OnHeartbeat(double network_time, double current_time);
+	virtual bool OnFinish(double network_time);
+
 protected:
 	friend class FinishMessage;
 
@@ -272,8 +283,10 @@ protected:
 	 * will then be disabled and eventually deleted. When returning
 	 * false, an implementation should also call Error() to indicate what
 	 * happened.
+	 *
+	 * @param network_time The network time when the flush was triggered.
 	 */
-	virtual bool DoFlush() = 0;
+	virtual bool DoFlush(double network_time) = 0;
 
 	/**
 	 * Writer-specific method implementing log rotation.  Most directly
@@ -314,20 +327,19 @@ protected:
 
 	/**
 	 * Writer-specific method called just before the threading system is
-	 * going to shutdown.
+	 * going to shutdown. It is assumed that once this messages returns,
+	 * the thread can be safely terminated.
 	 *
-	 * This method can be overridden but one must call
-	 * WriterBackend::DoFinish().
+	 * @param network_time The network time when the finish is triggered.
 	 */
-	virtual bool DoFinish() { return MsgThread::DoFinish(); }
-
+	virtual bool DoFinish(double network_time) = 0;
 	/**
 	 * Triggered by regular heartbeat messages from the main thread.
 	 *
-	 * This method can be overridden but one must call
-	 * WriterBackend::DoHeartbeat().
+	 * This method can be overridden. Default implementation does
+	 * nothing.
 	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
+	virtual bool DoHeartbeat(double network_time, double current_time) = 0;
 
 private:
 	/**
diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc
index 21bde0d43c..577003926b 100644
--- a/src/logging/WriterFrontend.cc
+++ b/src/logging/WriterFrontend.cc
@@ -81,19 +81,13 @@ private:
 class FlushMessage : public threading::InputMessage<WriterBackend>
 {
 public:
-	FlushMessage(WriterBackend* backend)
-		: threading::InputMessage<WriterBackend>("Flush", backend)	{}
+	FlushMessage(WriterBackend* backend, double network_time)
+		: threading::InputMessage<WriterBackend>("Flush", backend),
+		network_time(network_time) {}
 
-	virtual bool Process() { return Object()->Flush(); }
-};
-
-class FinishMessage : public threading::InputMessage<WriterBackend>
-{
-public:
-	FinishMessage(WriterBackend* backend)
-		: threading::InputMessage<WriterBackend>("Finish", backend)	{}
-
-	virtual bool Process() { return Object()->DoFinish(); }
+	virtual bool Process() { return Object()->Flush(network_time); }
+private:
+	double network_time;
 };
 
 }
@@ -240,7 +234,7 @@ void WriterFrontend::SetBuf(bool enabled)
 		FlushWriteBuffer();
 	}
 
-void WriterFrontend::Flush()
+void WriterFrontend::Flush(double network_time)
 	{
 	if ( disabled )
 		return;
@@ -248,7 +242,7 @@ void WriterFrontend::Flush()
 	FlushWriteBuffer();
 
 	if ( backend )
-		backend->SendIn(new FlushMessage(backend));
+		backend->SendIn(new FlushMessage(backend, network_time));
 	}
 
 void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
@@ -266,17 +260,6 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool
 		log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating);
 	}
 
-void WriterFrontend::Finish()
-	{
-	if ( disabled )
-		return;
-
-	FlushWriteBuffer();
-
-	if ( backend )
-		backend->SendIn(new FinishMessage(backend));
-	}
-
 void WriterFrontend::DeleteVals(Value** vals)
 	{
 	// Note this code is duplicated in Manager::DeleteVals().
diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h
index 8a0dce4645..6581fb1c1b 100644
--- a/src/logging/WriterFrontend.h
+++ b/src/logging/WriterFrontend.h
@@ -114,8 +114,10 @@ public:
 	 * message back that will asynchronously call Disable().
 	 *
 	 * This method must only be called from the main thread.
+	 *
+	 * @param network_time The network time when the flush was triggered.
 	 */
-	void Flush();
+	void Flush(double network_time);
 
 	/**
 	 * Triggers log rotation.
@@ -138,8 +140,10 @@ public:
 	 * sends a message back that will asynchronously call Disable().
 	 *
 	 * This method must only be called from the main thread.
+	 *
+	 * @param network_time The network time when the finish was triggered.
 	 */
-	void Finish();
+	void Finish(double network_time);
 
 	/**
 	 * Explicitly triggers a transfer of all potentially buffered Write()
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 6e5ceef678..ab68cd77d8 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -18,7 +18,7 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	file = 0;
 
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
-	include_header = BifConst::LogAscii::include_header;
+	include_meta = BifConst::LogAscii::include_meta;
 
 	separator_len = BifConst::LogAscii::separator->Len();
 	separator = new char[separator_len];
@@ -40,10 +40,10 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(),
 	       unset_field_len);
 
-	header_prefix_len = BifConst::LogAscii::header_prefix->Len();
-	header_prefix = new char[header_prefix_len];
-	memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(),
-	       header_prefix_len);
+	meta_prefix_len = BifConst::LogAscii::meta_prefix->Len();
+	meta_prefix = new char[meta_prefix_len];
+	memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(),
+	       meta_prefix_len);
 
 	desc.EnableEscaping();
 	desc.AddEscapeSequence(separator, separator_len);
@@ -51,24 +51,39 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 
 Ascii::~Ascii()
 	{
+	// Normally, the file will be closed here already via the Finish()
+	// message. But when we terminate abnormally, we may still have it
+	// open.
 	if ( file )
-		fclose(file);
+		CloseFile(0);
 
 	delete [] separator;
 	delete [] set_separator;
 	delete [] empty_field;
 	delete [] unset_field;
-	delete [] header_prefix;
+	delete [] meta_prefix;
 	}
 
 bool Ascii::WriteHeaderField(const string& key, const string& val)
 	{
-	string str = string(header_prefix, header_prefix_len) +
+	string str = string(meta_prefix, meta_prefix_len) +
 		key + string(separator, separator_len) + val + "\n";
 
 	return (fwrite(str.c_str(), str.length(), 1, file) == 1);
 	}
 
+void Ascii::CloseFile(double t)
+	{
+	if ( ! file )
+		return;
+
+	if ( include_meta )
+		WriteHeaderField("end", t ? Timestamp(t) : "<abnormal termination>");
+
+	fclose(file);
+	file = 0;
+	}
+
 bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
 	{
 	string path = info.path;
@@ -81,17 +96,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 	if ( ! (file = fopen(fname.c_str(), "w")) )
 		{
 		Error(Fmt("cannot open %s: %s", fname.c_str(),
-			  strerror(errno)));
+			  Strerror(errno)));
 
 		return false;
 		}
 
-	if ( include_header )
+	if ( include_meta )
 		{
 		string names;
 		string types;
 
-		string str = string(header_prefix, header_prefix_len)
+		string str = string(meta_prefix, meta_prefix_len)
 			+ "separator " // Always use space as separator here.
 			+ get_escaped_string(string(separator, separator_len), false)
 			+ "\n";
@@ -105,8 +120,9 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 		            string(empty_field, empty_field_len), false)) &&
 		        WriteHeaderField("unset_field", get_escaped_string(
 		            string(unset_field, unset_field_len), false)) &&
-		        WriteHeaderField("path", get_escaped_string(path, false))) )
-			goto write_error;
+		        WriteHeaderField("path", get_escaped_string(path, false)) &&
+			WriteHeaderField("start", Timestamp(info.network_time))) )
+		goto write_error;
 
 		for ( int i = 0; i < num_fields; ++i )
 			{
@@ -128,21 +144,23 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 	return true;
 
 write_error:
-	Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
+	Error(Fmt("error writing to %s: %s", fname.c_str(), Strerror(errno)));
 	return false;
 	}
 
-bool Ascii::DoFlush()
+bool Ascii::DoFlush(double network_time)
 	{
 	fflush(file);
 	return true;
 	}
 
-bool Ascii::DoFinish()
+bool Ascii::DoFinish(double network_time)
 	{
-	return WriterBackend::DoFinish();
+	CloseFile(network_time);
+	return true;
 	}
 
+
 bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 	{
 	if ( ! val->present )
@@ -307,16 +325,33 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 
 	desc.AddRaw("\n", 1);
 
-	if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 )
+	const char* bytes = (const char*)desc.Bytes();
+	int len = desc.Len();
+
+	// Make sure the line doesn't look like meta information.
+	if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 )
 		{
-		Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
-		return false;
+		// It would so escape the first character.
+		char buf[16];
+		snprintf(buf, sizeof(buf), "\\x%02x", bytes[0]);
+		if ( fwrite(buf, strlen(buf), 1, file) != 1 )
+			goto write_error;
+
+		++bytes;
+		--len;
 		}
 
+	if ( fwrite(bytes, len, 1, file) != 1 )
+		goto write_error;
+
 	if ( IsBuf() )
 		fflush(file);
 
 	return true;
+
+write_error:
+	Error(Fmt("error writing to %s: %s", fname.c_str(), Strerror(errno)));
+	return false;
 	}
 
 bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
@@ -325,8 +360,7 @@ bool Ascii::DoRotate(string rotated_path, double open, double close, bool termin
 	if ( ! file || IsSpecial(Info().path) )
 		return true;
 
-	fclose(file);
-	file = 0;
+	CloseFile(close);
 
 	string nname = rotated_path + "." + LogExt();
 	rename(fname.c_str(), nname.c_str());
@@ -346,9 +380,28 @@ bool Ascii::DoSetBuf(bool enabled)
 	return true;
 	}
 
+bool Ascii::DoHeartbeat(double network_time, double current_time)
+	{
+	// Nothing to do.
+	return true;
+	}
+
 string Ascii::LogExt()
 	{
 	const char* ext = getenv("BRO_LOG_SUFFIX");
 	if ( ! ext ) ext = "log";
 	return ext;
 	}
+
+string Ascii::Timestamp(double t)
+	{
+	struct tm tm;
+	char buf[128];
+	const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
+	time_t teatime = time_t(t);
+
+	localtime_r(&teatime, &tm);
+	strftime(buf, sizeof(buf), date_fmt, &tm);
+	return buf;
+	}
+
diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h
index a95e644d49..857954ce37 100644
--- a/src/logging/writers/Ascii.h
+++ b/src/logging/writers/Ascii.h
@@ -26,13 +26,16 @@ protected:
 	virtual bool DoSetBuf(bool enabled);
 	virtual bool DoRotate(string rotated_path, double open,
 			      double close, bool terminating);
-	virtual bool DoFlush();
-	virtual bool DoFinish();
+	virtual bool DoFlush(double network_time);
+	virtual bool DoFinish(double network_time);
+	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
 	bool IsSpecial(string path) 	{ return path.find("/dev/") == 0; }
 	bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
 	bool WriteHeaderField(const string& key, const string& value);
+	void CloseFile(double t);
+	string Timestamp(double t);
 
 	FILE* file;
 	string fname;
@@ -40,7 +43,7 @@ private:
 
 	// Options set from the script-level.
 	bool output_to_stdout;
-	bool include_header;
+	bool include_meta;
 
 	char* separator;
 	int separator_len;
@@ -54,8 +57,8 @@ private:
 	char* unset_field;
 	int unset_field_len;
 
-	char* header_prefix;
-	int header_prefix_len;
+	char* meta_prefix;
+	int meta_prefix_len;
 };
 
 }
diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc
index b34ea3412a..1978a8b781 100644
--- a/src/logging/writers/DataSeries.cc
+++ b/src/logging/writers/DataSeries.cc
@@ -311,7 +311,7 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
 			}
 
 		else
-			Error(Fmt("cannot dump schema: %s", strerror(errno)));
+			Error(Fmt("cannot dump schema: %s", Strerror(errno)));
 		}
 
 	compress_type = Extent::compress_all;
@@ -343,7 +343,7 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
 	return OpenLog(info.path);
 	}
 
-bool DataSeries::DoFlush()
+bool DataSeries::DoFlush(double network_time)
 {
 	// Flushing is handled by DataSeries automatically, so this function
 	// doesn't do anything.
@@ -366,11 +366,10 @@ void DataSeries::CloseLog()
 	log_file = 0;
 	}
 
-bool DataSeries::DoFinish()
+bool DataSeries::DoFinish(double network_time)
 {
 	CloseLog();
-
-	return WriterBackend::DoFinish();
+	return true;
 }
 
 bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
@@ -420,4 +419,9 @@ bool DataSeries::DoSetBuf(bool enabled)
 	return true;
 }
 
+bool DataSeries::DoHeartbeat(double network_time, double current_time)
+{
+	return true;
+}
+
 #endif /* USE_DATASERIES */
diff --git a/src/logging/writers/DataSeries.h b/src/logging/writers/DataSeries.h
index 0ae3572b76..31d17a1a7b 100644
--- a/src/logging/writers/DataSeries.h
+++ b/src/logging/writers/DataSeries.h
@@ -34,8 +34,9 @@ protected:
 	virtual bool DoSetBuf(bool enabled);
 	virtual bool DoRotate(string rotated_path, double open,
 			      double close, bool terminating);
-	virtual bool DoFlush();
-	virtual bool DoFinish();
+	virtual bool DoFlush(double network_time);
+	virtual bool DoFinish(double network_time);
+	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
 	static const size_t ROW_MIN = 2048;			// Minimum extent size.
diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h
index 7e2e4ef4eb..c6d7cba56a 100644
--- a/src/logging/writers/None.h
+++ b/src/logging/writers/None.h
@@ -26,8 +26,9 @@ protected:
 	virtual bool DoSetBuf(bool enabled)	{ return true; }
 	virtual bool DoRotate(string rotated_path, double open,
 			      double close, bool terminating);
-	virtual bool DoFlush()	{ return true; }
-	virtual bool DoFinish()	{ WriterBackend::DoFinish(); return true; }
+	virtual bool DoFlush(double network_time)	{ return true; }
+	virtual bool DoFinish(double network_time)	{ return true; }
+	virtual bool DoHeartbeat(double network_time, double current_time)	{ return true; }
 };
 
 }
diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index dfa4c28eda..88c4ac0965 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -78,24 +78,22 @@ const char* BasicThread::Fmt(const char* format, ...)
 	return buf;
 	}
 
+const char* BasicThread::Strerror(int err)
+	{
+	static char buf[128] = "<not set>";
+	strerror_r(err, buf, sizeof(buf));
+	return buf;
+	}
+
 void BasicThread::Start()
 	{
 
 	if ( started )
 		return;
 
-	int err = pthread_mutex_init(&terminate, 0);
-	if ( err != 0  )
-		reporter->FatalError("Cannot create terminate mutex for thread %s: %s", name.c_str(), strerror(err));
-
-	// We use this like a binary semaphore and acquire it immediately.
-	err = pthread_mutex_lock(&terminate);
+	int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
 	if ( err != 0 )
-		reporter->FatalError("Cannot aquire terminate mutex for thread %s: %s", name.c_str(), strerror(err));
-
-	err = pthread_create(&pthread, 0, BasicThread::launcher, this);
-	if ( err != 0 )
-		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), strerror(err));
+		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), Strerror(err));
 
 	DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
 
@@ -114,12 +112,6 @@ void BasicThread::Stop()
 
 	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
 
-	// Signal that it's ok for the thread to exit now by unlocking the
-	// mutex.
-	int err = pthread_mutex_unlock(&terminate);
-	if ( err != 0 )
-		reporter->FatalError("Failure flagging terminate condition for thread %s: %s", name.c_str(), strerror(err));
-
 	terminating = true;
 
 	OnStop();
@@ -130,16 +122,13 @@ void BasicThread::Join()
 	if ( ! started )
 		return;
 
-	if ( ! terminating )
-		Stop();
+	assert(terminating);
 
 	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
 
 	if ( pthread_join(pthread, 0) != 0  )
 		reporter->FatalError("Failure joining thread %s", name.c_str());
 
-	pthread_mutex_destroy(&terminate);
-
 	DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
 
 	pthread = 0;
@@ -178,10 +167,6 @@ void* BasicThread::launcher(void *arg)
 	// Run thread's main function.
 	thread->Run();
 
-	// Wait until somebody actually wants us to terminate.
-	if ( pthread_mutex_lock(&thread->terminate) != 0 )
-		reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
-
 	return 0;
 	}
 
diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h
index cc87ae03bc..d47eb5c3c3 100644
--- a/src/threading/BasicThread.h
+++ b/src/threading/BasicThread.h
@@ -96,6 +96,14 @@ public:
 	 */
 	const char* Fmt(const char* format, ...);
 
+	/**
+	 * A version of strerror() that the thread can safely use. This is
+	 * essentially a wrapper around strerror_r(). Note that it keeps a
+	 * single static buffer internally so the result remains valid only
+	 * until the next call.
+	 */
+	const char* Strerror(int err);
+
 protected:
 	friend class Manager;
 
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 6a3d496325..81ef123661 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -16,9 +16,17 @@ namespace threading  {
 class FinishMessage : public InputMessage<MsgThread>
 {
 public:
-	FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread)	{ }
+	FinishMessage(MsgThread* thread, double network_time) : InputMessage<MsgThread>("Finish", thread),
+		network_time(network_time) { }
 
-	virtual bool Process()	{ return Object()->DoFinish(); }
+	virtual bool Process()	{
+		bool result = Object()->OnFinish(network_time);
+		Object()->Finished();
+		return result;
+	}
+
+private:
+	double network_time;
 };
 
 // A dummy message that's only purpose is unblock the current read operation
@@ -39,7 +47,10 @@ public:
 		: InputMessage<MsgThread>("Heartbeat", thread)
 		{ network_time = arg_network_time; current_time = arg_current_time; }
 
-	virtual bool Process()	{ return Object()->DoHeartbeat(network_time, current_time); }
+	virtual bool Process()	{
+		Object()->HeartbeatInChild();
+		return Object()->OnHeartbeat(network_time, current_time);
+	}
 
 private:
 	double network_time;
@@ -146,8 +157,11 @@ MsgThread::MsgThread() : BasicThread()
 
 void MsgThread::OnStop()
 	{
+	if ( finished )
+		return;
+
 	// Signal thread to terminate and wait until it has acknowledged.
-	SendIn(new FinishMessage(this), true);
+	SendIn(new FinishMessage(this, network_time), true);
 
 	int cnt = 0;
 	while ( ! finished )
@@ -161,6 +175,8 @@ void MsgThread::OnStop()
 		usleep(1000);
 		}
 
+	Finished();
+
 	// One more message to make sure the current queue read operation unblocks.
 	SendIn(new UnblockMessage(this), true);
 	}
@@ -170,7 +186,7 @@ void MsgThread::Heartbeat()
 	SendIn(new HeartbeatMessage(this, network_time, current_time()));
 	}
 
-bool MsgThread::DoHeartbeat(double network_time, double current_time)
+void MsgThread::HeartbeatInChild()
 	{
 	string n = Name();
 
@@ -179,16 +195,13 @@ bool MsgThread::DoHeartbeat(double network_time, double current_time)
 		cnt_sent_out - queue_out.Size());
 
 	SetOSName(n.c_str());
-
-	return true;
 	}
 
-bool MsgThread::DoFinish()
+void MsgThread::Finished()
 	{
 	// This is thread-safe "enough", we're the only one ever writing
 	// there.
 	finished = true;
-	return true;
 	}
 
 void MsgThread::Info(const char* msg)
diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h
index a917f54396..67ab9517c5 100644
--- a/src/threading/MsgThread.h
+++ b/src/threading/MsgThread.h
@@ -189,39 +189,45 @@ protected:
 	 *
 	 * This is method is called regularly by the threading::Manager.
 	 *
-	 * Can be overriden in derived classed to hook into the heart beat,
-	 * but must call the parent implementation. Note that this method is
-	 * always called by the main thread and must not access data of the
-	 * child thread directly. See DoHeartbeat() if you want to do
-	 * something on the child-side.
+	 * Can be overriden in derived classed to hook into the heart beat
+	 * sending, but must call the parent implementation. Note that this
+	 * method is always called by the main thread and must not access
+	 * data of the child thread directly. Implement OnHeartbeat() if you
+	 * want to do something on the child-side.
 	 */
 	virtual void Heartbeat();
 
-	/**
-	 * Overriden from BasicThread.
-	 *
+	/** Flags that the child process has finished processing. Called from child.
 	 */
-	virtual void Run();
-	virtual void OnStop();
+	void Finished();
+
+	/** Internal heartbeat processing. Called from child.
+	 */
+	void HeartbeatInChild();
 
 	/**
 	 * Regulatly triggered for execution in the child thread.
 	 *
-	 * When overriding, one must call the parent class' implementation.
-	 *
 	 * network_time: The network_time when the heartbeat was trigger by
 	 * the main thread.
 	 *
 	 * current_time: Wall clock when the heartbeat was trigger by the
 	 * main thread.
 	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
+	virtual bool OnHeartbeat(double network_time, double current_time) = 0;
 
 	/** Triggered for execution in the child thread just before shutting threads down.
 	 *  The child thread should finish its operations and then *must*
 	 *  call this class' implementation.
 	 */
-	virtual bool DoFinish();
+	virtual bool OnFinish(double network_time) = 0;
+
+	/**
+	 * Overriden from BasicThread.
+	 *
+	 */
+	virtual void Run();
+	virtual void OnStop();
 
 private:
 	/**
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log
new file mode 100644
index 0000000000..a2610bb522
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log
@@ -0,0 +1,12 @@
+PREFIX<>separator |
+PREFIX<>set_separator|,
+PREFIX<>empty_field|EMPTY
+PREFIX<>unset_field|NOT-SET
+PREFIX<>path|ssh
+PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
+PREFIX<>types|time|addr|port|addr|port|string|string|bool
+1342126762.852986|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
+1342126762.852986|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
+1342126762.852986|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
+1342126762.852986|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
+1342126762.852986|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
deleted file mode 100644
index 10275205a5..0000000000
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log
+++ /dev/null
@@ -1,12 +0,0 @@
-PREFIX<>separator |
-PREFIX<>set_separator|,
-PREFIX<>empty_field|EMPTY
-PREFIX<>unset_field|NOT-SET
-PREFIX<>path|ssh
-PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
-PREFIX<>types|time|addr|port|addr|port|string|string|bool
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
new file mode 100644
index 0000000000..72df0d73d4
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
@@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#start	2012-07-12-21-00-27
+#fields	data	c
+#types	string	count
+Test1	42
+\x23Kaputt	42
+Test2	42
+#end	2012-07-12-21-00-27
diff --git a/testing/btest/core/expr-exception.bro b/testing/btest/core/expr-exception.bro
index 66f9b78c4b..9e84717935 100644
--- a/testing/btest/core/expr-exception.bro
+++ b/testing/btest/core/expr-exception.bro
@@ -2,7 +2,7 @@
 # shouldn't abort Bro entirely, but just return from the function body.
 #
 # @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT >output
-# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
 # @TEST-EXEC: btest-diff output
 
 event connection_established(c: connection)
diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro
index d87d014a21..afbee3f6d9 100644
--- a/testing/btest/istate/events-ssl.bro
+++ b/testing/btest/istate/events-ssl.bro
@@ -6,10 +6,13 @@
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
-# @TEST-EXEC: cmp sender/http.log receiver/http.log
 # 
-# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
-# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
+# @TEST-EXEC: cat sender/http.log   $SCRIPTS/diff-remove-timestamps >sender.http.log
+# @TEST-EXEC: cat receiver/http.log $SCRIPTS/diff-remove-timestamps >receiver.http.log
+# @TEST-EXEC: cmp sender.http.log receiver.http.log
+# 
+# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' | $SCRIPTS/diff-remove-timestamps >events.snd.log
+# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' | $SCRIPTS/diff-remove-timestamps  >events.rec.log
 # @TEST-EXEC: btest-diff events.rec.log
 # @TEST-EXEC: btest-diff events.snd.log
 # @TEST-EXEC: cmp events.rec.log events.snd.log
diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro
index fe588b5c3b..1f05dfc729 100644
--- a/testing/btest/istate/events.bro
+++ b/testing/btest/istate/events.bro
@@ -1,12 +1,15 @@
 # @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
-# @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
+# @TEST-EXEC: btest-bg-run sender   bro -Bthreading,logging,comm -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
+# @TEST-EXEC: btest-bg-run receiver bro -Bthreading,logging,comm  ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
-# @TEST-EXEC: cmp sender/http.log receiver/http.log
+# 
+# @TEST-EXEC: cat sender/http.log   $SCRIPTS/diff-remove-timestamps >sender.http.log
+# @TEST-EXEC: cat receiver/http.log $SCRIPTS/diff-remove-timestamps >receiver.http.log
+# @TEST-EXEC: cmp sender.http.log receiver.http.log
 # 
 # @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
 # @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-empty.bro b/testing/btest/scripts/base/frameworks/logging/ascii-empty.bro
index 9dace5d52a..0bb5900e30 100644
--- a/testing/btest/scripts/base/frameworks/logging/ascii-empty.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-empty.bro
@@ -1,12 +1,13 @@
 #
 # @TEST-EXEC: bro -b %INPUT
-# @TEST-EXEC: btest-diff ssh.log
+# @TEST-EXEC: cat ssh.log | grep -v PREFIX.*20..- >ssh-filtered.log
+# @TEST-EXEC: btest-diff ssh-filtered.log
 
 redef LogAscii::output_to_stdout = F;
 redef LogAscii::separator = "|";
 redef LogAscii::empty_field = "EMPTY";
 redef LogAscii::unset_field = "NOT-SET";
-redef LogAscii::header_prefix = "PREFIX<>";
+redef LogAscii::meta_prefix = "PREFIX<>";
 
 module SSH;
 
diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-line-like-comment.bro b/testing/btest/scripts/base/frameworks/logging/ascii-line-like-comment.bro
new file mode 100644
index 0000000000..4670811b2a
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-line-like-comment.bro
@@ -0,0 +1,23 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff test.log
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		data: string &log;
+		c: count &log &default=42;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Info]);
+	Log::write(Test::LOG, [$data="Test1"]);
+	Log::write(Test::LOG, [$data="#Kaputt"]);
+	Log::write(Test::LOG, [$data="Test2"]);
+}
+
diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-options.bro b/testing/btest/scripts/base/frameworks/logging/ascii-options.bro
index 8c228c1384..474b179536 100644
--- a/testing/btest/scripts/base/frameworks/logging/ascii-options.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-options.bro
@@ -4,7 +4,7 @@
 
 redef LogAscii::output_to_stdout = F;
 redef LogAscii::separator = "|";
-redef LogAscii::include_header = F;
+redef LogAscii::include_meta = F;
 
 module SSH;
 
diff --git a/testing/btest/scripts/base/frameworks/logging/remote-types.bro b/testing/btest/scripts/base/frameworks/logging/remote-types.bro
index f1ef4f0c31..3f102e6319 100644
--- a/testing/btest/scripts/base/frameworks/logging/remote-types.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote-types.bro
@@ -1,10 +1,12 @@
 # @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
-# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run sender bro -B threading,logging --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run receiver bro -B threading,logging --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 10
 # @TEST-EXEC: btest-diff receiver/test.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
+# @TEST-EXEC: cat receiver/test.log | egrep -v '#start|#end' >r.log
+# @TEST-EXEC: cat sender/test.log   | egrep -v '#start|#end' >s.log
+# @TEST-EXEC: cmp r.log s.log
 
 # Remote version testing all types.
 
diff --git a/testing/btest/scripts/base/frameworks/logging/remote.bro b/testing/btest/scripts/base/frameworks/logging/remote.bro
index 8375d7915a..48683148f5 100644
--- a/testing/btest/scripts/base/frameworks/logging/remote.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote.bro
@@ -8,9 +8,11 @@
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
-# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
-# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+# @TEST-EXEC: ( cd sender && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: ( cd receiver && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: cmp receiver/c.test.log sender/c.test.log
+# @TEST-EXEC: cmp receiver/c.test.failure.log sender/c.test.failure.log
+# @TEST-EXEC: cmp receiver/c.test.success.log sender/c.test.success.log
 
 # This is the common part loaded by both sender and receiver.
 module Test;
diff --git a/testing/btest/scripts/base/frameworks/notice/default-policy-order.test b/testing/btest/scripts/base/frameworks/notice/default-policy-order.test
index 6e53bd3b54..d5d3f4c3fa 100644
--- a/testing/btest/scripts/base/frameworks/notice/default-policy-order.test
+++ b/testing/btest/scripts/base/frameworks/notice/default-policy-order.test
@@ -1,10 +1,10 @@
 # This test checks that the default notice policy ordering does not
 # change from run to run.
 # @TEST-EXEC: bro -e ''
-# @TEST-EXEC: mv notice_policy.log notice_policy.log.1
+# @TEST-EXEC: cat notice_policy.log | $SCRIPTS/diff-remove-timestamps > notice_policy.log.1
 # @TEST-EXEC: bro -e ''
-# @TEST-EXEC: mv notice_policy.log notice_policy.log.2
+# @TEST-EXEC: cat notice_policy.log | $SCRIPTS/diff-remove-timestamps > notice_policy.log.2
 # @TEST-EXEC: bro -e ''
-# @TEST-EXEC: mv notice_policy.log notice_policy.log.3
+# @TEST-EXEC: cat notice_policy.log | $SCRIPTS/diff-remove-timestamps > notice_policy.log.3
 # @TEST-EXEC: diff notice_policy.log.1 notice_policy.log.2
 # @TEST-EXEC: diff notice_policy.log.1 notice_policy.log.3
diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index 063f1e4900..2b029789de 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -1,5 +1,8 @@
 #! /usr/bin/env bash
 #
-# Replace anything which looks like timestamps with XXXs.
+# Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs).
+
+sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \
+sed 's/^#\(start\|end\).20..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' | \
+grep -v '#start' | grep -v '#end'
 
-sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g'

From 1ca0d970fc8c3972511067cfbdf9314a6c35d0eb Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 16 Jul 2012 13:39:19 -0700
Subject: [PATCH 048/238] Removing the thread kill functionality.

Wasn't really used and has potential for trouble.
---
 src/main.cc              | 6 ------
 src/threading/Manager.cc | 8 --------
 src/threading/Manager.h  | 9 ---------
 3 files changed, 23 deletions(-)

diff --git a/src/main.cc b/src/main.cc
index d94a32df63..d3937b3449 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -361,12 +361,6 @@ RETSIGTYPE sig_handler(int signo)
 	set_processing_status("TERMINATING", "sig_handler");
 	signal_val = signo;
 
-	if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) )
-		// If the thread manager is already terminating (i.e.,
-		// waiting for child threads to exit), another term signal
-		// will send the threads a kill.
-		thread_mgr->KillThreads();
-
 	return RETSIGVAL;
 	}
 
diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc
index d8f3936037..8e0610a056 100644
--- a/src/threading/Manager.cc
+++ b/src/threading/Manager.cc
@@ -48,14 +48,6 @@ void Manager::Terminate()
 	terminating = false;
 	}
 
-void Manager::KillThreads()
-	{
-	DBG_LOG(DBG_THREADING, "Killing threads ...");
-
-	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
-		(*i)->Kill();
-	}
-
 void Manager::AddThread(BasicThread* thread)
 	{
 	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
diff --git a/src/threading/Manager.h b/src/threading/Manager.h
index 1afd115da0..1c7914fcde 100644
--- a/src/threading/Manager.h
+++ b/src/threading/Manager.h
@@ -49,15 +49,6 @@ public:
 	 */
 	bool Terminating() const	{ return terminating; }
 
-	/**
-	 * Immediately kills all child threads. It does however not yet join
-	 * them, one still needs to call Terminate() for that.
-	 *
-	 * This method is safe to call from a signal handler, and can in fact
-	 * be called while Terminate() is already in progress.
-	 */
-	void KillThreads();
-
 	typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
 
 	/**

From c8789cff94c5200674ad08199a1f800882aabf72 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 16 Jul 2012 13:40:19 -0700
Subject: [PATCH 049/238] If a thread doesn't terminate, we log that but not
 longer proceed (because it could hang later still).

Also logging to stderr as well to make sure one sees it.

Also adding code to the ASCII writer to catch termination
inconsistencies.
---
 aux/binpac                   |  2 +-
 aux/bro-aux                  |  2 +-
 aux/broccoli                 |  2 +-
 aux/broctl                   |  2 +-
 cmake                        |  2 +-
 src/logging/writers/Ascii.cc | 14 ++++++++++++++
 src/logging/writers/Ascii.h  |  1 +
 src/threading/MsgThread.cc   |  8 +++-----
 8 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 4ad8d15b63..b4094cb75e 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 4ad8d15b6395925c9875c9d2912a6cc3b4918e0a
+Subproject commit b4094cb75e0a7769123f7db1f5d73f3f9f1c3977
diff --git a/aux/bro-aux b/aux/bro-aux
index c691c01e9c..2038e3de04 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
+Subproject commit 2038e3de042115c3caa706426e16c830c1fd1e9e
diff --git a/aux/broccoli b/aux/broccoli
index 8234b8903c..07866915a1 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 8234b8903cbc775f341bdb6a1c0159981d88d27b
+Subproject commit 07866915a1450ddd25b888917f494b4824b0cc3f
diff --git a/aux/broctl b/aux/broctl
index d5ecd1a42c..892b60edb9 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit d5ecd1a42c04b0dca332edc31811e5a6d0f7f2fb
+Subproject commit 892b60edb967bb456872638f22ba994e84530137
diff --git a/cmake b/cmake
index 2a72c5e08e..96f3d92aca 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130
+Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index ab68cd77d8..a0d4504d64 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -16,6 +16,7 @@ using threading::Field;
 Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	{
 	file = 0;
+	ascii_done = false;
 
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
 	include_meta = BifConst::LogAscii::include_meta;
@@ -51,6 +52,12 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 
 Ascii::~Ascii()
 	{
+	if ( ! ascii_done )
+		{
+		fprintf(stderr, "missing finish message\n");
+		abort();
+		}
+
 	// Normally, the file will be closed here already via the Finish()
 	// message. But when we terminate abnormally, we may still have it
 	// open.
@@ -156,6 +163,13 @@ bool Ascii::DoFlush(double network_time)
 
 bool Ascii::DoFinish(double network_time)
 	{
+	if ( ascii_done )
+		{
+		fprintf(stderr, "duplicate finish message\n");
+		abort();
+		}
+
+	ascii_done = true;
 	CloseFile(network_time);
 	return true;
 	}
diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h
index 857954ce37..c2cd33f203 100644
--- a/src/logging/writers/Ascii.h
+++ b/src/logging/writers/Ascii.h
@@ -40,6 +40,7 @@ private:
 	FILE* file;
 	string fname;
 	ODesc desc;
+	bool ascii_done;
 
 	// Options set from the script-level.
 	bool output_to_stdout;
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 81ef123661..e4cda1e84d 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -166,17 +166,15 @@ void MsgThread::OnStop()
 	int cnt = 0;
 	while ( ! finished )
 		{
-		if ( ++cnt > 1000 ) // Insurance against broken threads ...
+		if ( ++cnt % 2000 == 0 ) // Insurance against broken threads ...
 			{
-			reporter->Warning("thread %s didn't finish in time", Name().c_str());
-			break;
+			reporter->Warning("thread %s has not yet terminated ...", Name().c_str());
+			fprintf(stderr, "warning: thread %s has not yet terminated ...", Name().c_str());
 			}
 
 		usleep(1000);
 		}
 
-	Finished();
-
 	// One more message to make sure the current queue read operation unblocks.
 	SendIn(new UnblockMessage(this), true);
 	}

From f7a6407ab1213d95f074e47c39061f541f630944 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 12 Jul 2012 13:44:24 -0700
Subject: [PATCH 050/238] Reworking thread termination logic.

Turns out the finish methods weren't called correctly, caused by a
mess up with method names which all sounded too similar and the wrong
one ended up being called. I've reworked this by changing the
thread/writer/reader interfaces, which actually also simplifies them
by getting rid of the requirement for writer backends to call their
parent methods (i.e., less opportunity for errors).

This commit also includes the following (because I noticed the problem
above when working on some of these):

     - The ASCII log writer now includes "#start <timestamp>" and
      "#end <timestamp> lines in the each file. The latter supersedes
      Bernhard's "EOF" patch.

      This required a number of tests updates. The standard canonifier
      removes the timestamps, but some tests compare files directly,
      which doesn't work if they aren't printing out the same
      timestamps (like the comm tests).

     - The above required yet another change to the writer API to
       network_time to methods.

     - Renamed ASCII logger "header" options to "meta".

     - Fixes #763 "Escape # when first character in log file line".

All btests pass for me on Linux FC15. Will try MacOS next.
---
 src/logging/writers/Ascii.cc | 7 +------
 src/threading/MsgThread.cc   | 2 ++
 2 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index a0d4504d64..3bc4ef4b38 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -52,12 +52,6 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 
 Ascii::~Ascii()
 	{
-	if ( ! ascii_done )
-		{
-		fprintf(stderr, "missing finish message\n");
-		abort();
-		}
-
 	// Normally, the file will be closed here already via the Finish()
 	// message. But when we terminate abnormally, we may still have it
 	// open.
@@ -170,6 +164,7 @@ bool Ascii::DoFinish(double network_time)
 		}
 
 	ascii_done = true;
+
 	CloseFile(network_time);
 	return true;
 	}
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index e4cda1e84d..45fbf6afa5 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -175,6 +175,8 @@ void MsgThread::OnStop()
 		usleep(1000);
 		}
 
+	Finished();
+
 	// One more message to make sure the current queue read operation unblocks.
 	SendIn(new UnblockMessage(this), true);
 	}

From f6b883bafc71840e146768b966d37a9229559c18 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 17 Jul 2012 17:09:49 -0700
Subject: [PATCH 051/238] Further reworking the thread API.

---
 src/logging/WriterFrontend.cc   |  3 ---
 src/logging/WriterFrontend.h    |  2 +-
 src/threading/MsgThread.cc      | 13 +++++--------
 src/threading/MsgThread.h       |  1 +
 src/util.cc                     | 22 ++++++++++++++++++++++
 src/util.h                      |  6 ++++++
 testing/scripts/diff-canonifier |  2 +-
 7 files changed, 36 insertions(+), 13 deletions(-)

diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc
index 577003926b..b816327e9c 100644
--- a/src/logging/WriterFrontend.cc
+++ b/src/logging/WriterFrontend.cc
@@ -141,9 +141,6 @@ void WriterFrontend::Stop()
 	{
 	FlushWriteBuffer();
 	SetDisable();
-
-	if ( backend )
-		backend->Stop();
 	}
 
 void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num_fields, const Field* const * arg_fields)
diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h
index 6581fb1c1b..e8f3d06d6c 100644
--- a/src/logging/WriterFrontend.h
+++ b/src/logging/WriterFrontend.h
@@ -50,7 +50,7 @@ public:
 
 	/**
 	 * Stops all output to this writer. Calling this methods disables all
-	 * message forwarding to the backend and stops the backend thread.
+	 * message forwarding to the backend.
 	 *
 	 * This method must only be called from the main thread.
 	 */
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 45fbf6afa5..f101d0ca3c 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -152,12 +152,13 @@ MsgThread::MsgThread() : BasicThread()
 	{
 	cnt_sent_in = cnt_sent_out = 0;
 	finished = false;
+	stopped = false;
 	thread_mgr->AddMsgThread(this);
 	}
 
 void MsgThread::OnStop()
 	{
-	if ( finished )
+	if ( stopped )
 		return;
 
 	// Signal thread to terminate and wait until it has acknowledged.
@@ -303,13 +304,8 @@ BasicInputMessage* MsgThread::RetrieveIn()
 
 void MsgThread::Run()
 	{
-	while ( true )
+	while ( ! finished )
 		{
-		// When requested to terminate, we only do so when
-		// all input has been processed.
-		if ( Terminating() && ! queue_in.Ready() )
-			break;
-
 		BasicInputMessage* msg = RetrieveIn();
 
 		bool result = msg->Process();
@@ -318,12 +314,13 @@ void MsgThread::Run()
 			{
 			string s = msg->Name() + " failed, terminating thread (MsgThread)";
 			Error(s.c_str());
-			Stop();
 			break;
 			}
 
 		delete msg;
 		}
+
+    Finished();
 	}
 
 void MsgThread::GetStats(Stats* stats)
diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h
index 67ab9517c5..d929c1f806 100644
--- a/src/threading/MsgThread.h
+++ b/src/threading/MsgThread.h
@@ -293,6 +293,7 @@ private:
 	uint64_t cnt_sent_out;	// Counts message sent by child.
 
 	bool finished;	// Set to true by Finished message.
+	bool stopped;   // Set to true by OnStop().
 };
 
 /**
diff --git a/src/util.cc b/src/util.cc
index 3cfa5fca1c..b7a4683597 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1290,6 +1290,28 @@ uint64 calculate_unique_id(size_t pool)
 	return HashKey::HashBytes(&(uid_pool[pool].key), sizeof(uid_pool[pool].key));
 	}
 
+bool safe_write(int fd, const char* data, int len)
+	{
+    return true;
+	while ( len > 0 )
+		{
+		int n = write(fd, data, len);
+
+		if ( n < 0 )
+			{
+			if ( errno == EINTR )
+				continue;
+
+			return false;
+			}
+
+		data += n;
+		len -= n;
+		}
+
+	return true;
+	}
+
 void out_of_memory(const char* where)
 	{
 	reporter->FatalError("out of memory in %s.\n", where);
diff --git a/src/util.h b/src/util.h
index e4c995f45f..075c2af7c2 100644
--- a/src/util.h
+++ b/src/util.h
@@ -289,6 +289,11 @@ inline size_t pad_size(size_t size)
 
 #define padded_sizeof(x) (pad_size(sizeof(x)))
 
+// Like write() but handles interrupted system calls by restarting. Returns
+// true if the write was successful, otherwise sets errno. This function is
+// thread-safe as long as no two threads write to the same descriptor.
+extern bool safe_write(int fd, const char* data, int len);
+
 extern void out_of_memory(const char* where);
 
 inline void* safe_realloc(void* ptr, size_t size)
@@ -338,4 +343,5 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al
 // handed out by malloc.
 extern void get_memory_usage(unsigned int* total,
 			     unsigned int* malloced);
+
 #endif
diff --git a/testing/scripts/diff-canonifier b/testing/scripts/diff-canonifier
index 3cb213a3f7..4d04b3372c 100755
--- a/testing/scripts/diff-canonifier
+++ b/testing/scripts/diff-canonifier
@@ -2,4 +2,4 @@
 #
 # Default canonifier used with the tests in testing/btest/*.
 
-`dirname $0`/diff-remove-timestamps
+`dirname $0`/diff-remove-timestamps | grep -v XXX

From e90918aa509c6c44078707d147144e62dc4bc4d4 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 17 Jul 2012 19:02:36 -0700
Subject: [PATCH 052/238] Moving the ASCII writer over to use UNIX I/O rather
 than stdio.

---
 src/logging/writers/Ascii.cc | 40 +++++++++++++++++++-----------------
 src/logging/writers/Ascii.h  |  2 +-
 src/threading/BasicThread.cc |  5 ++---
 src/util.cc                  |  1 -
 4 files changed, 24 insertions(+), 24 deletions(-)

diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 3bc4ef4b38..c1f307fb4e 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -15,7 +15,7 @@ using threading::Field;
 
 Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	{
-	file = 0;
+	fd = 0;
 	ascii_done = false;
 
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
@@ -53,9 +53,8 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 Ascii::~Ascii()
 	{
 	// Normally, the file will be closed here already via the Finish()
-	// message. But when we terminate abnormally, we may still have it
-	// open.
-	if ( file )
+	// message. But when we terminate abnormally, we may still have it open.
+	if ( fd )
 		CloseFile(0);
 
 	delete [] separator;
@@ -70,23 +69,25 @@ bool Ascii::WriteHeaderField(const string& key, const string& val)
 	string str = string(meta_prefix, meta_prefix_len) +
 		key + string(separator, separator_len) + val + "\n";
 
-	return (fwrite(str.c_str(), str.length(), 1, file) == 1);
+	return safe_write(fd, str.c_str(), str.length());
 	}
 
 void Ascii::CloseFile(double t)
 	{
-	if ( ! file )
+	if ( ! fd)
 		return;
 
 	if ( include_meta )
 		WriteHeaderField("end", t ? Timestamp(t) : "<abnormal termination>");
 
-	fclose(file);
-	file = 0;
+	close(fd);
+	fd = 0;
 	}
 
 bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
 	{
+    assert(! fd);
+
 	string path = info.path;
 
 	if ( output_to_stdout )
@@ -94,11 +95,13 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 
 	fname = IsSpecial(path) ? path : path + "." + LogExt();
 
-	if ( ! (file = fopen(fname.c_str(), "w")) )
+	fd = open(fname.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0777);
+
+	if ( fd < 0 )
 		{
 		Error(Fmt("cannot open %s: %s", fname.c_str(),
 			  Strerror(errno)));
-
+		fd = 0;
 		return false;
 		}
 
@@ -112,7 +115,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 			+ get_escaped_string(string(separator, separator_len), false)
 			+ "\n";
 
-		if( fwrite(str.c_str(), str.length(), 1, file) != 1 )
+		if ( ! safe_write(fd, str.c_str(), str.length()) )
 			goto write_error;
 
 		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
@@ -151,7 +154,7 @@ write_error:
 
 bool Ascii::DoFlush(double network_time)
 	{
-	fflush(file);
+	fsync(fd);
 	return true;
 	}
 
@@ -318,7 +321,7 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
-	if ( ! file )
+	if ( ! fd )
 		DoInit(Info(), NumFields(), Fields());
 
 	desc.Clear();
@@ -337,24 +340,23 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	const char* bytes = (const char*)desc.Bytes();
 	int len = desc.Len();
 
-	// Make sure the line doesn't look like meta information.
 	if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 )
 		{
 		// It would so escape the first character.
 		char buf[16];
 		snprintf(buf, sizeof(buf), "\\x%02x", bytes[0]);
-		if ( fwrite(buf, strlen(buf), 1, file) != 1 )
+		if ( ! safe_write(fd, buf, strlen(buf)) )
 			goto write_error;
 
 		++bytes;
 		--len;
 		}
 
-	if ( fwrite(bytes, len, 1, file) != 1 )
+	if ( ! safe_write(fd, bytes, len) )
 		goto write_error;
 
-	if ( IsBuf() )
-		fflush(file);
+        if ( IsBuf() )
+		fsync(fd);
 
 	return true;
 
@@ -366,7 +368,7 @@ write_error:
 bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
 	{
 	// Don't rotate special files or if there's not one currently open.
-	if ( ! file || IsSpecial(Info().path) )
+	if ( ! fd || IsSpecial(Info().path) )
 		return true;
 
 	CloseFile(close);
diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h
index c2cd33f203..371ded4344 100644
--- a/src/logging/writers/Ascii.h
+++ b/src/logging/writers/Ascii.h
@@ -37,7 +37,7 @@ private:
 	void CloseFile(double t);
 	string Timestamp(double t);
 
-	FILE* file;
+	int fd;
 	string fname;
 	ODesc desc;
 	bool ascii_done;
diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index 88c4ac0965..075581e9db 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -87,18 +87,17 @@ const char* BasicThread::Strerror(int err)
 
 void BasicThread::Start()
 	{
-
 	if ( started )
 		return;
 
+	started = true;
+
 	int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
 	if ( err != 0 )
 		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), Strerror(err));
 
 	DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
 
-	started = true;
-
 	OnStart();
 	}
 
diff --git a/src/util.cc b/src/util.cc
index b7a4683597..553944c69c 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1292,7 +1292,6 @@ uint64 calculate_unique_id(size_t pool)
 
 bool safe_write(int fd, const char* data, int len)
 	{
-    return true;
 	while ( len > 0 )
 		{
 		int n = write(fd, data, len);

From 490859cfeff6b8747a09e31122ec0afc60e318d0 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 17 Jul 2012 19:36:30 -0700
Subject: [PATCH 053/238] Reworking forceful thread termination.

Ctrl-C now kills a thread even if it hangs at termination. And readded
a (rather long) timeout to kill threads automatically that don't
shutdown.
---
 src/logging/WriterBackend.cc |  4 +---
 src/logging/writers/Ascii.cc |  1 +
 src/threading/BasicThread.cc | 10 +++++-----
 src/threading/Manager.cc     |  8 ++++++++
 src/threading/Manager.h      |  7 +++++++
 src/threading/MsgThread.cc   | 32 ++++++++++++++++++++++++++++----
 6 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index 68b0b506a1..a284c56201 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -162,9 +162,7 @@ bool WriterBackend::Init(const WriterInfo& arg_info, int arg_num_fields, const F
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
-	string name = Fmt("%s/%s", info.path.c_str(), frontend_name.c_str());
-
-	SetName(name);
+	SetName(frontend->Name());
 
 	if ( ! DoInit(arg_info, arg_num_fields, arg_fields) )
 		{
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index c1f307fb4e..20963d1535 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -169,6 +169,7 @@ bool Ascii::DoFinish(double network_time)
 	ascii_done = true;
 
 	CloseFile(network_time);
+
 	return true;
 	}
 
diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index 075581e9db..e7fb3f4c84 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -125,7 +125,7 @@ void BasicThread::Join()
 
 	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
 
-	if ( pthread_join(pthread, 0) != 0  )
+	if ( pthread && pthread_join(pthread, 0) != 0  )
 		reporter->FatalError("Failure joining thread %s", name.c_str());
 
 	DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
@@ -135,13 +135,13 @@ void BasicThread::Join()
 
 void BasicThread::Kill()
 	{
+	terminating = true;
+
 	if ( ! (started && pthread) )
 		return;
 
-	// I believe this is safe to call from a signal handler ... Not error
-	// checking so that killing doesn't bail out if we have already
-	// terminated.
-	pthread_kill(pthread, SIGKILL);
+	pthread = 0;
+	pthread_kill(pthread, SIGTERM);
 	}
 
 void* BasicThread::launcher(void *arg)
diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc
index 8e0610a056..f1f9307b03 100644
--- a/src/threading/Manager.cc
+++ b/src/threading/Manager.cc
@@ -83,6 +83,14 @@ double Manager::NextTimestamp(double* network_time)
 	return -1.0;
 	}
 
+void Manager::KillThreads()
+	{
+        DBG_LOG(DBG_THREADING, "Killing threads ...");
+	
+        for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+                (*i)->Kill();
+        }
+
 void Manager::Process()
 	{
 	bool do_beat = false;
diff --git a/src/threading/Manager.h b/src/threading/Manager.h
index 1c7914fcde..be81c69ba0 100644
--- a/src/threading/Manager.h
+++ b/src/threading/Manager.h
@@ -106,6 +106,13 @@ protected:
 	 */
 	virtual double NextTimestamp(double* network_time);
 
+	/**
+	 * Kills all thread immediately. Note that this may cause race conditions
+	 * if a child thread currently holds a lock that might block somebody
+	 * else.
+	 */
+	virtual void KillThreads();
+
 	/**
 	 * Part of the IOSource interface.
 	 */
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index f101d0ca3c..3913624654 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -156,6 +156,9 @@ MsgThread::MsgThread() : BasicThread()
 	thread_mgr->AddMsgThread(this);
 	}
 
+// Set by Bro's main signal handler.
+extern int signal_val;
+
 void MsgThread::OnStop()
 	{
 	if ( stopped )
@@ -164,13 +167,31 @@ void MsgThread::OnStop()
 	// Signal thread to terminate and wait until it has acknowledged.
 	SendIn(new FinishMessage(this, network_time), true);
 
+	int old_signal_val = signal_val;
+	signal_val = 0;
+
 	int cnt = 0;
+	bool aborted = 0;
+
 	while ( ! finished )
 		{
-		if ( ++cnt % 2000 == 0 ) // Insurance against broken threads ...
+                // Terminate if we get another kill signal.
+		if ( signal_val == SIGTERM || signal_val == SIGINT )
 			{
-			reporter->Warning("thread %s has not yet terminated ...", Name().c_str());
-			fprintf(stderr, "warning: thread %s has not yet terminated ...", Name().c_str());
+			// Abort all threads here so that we won't hang next
+			// on another one.
+                        fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name().c_str());
+			thread_mgr->KillThreads();
+			aborted = true;
+			break;
+			}
+
+		if ( ++cnt % 10000 == 0 ) // Insurance against broken threads ...
+			{
+                        fprintf(stderr, "killing thread %s ...\n", Name().c_str());
+			Kill();
+			aborted = true;
+			break;
 			}
 
 		usleep(1000);
@@ -178,8 +199,11 @@ void MsgThread::OnStop()
 
 	Finished();
 
+	signal_val = old_signal_val;
+
 	// One more message to make sure the current queue read operation unblocks.
-	SendIn(new UnblockMessage(this), true);
+	if ( ! aborted )
+		SendIn(new UnblockMessage(this), true);
 	}
 
 void MsgThread::Heartbeat()

From 87e10b5f97a897f8c5fac2f983379a8c8966dcae Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 18 Jul 2012 12:47:13 -0700
Subject: [PATCH 054/238] Further threading and API restructuring for logging
 and input frameworks.

There were a number of cases that weren't thread-safe. In particular,
we don't use std::string anymore for anything that's passed between
threads (but instead plain old const char*, with manual memmory
managmenet).

This is still a check-point commit, I'll do more testing.
---
 src/RemoteSerializer.cc                |   4 +-
 src/input/Manager.cc                   |  68 ++++++------
 src/input/ReaderBackend.cc             |  21 ++--
 src/input/ReaderBackend.h              |  49 +++++++--
 src/input/ReaderFrontend.cc            |  30 +++---
 src/input/ReaderFrontend.h             |  27 ++---
 src/input/readers/Ascii.cc             |  23 +++--
 src/input/readers/Benchmark.cc         |   8 +-
 src/input/readers/Raw.cc               |  10 +-
 src/logging/Manager.cc                 |  89 +++++++++-------
 src/logging/Manager.h                  |   6 +-
 src/logging/WriterBackend.cc           |  40 +++++---
 src/logging/WriterBackend.h            |  51 +++++++--
 src/logging/WriterFrontend.cc          |  46 ++++-----
 src/logging/WriterFrontend.h           |  18 ++--
 src/logging/writers/Ascii.cc           |  45 ++++----
 src/logging/writers/Ascii.h            |   2 +-
 src/logging/writers/DataSeries.cc      |  15 +--
 src/logging/writers/DataSeries.h       |   2 +-
 src/logging/writers/None.cc            |  21 +++-
 src/logging/writers/None.h             |   2 +-
 src/threading/BasicThread.cc           |  95 ++++++++++++-----
 src/threading/BasicThread.h            |  55 ++++++++--
 src/threading/Manager.cc               |  19 +++-
 src/threading/MsgThread.cc             | 137 +++++++++++++------------
 src/threading/MsgThread.h              |  18 ++--
 src/threading/Queue.h                  |  82 ++++++++++++---
 src/threading/SerialTypes.cc           |  48 +++++++--
 src/threading/SerialTypes.h            |  34 ++++--
 testing/btest/istate/events.bro        |   4 +-
 testing/scripts/diff-remove-timestamps |   4 +-
 31 files changed, 692 insertions(+), 381 deletions(-)

diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc
index 9409a34634..7ed8b9318e 100644
--- a/src/RemoteSerializer.cc
+++ b/src/RemoteSerializer.cc
@@ -2692,12 +2692,12 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 
 	int id, writer;
 	int num_fields;
-	logging::WriterBackend::WriterInfo info;
+	logging::WriterBackend::WriterInfo* info = new logging::WriterBackend::WriterInfo();
 
 	bool success = fmt.Read(&id, "id") &&
 		fmt.Read(&writer, "writer") &&
 		fmt.Read(&num_fields, "num_fields") &&
-		info.Read(&fmt);
+		info->Read(&fmt);
 
 	if ( ! success )
 		goto error;
diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 1c6b69e8ec..f38613a6f8 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -71,7 +71,7 @@ declare(PDict, InputHash);
 class Manager::Stream {
 public:
 	string name;
-	ReaderBackend::ReaderInfo info;
+	ReaderBackend::ReaderInfo* info;
 	bool removed;
 
 	StreamType stream_type; // to distinguish between event and table streams
@@ -257,7 +257,6 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type)
 
 	assert(ir->factory);
 
-	frontend->SetTypeName(ir->name);
 	ReaderBackend* backend = (*ir->factory)(frontend);
 	assert(backend);
 
@@ -291,9 +290,6 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 
 	EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal();
 
-	ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt());
-	assert(reader_obj);
-
 	// get the source ...
 	Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source"));
 	assert ( sourceval != 0 );
@@ -301,21 +297,22 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 	string source((const char*) bsource->Bytes(), bsource->Len());
 	Unref(sourceval);
 
-	EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal();
-	Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
+	ReaderBackend::ReaderInfo* rinfo = new ReaderBackend::ReaderInfo();
+	rinfo->source = copy_string(source.c_str());
 
+	EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal();
 	switch ( mode->InternalInt() )
 		{
 		case 0:
-			info->info.mode = MODE_MANUAL;
+			rinfo->mode = MODE_MANUAL;
 			break;
 
 		case 1:
-			info->info.mode = MODE_REREAD;
+			rinfo->mode = MODE_REREAD;
 			break;
 
 		case 2:
-			info->info.mode = MODE_STREAM;
+			rinfo->mode = MODE_STREAM;
 			break;
 
 		default:
@@ -324,12 +321,16 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 
 	Unref(mode);
 
+	Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
+
+	ReaderFrontend* reader_obj = new ReaderFrontend(*rinfo, reader);
+	assert(reader_obj);
+
 	info->reader = reader_obj;
 	info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault
 	info->name = name;
 	info->config = config->AsTableVal(); // ref'd by LookupWithDefault
-
-	info->info.source = source;
+	info->info = rinfo;
 
 	Ref(description);
 	info->description = description;
@@ -344,7 +345,7 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 			ListVal* index = info->config->RecoverIndex(k);
 			string key = index->Index(0)->AsString()->CheckString();
 			string value = v->Value()->AsString()->CheckString();
-			info->info.config.insert(std::make_pair(key, value));
+			info->info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
 			Unref(index);
 			delete k;
 			}
@@ -475,7 +476,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	assert(stream->reader);
 
-	stream->reader->Init(stream->info, stream->num_fields, logf );
+	stream->reader->Init(stream->num_fields, logf );
 
 	readers[stream->reader] = stream;
 
@@ -652,7 +653,7 @@ bool Manager::CreateTableStream(RecordVal* fval)
 
 
 	assert(stream->reader);
-	stream->reader->Init(stream->info, fieldsV.size(), fields );
+	stream->reader->Init(fieldsV.size(), fields );
 
 	readers[stream->reader] = stream;
 
@@ -791,17 +792,19 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 
 		else
 			{
-			Field* field = new Field();
-			field->name = nameprepend + rec->FieldName(i);
-			field->type = rec->FieldType(i)->Tag();
+			string name = nameprepend + rec->FieldName(i);
+			const char* secondary = 0;
+			TypeTag ty = rec->FieldType(i)->Tag();
+			TypeTag st = TYPE_VOID;
+			bool optional = false;
 
-			if ( field->type == TYPE_TABLE )
-				field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag();
+			if ( ty == TYPE_TABLE )
+				st = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag();
 
-			else if ( field->type == TYPE_VECTOR )
-				field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag();
+			else if ( ty == TYPE_VECTOR )
+				st = rec->FieldType(i)->AsVectorType()->YieldType()->Tag();
 
-			else if ( field->type == TYPE_PORT &&
+			else if ( ty == TYPE_PORT &&
 				  rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) )
 				{
 				// we have an annotation for the second column
@@ -811,12 +814,13 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 				assert(c);
 				assert(c->Type()->Tag() == TYPE_STRING);
 
-				field->secondary_name = c->AsStringVal()->AsString()->CheckString();
+				secondary = c->AsStringVal()->AsString()->CheckString();
 				}
 
 			if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) )
-				field->optional = true;
+				optional = true;
 
+			Field* field = new Field(name.c_str(), secondary, ty, st, optional);
 			fields->push_back(field);
 			}
 		}
@@ -1230,7 +1234,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 #endif
 
 	// Send event that the current update is indeed finished.
-	SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info.source.c_str()));
+	SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source));
 	}
 
 void Manager::Put(ReaderFrontend* reader, Value* *vals)
@@ -1707,7 +1711,7 @@ int Manager::GetValueLength(const Value* val) {
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		length += val->val.string_val->size();
+		length += val->val.string_val.length;
 		break;
 		}
 
@@ -1806,8 +1810,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length());
-		return val->val.string_val->size();
+		memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length);
+		return val->val.string_val.length;
 		}
 
 	case TYPE_ADDR:
@@ -1955,7 +1959,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 
 	case TYPE_STRING:
 		{
-		BroString *s = new BroString(*(val->val.string_val));
+		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 0);
 		return new StringVal(s);
 		}
 
@@ -2039,8 +2043,8 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 	case TYPE_ENUM: {
 		// well, this is kind of stupid, because EnumType just mangles the module name and the var name together again...
 		// but well
-		string module = extract_module_name(val->val.string_val->c_str());
-		string var = extract_var_name(val->val.string_val->c_str());
+		string module = extract_module_name(val->val.string_val.data);
+		string var = extract_var_name(val->val.string_val.data);
 		bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str());
 		if ( index == -1 )
 			reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s",
diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc
index 84106a3c94..88a78c3cd7 100644
--- a/src/input/ReaderBackend.cc
+++ b/src/input/ReaderBackend.cc
@@ -56,22 +56,24 @@ private:
 
 class SendEventMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val)
+	SendEventMessage(ReaderFrontend* reader, const char* name, const int num_vals, Value* *val)
 		: threading::OutputMessage<ReaderFrontend>("SendEvent", reader),
-		name(name), num_vals(num_vals), val(val) {}
+		name(copy_string(name)), num_vals(num_vals), val(val) {}
+
+	virtual ~SendEventMessage()	{ delete [] name; }
 
 	virtual bool Process()
 		{
 		bool success = input_mgr->SendEvent(name, num_vals, val);
 
 		if ( ! success )
-			reporter->Error("SendEvent for event %s failed", name.c_str());
+			reporter->Error("SendEvent for event %s failed", name);
 
 		return true; // We do not want to die if sendEvent fails because the event did not return.
 		}
 
 private:
-	const string name;
+	const char* name;
 	const int num_vals;
 	Value* *val;
 };
@@ -146,12 +148,14 @@ ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread()
 	{
 	disabled = true; // disabled will be set correcty in init.
 	frontend = arg_frontend;
+	info = new ReaderInfo(frontend->Info());
 
 	SetName(frontend->Name());
 	}
 
 ReaderBackend::~ReaderBackend()
 	{
+	delete info;
 	}
 
 void ReaderBackend::Put(Value* *val)
@@ -169,7 +173,7 @@ void ReaderBackend::Clear()
 	SendOut(new ClearMessage(frontend));
 	}
 
-void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals)
+void ReaderBackend::SendEvent(const char* name, const int num_vals, Value* *vals)
 	{
 	SendOut(new SendEventMessage(frontend, name, num_vals, vals));
 	}
@@ -184,17 +188,14 @@ void ReaderBackend::SendEntry(Value* *vals)
 	SendOut(new SendEntryMessage(frontend, vals));
 	}
 
-bool ReaderBackend::Init(const ReaderInfo& arg_info, const int arg_num_fields,
+bool ReaderBackend::Init(const int arg_num_fields,
 		         const threading::Field* const* arg_fields)
 	{
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
-	SetName("InputReader/"+info.source);
-
 	// disable if DoInit returns error.
-	int success = DoInit(arg_info, arg_num_fields, arg_fields);
+	int success = DoInit(*info, arg_num_fields, arg_fields);
 
 	if ( ! success )
 		{
diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h
index 1e77a61f37..7626cc25ed 100644
--- a/src/input/ReaderBackend.h
+++ b/src/input/ReaderBackend.h
@@ -34,7 +34,10 @@ enum ReaderMode {
 	 * for new appended data. When new data is appended is has to be sent
 	 * using the Put api functions.
 	 */
-	MODE_STREAM
+	MODE_STREAM,
+
+	/** Internal dummy mode for initialization. */
+	MODE_NONE
 };
 
 class ReaderFrontend;
@@ -70,14 +73,17 @@ public:
 	 */
 	struct ReaderInfo
 		{
-		typedef std::map<string, string> config_map;
+		// Structure takes ownership of the strings.
+		typedef std::map<const char*, const char*> config_map;
 
 		/**
 		 * A string left to the interpretation of the reader
 		 * implementation; it corresponds to the value configured on
 		 * the script-level for the logging filter.
+		 *
+		 * Structure takes ownership of the string.
 		 */
-		string source;
+		const char* source;
 
 		/**
 		 * A map of key/value pairs corresponding to the relevant
@@ -89,6 +95,35 @@ public:
 		 * The opening mode for the input source.
 		 */
 		ReaderMode mode;
+
+		ReaderInfo()
+			{
+			source = 0;
+			mode = MODE_NONE;
+			}
+
+		ReaderInfo(const ReaderInfo& other)
+			{
+			source = other.source ? copy_string(other.source) : 0;
+			mode = other.mode;
+
+			for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
+				config.insert(std::make_pair(copy_string(i->first), copy_string(i->second)));
+			}
+
+		~ReaderInfo()
+			{
+			delete [] source;
+
+			for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
+				{
+				delete [] i->first;
+				delete [] i->second;
+				}
+			}
+
+		private:
+		const ReaderInfo& operator=(const ReaderInfo& other); // Disable.
 		};
 
 	/**
@@ -106,7 +141,7 @@ public:
 	 *
 	 * @return False if an error occured.
 	 */
-	bool Init(const ReaderInfo& info, int num_fields, const threading::Field* const* fields);
+	bool Init(int num_fields, const threading::Field* const* fields);
 
 	/**
 	 * Force trigger an update of the input stream. The action that will
@@ -133,7 +168,7 @@ public:
 	/**
 	 * Returns the additional reader information into the constructor.
 	 */
-	const ReaderInfo& Info() const	{ return info; }
+	const ReaderInfo& Info() const	{ return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -209,7 +244,7 @@ protected:
 	 *
 	 * @param vals the values to be given to the event
 	 */
-	void SendEvent(const string& name, const int num_vals, threading::Value* *vals);
+	void SendEvent(const char* name, const int num_vals, threading::Value* *vals);
 
 	// Content-sending-functions (simple mode). Include table-specific
 	// functionality that simply is not used if we have no table.
@@ -291,7 +326,7 @@ private:
 	// from this class, it's running in a different thread!
 	ReaderFrontend* frontend;
 
-	ReaderInfo info;
+	ReaderInfo* info;
 	unsigned int num_fields;
 	const threading::Field* const * fields; // raw mapping
 
diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc
index 7e4ef201b1..a8528c002d 100644
--- a/src/input/ReaderFrontend.cc
+++ b/src/input/ReaderFrontend.cc
@@ -11,18 +11,17 @@ namespace input {
 class InitMessage : public threading::InputMessage<ReaderBackend>
 {
 public:
-	InitMessage(ReaderBackend* backend, const ReaderBackend::ReaderInfo& info,
+	InitMessage(ReaderBackend* backend,
 		    const int num_fields, const threading::Field* const* fields)
 		: threading::InputMessage<ReaderBackend>("Init", backend),
-		info(info), num_fields(num_fields), fields(fields) { }
+		num_fields(num_fields), fields(fields) { }
 
 	virtual bool Process()
 		{
-		return Object()->Init(info, num_fields, fields);
+		return Object()->Init(num_fields, fields);
 		}
 
 private:
-	const ReaderBackend::ReaderInfo info;
 	const int num_fields;
 	const threading::Field* const* fields;
 };
@@ -37,21 +36,26 @@ public:
 	virtual bool Process() { return Object()->Update(); }
 };
 
-ReaderFrontend::ReaderFrontend(bro_int_t type)
+ReaderFrontend::ReaderFrontend(const ReaderBackend::ReaderInfo& arg_info, EnumVal* type)
 	{
 	disabled = initialized = false;
-	ty_name = "<not set>";
-	backend = input_mgr->CreateBackend(this, type);
+	info = new ReaderBackend::ReaderInfo(arg_info);
 
+	const char* t = type->Type()->AsEnumType()->Lookup(type->InternalInt());
+	name = copy_string(fmt("%s/%s", arg_info.source, t));
+
+	backend = input_mgr->CreateBackend(this, type->InternalInt());
 	assert(backend);
 	backend->Start();
 	}
 
 ReaderFrontend::~ReaderFrontend()
 	{
+	delete [] name;
+	delete info;
 	}
 
-void ReaderFrontend::Init(const ReaderBackend::ReaderInfo& arg_info, const int arg_num_fields,
+void ReaderFrontend::Init(const int arg_num_fields,
 		          const threading::Field* const* arg_fields)
 	{
 	if ( disabled )
@@ -60,12 +64,11 @@ void ReaderFrontend::Init(const ReaderBackend::ReaderInfo& arg_info, const int a
 	if ( initialized )
 		reporter->InternalError("reader initialize twice");
 
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 	initialized = true;
 
-	backend->SendIn(new InitMessage(backend, info, num_fields, fields));
+	backend->SendIn(new InitMessage(backend, num_fields, fields));
 	}
 
 void ReaderFrontend::Update()
@@ -82,12 +85,9 @@ void ReaderFrontend::Update()
 	backend->SendIn(new UpdateMessage(backend));
 	}
 
-string ReaderFrontend::Name() const
+const char* ReaderFrontend::Name() const
 	{
-	if ( ! info.source.size() )
-		return ty_name;
-
-	return ty_name + "/" + info.source;
+	return name;
 	}
 
 }
diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h
index 93e416e65b..a93f7703ac 100644
--- a/src/input/ReaderFrontend.h
+++ b/src/input/ReaderFrontend.h
@@ -4,10 +4,11 @@
 #define INPUT_READERFRONTEND_H
 
 #include "ReaderBackend.h"
-
 #include "threading/MsgThread.h"
 #include "threading/SerialTypes.h"
 
+#include "Val.h"
+
 namespace input {
 
 class Manager;
@@ -25,6 +26,8 @@ public:
 	/**
 	 * Constructor.
 	 *
+	 * info: The meta information struct for the writer.
+	 *
 	 * type: The backend writer type, with the value corresponding to the
 	 * script-level \c Input::Reader enum (e.g., \a READER_ASCII). The
 	 * frontend will internally instantiate a ReaderBackend of the
@@ -32,7 +35,7 @@ public:
 	 *
 	 * Frontends must only be instantiated by the main thread.
 	 */
-	ReaderFrontend(bro_int_t type);
+	ReaderFrontend(const ReaderBackend::ReaderInfo& info, EnumVal* type);
 
 	/**
 	 * Destructor.
@@ -52,7 +55,7 @@ public:
 	 *
 	 * This method must only be called from the main thread.
 	 */
-	void Init(const ReaderBackend::ReaderInfo& info, const int arg_num_fields, const threading::Field* const* fields);
+	void Init(const int arg_num_fields, const threading::Field* const* fields);
 
 	/**
 	 * Force an update of the current input source. Actual action depends
@@ -100,12 +103,12 @@ public:
 	 *
 	 * This method is safe to call from any thread.
 	 */
-	string Name() const;
+	const char* Name() const;
 
 	/**
 	 * Returns the additional reader information passed into the constructor.
 	 */
-	const ReaderBackend::ReaderInfo& Info() const	{ return info; }
+	const ReaderBackend::ReaderInfo& Info() const	{ assert(info); return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -120,24 +123,14 @@ public:
 protected:
 	friend class Manager;
 
-	/**
-	 * Returns the name of the backend's type.
-	 */
-	const string& TypeName() const	{ return ty_name; }
-
-	/**
-	 * Sets the name of the backend's type.
-	 */
-	void SetTypeName(const string& name)	{ ty_name = name; }
-
 private:
 	ReaderBackend* backend;	// The backend we have instanatiated.
-	ReaderBackend::ReaderInfo info;	// Meta information as passed to Init().
+	ReaderBackend::ReaderInfo* info;	// Meta information.
 	const threading::Field* const*  fields;	// The input fields.
 	int num_fields;		// Information as passed to Init().
-	string ty_name;		// Backend type, set by manager.
 	bool disabled;		// True if disabled.
 	bool initialized;	// True if initialized.
+	const char* name;	// Descriptive name.
 };
 
 }
diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 7f93a3138c..73821d7cb6 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -87,10 +87,10 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
 	{
 	mtime = 0;
 
-	file = new ifstream(info.source.c_str());
+	file = new ifstream(info.source);
 	if ( ! file->is_open() )
 		{
-		Error(Fmt("Init: cannot open %s", info.source.c_str()));
+		Error(Fmt("Init: cannot open %s", info.source));
 		delete(file);
 		file = 0;
 		return false;
@@ -98,7 +98,7 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
 
 	if ( ReadHeader(false) == false )
 		{
-		Error(Fmt("Init: cannot open %s; headers are incorrect", info.source.c_str()));
+		Error(Fmt("Init: cannot open %s; headers are incorrect", info.source));
 		file->close();
 		delete(file);
 		file = 0;
@@ -164,20 +164,20 @@ bool Ascii::ReadHeader(bool useCached)
 				}
 
 			Error(Fmt("Did not find requested field %s in input data file %s.",
-				  field->name.c_str(), Info().source.c_str()));
+				  field->name, Info().source));
 			return false;
 			}
 
 
 		FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]);
 
-		if ( field->secondary_name != "" )
+		if ( field->secondary_name && strlen(field->secondary_name) != 0 )
 			{
 			map<string, uint32_t>::iterator fit2 = ifields.find(field->secondary_name);
 			if ( fit2 == ifields.end() )
 				{
 				Error(Fmt("Could not find requested port type field %s in input data file.",
-					  field->secondary_name.c_str()));
+					  field->secondary_name));
 				return false;
 				}
 
@@ -220,7 +220,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 	switch ( field.type ) {
 	case TYPE_ENUM:
 	case TYPE_STRING:
-		val->val.string_val = new string(s);
+		val->val.string_val.length = s.size();
+		val->val.string_val.data = copy_string(s.c_str());
 		break;
 
 	case TYPE_BOOL:
@@ -367,9 +368,9 @@ bool Ascii::DoUpdate()
 			{
 			// check if the file has changed
 			struct stat sb;
-			if ( stat(Info().source.c_str(), &sb) == -1 )
+			if ( stat(Info().source, &sb) == -1 )
 				{
-				Error(Fmt("Could not get stat for %s", Info().source.c_str()));
+				Error(Fmt("Could not get stat for %s", Info().source));
 				return false;
 				}
 
@@ -403,10 +404,10 @@ bool Ascii::DoUpdate()
 				file = 0;
 				}
 
-			file = new ifstream(Info().source.c_str());
+			file = new ifstream(Info().source);
 			if ( ! file->is_open() )
 				{
-				Error(Fmt("cannot open %s", Info().source.c_str()));
+				Error(Fmt("cannot open %s", Info().source));
 				return false;
 				}
 
diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc
index 28afdc1c89..b8cec0f14d 100644
--- a/src/input/readers/Benchmark.cc
+++ b/src/input/readers/Benchmark.cc
@@ -38,7 +38,7 @@ void Benchmark::DoClose()
 
 bool Benchmark::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
 	{
-	num_lines = atoi(info.source.c_str());
+	num_lines = atoi(info.source);
 
 	if ( autospread != 0.0 )
 		autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
@@ -126,8 +126,12 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 		assert(false); // no enums, please.
 
 	case TYPE_STRING:
-		val->val.string_val = new string(RandomString(10));
+		{
+		string rnd = RandomString(10);
+		val->val.string_val.data = copy_string(rnd.c_str());
+		val->val.string_val.length = rnd.size();
 		break;
+		}
 
 	case TYPE_BOOL:
 		val->val.int_val = 1; // we never lie.
diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc
index f62e966883..ac96e5c0f5 100644
--- a/src/input/readers/Raw.cc
+++ b/src/input/readers/Raw.cc
@@ -108,7 +108,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
 	firstrun = true;
 	bool result;
 
-	if ( info.source.length() == 0 )
+	if ( ! info.source || strlen(info.source) == 0 )
 		{
 		Error("No source path provided");
 		return false;
@@ -129,11 +129,12 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
 		}
 
 	// do Initialization
-	char last = info.source[info.source.length()-1];
+	string source = string(info.source);
+	char last = info.source[source.length() - 1];
 	if ( last == '|' )
 		{
 		execute = true;
-		fname = info.source.substr(0, fname.length() - 1);
+		fname = source.substr(0, fname.length() - 1);
 
 		if ( (info.mode != MODE_MANUAL) )
 			{
@@ -237,7 +238,8 @@ bool Raw::DoUpdate()
 
 		// filter has exactly one text field. convert to it.
 		Value* val = new Value(TYPE_STRING, true);
-		val->val.string_val = new string(line);
+		val->val.string_val.data = copy_string(line.c_str());
+		val->val.string_val.length = line.size();
 		fields[0] = val;
 
 		Put(fields);
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 1808b83738..fd970c48b2 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -6,6 +6,7 @@
 #include "../EventHandler.h"
 #include "../NetVar.h"
 #include "../Net.h"
+#include "../Type.h"
 
 #include "threading/Manager.h"
 #include "threading/SerialTypes.h"
@@ -75,7 +76,7 @@ struct Manager::WriterInfo {
 	double interval;
 	Func* postprocessor;
 	WriterFrontend* writer;
-	WriterBackend::WriterInfo info;
+	WriterBackend::WriterInfo* info;
 	};
 
 struct Manager::Stream {
@@ -118,6 +119,7 @@ Manager::Stream::~Stream()
 
 		Unref(winfo->type);
 		delete winfo->writer;
+		delete winfo->info;
 		delete winfo;
 		}
 
@@ -193,7 +195,6 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
 
 	assert(ld->factory);
 
-	frontend->ty_name = ld->name;
 	WriterBackend* backend = (*ld->factory)(frontend);
 	assert(backend);
 
@@ -476,18 +477,17 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
 			return false;
 			}
 
-		threading::Field* field = new threading::Field();
-		field->name = new_path;
-		field->type = t->Tag();
-		field->optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
+		TypeTag st = TYPE_VOID;
 
-		if ( field->type == TYPE_TABLE )
-			field->subtype = t->AsSetType()->Indices()->PureType()->Tag();
+		if ( t->Tag() == TYPE_TABLE )
+			st = t->AsSetType()->Indices()->PureType()->Tag();
 
-		else if ( field->type == TYPE_VECTOR )
-			field->subtype = t->AsVectorType()->YieldType()->Tag();
+		else if ( t->Tag() == TYPE_VECTOR )
+			st = t->AsVectorType()->YieldType()->Tag();
 
-		filter->fields[filter->num_fields - 1] = field;
+		bool optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
+
+		filter->fields[filter->num_fields - 1] = new threading::Field(new_path.c_str(), 0, t->Tag(), st, optional);
 		}
 
 	return true;
@@ -594,7 +594,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
 		{
 		threading::Field* field = filter->fields[i];
 		DBG_LOG(DBG_LOGGING, "   field %10s: %s",
-			field->name.c_str(), type_name(field->type));
+			field->name, type_name(field->type));
 		}
 #endif
 
@@ -769,9 +769,9 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 			for ( int j = 0; j < filter->num_fields; ++j )
 				arg_fields[j] = new threading::Field(*filter->fields[j]);
 
-			WriterBackend::WriterInfo info;
-			info.path = path;
-			info.network_time = network_time;
+			WriterBackend::WriterInfo* info = new WriterBackend::WriterInfo;
+			info->path = copy_string(path.c_str());
+			info->network_time = network_time;
 
 			HashKey* k;
 			IterCookie* c = filter->config->AsTable()->InitForIteration();
@@ -782,7 +782,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 				ListVal* index = filter->config->RecoverIndex(k);
 				string key = index->Index(0)->AsString()->CheckString();
 				string value = v->Value()->AsString()->CheckString();
-				info.config.insert(std::make_pair(key, value));
+				info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
 				Unref(index);
 				delete k;
 				}
@@ -844,11 +844,16 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 			val->Type()->AsEnumType()->Lookup(val->InternalInt());
 
 		if ( s )
-			lval->val.string_val = new string(s);
+			{
+			lval->val.string_val.data = copy_string(s);
+			lval->val.string_val.length = strlen(s);
+			}
+
 		else
 			{
 			val->Type()->Error("enum type does not contain value", val);
-			lval->val.string_val = new string();
+			lval->val.string_val.data = copy_string("");
+			lval->val.string_val.length = 0;
 			}
 		break;
 		}
@@ -880,15 +885,20 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 	case TYPE_STRING:
 		{
 		const BroString* s = val->AsString();
-		lval->val.string_val =
-			new string((const char*) s->Bytes(), s->Len());
+		char* buf = new char[s->Len()];
+		memcpy(buf, s->Bytes(), s->Len());
+
+		lval->val.string_val.data = buf;
+		lval->val.string_val.length = s->Len();
 		break;
 		}
 
 	case TYPE_FILE:
 		{
 		const BroFile* f = val->AsFile();
-		lval->val.string_val = new string(f->Name());
+		string s = f->Name();
+		lval->val.string_val.data = copy_string(s.c_str());
+		lval->val.string_val.length = s.size();
 		break;
 		}
 
@@ -897,7 +907,9 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 		ODesc d;
 		const Func* f = val->AsFunc();
 		f->Describe(&d);
-		lval->val.string_val = new string(d.Description());
+		const char* s = d.Description();
+		lval->val.string_val.data = copy_string(s);
+		lval->val.string_val.length = strlen(s);
 		break;
 		}
 
@@ -977,7 +989,7 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 	return vals;
 	}
 
-WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
+WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
 				int num_fields, const threading::Field* const*  fields, bool local, bool remote)
 	{
 	Stream* stream = FindStream(id);
@@ -987,7 +999,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 		return 0;
 
 	Stream::WriterMap::iterator w =
-		stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info.path));
+		stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info->path));
 
 	if ( w != stream->writers.end() )
 		// If we already have a writer for this. That's fine, we just
@@ -1013,7 +1025,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 		{
 		Filter* f = *it;
 		if ( f->writer->AsEnum() == writer->AsEnum() &&
-		     f->path == info.path )
+		     f->path == info->path )
 			{
 			found_filter_match = true;
 			winfo->interval = f->interval;
@@ -1030,7 +1042,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 		}
 
 	stream->writers.insert(
-		Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info.path),
+		Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info->path),
 		winfo));
 
 	// Still need to set the WriterInfo's rotation parameters, which we
@@ -1038,11 +1050,11 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 	const char* base_time = log_rotate_base_time ?
 		log_rotate_base_time->AsString()->CheckString() : 0;
 
-	winfo->info.rotation_interval = winfo->interval;
-	winfo->info.rotation_base = parse_rotate_base_time(base_time);
+	winfo->info->rotation_interval = winfo->interval;
+	winfo->info->rotation_base = parse_rotate_base_time(base_time);
 
-	winfo->writer = new WriterFrontend(id, writer, local, remote);
-	winfo->writer->Init(winfo->info, num_fields, fields);
+	winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote);
+	winfo->writer->Init(num_fields, fields);
 
 	InstallRotationTimer(winfo);
 
@@ -1124,7 +1136,7 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
 			EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
 			remote_serializer->SendLogCreateWriter(peer, (*s)->id,
 							       &writer_val,
-							       i->second->info,
+							       *i->second->info,
 							       writer->NumFields(),
 							       writer->Fields());
 			}
@@ -1260,14 +1272,14 @@ void Manager::InstallRotationTimer(WriterInfo* winfo)
 		timer_mgr->Add(winfo->rotation_timer);
 
 		DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f",
-			winfo->writer->Name().c_str(), winfo->rotation_timer->Time());
+			winfo->writer->Name(), winfo->rotation_timer->Time());
 		}
 	}
 
 void Manager::Rotate(WriterInfo* winfo)
 	{
 	DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f",
-		winfo->writer->Name().c_str(), network_time);
+		winfo->writer->Name(), network_time);
 
 	// Build a temporary path for the writer to move the file to.
 	struct tm tm;
@@ -1278,15 +1290,14 @@ void Manager::Rotate(WriterInfo* winfo)
 	localtime_r(&teatime, &tm);
 	strftime(buf, sizeof(buf), date_fmt, &tm);
 
-	string tmp = string(fmt("%s-%s", winfo->writer->Info().path.c_str(), buf));
-
 	// Trigger the rotation.
+	const char* tmp = fmt("%s-%s", winfo->writer->Info().path, buf);
 	winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
 
 	++rotations_pending;
 	}
 
-bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
+bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
 		      double open, double close, bool terminating)
 	{
 	--rotations_pending;
@@ -1296,7 +1307,7 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
 		return true;
 
 	DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
-		writer->Name().c_str(), network_time, new_name.c_str());
+		writer->Name(), network_time, new_name);
 
 	WriterInfo* winfo = FindWriter(writer);
 	if ( ! winfo )
@@ -1305,8 +1316,8 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
 	// Create the RotationInfo record.
 	RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
 	info->Assign(0, winfo->type->Ref());
-	info->Assign(1, new StringVal(new_name.c_str()));
-	info->Assign(2, new StringVal(winfo->writer->Info().path.c_str()));
+	info->Assign(1, new StringVal(new_name));
+	info->Assign(2, new StringVal(winfo->writer->Info().path));
 	info->Assign(3, new Val(open, TYPE_TIME));
 	info->Assign(4, new Val(close, TYPE_TIME));
 	info->Assign(5, new Val(terminating, TYPE_BOOL));
diff --git a/src/logging/Manager.h b/src/logging/Manager.h
index 38dd9258b3..ae7a1796ba 100644
--- a/src/logging/Manager.h
+++ b/src/logging/Manager.h
@@ -162,8 +162,8 @@ protected:
 
 	//// Function also used by the RemoteSerializer.
 
-	// Takes ownership of fields.
-	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
+	// Takes ownership of fields and info.
+	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
 				int num_fields, const threading::Field* const* fields,
 				bool local, bool remote);
 
@@ -175,7 +175,7 @@ protected:
 	void SendAllWritersTo(RemoteSerializer::PeerID peer);
 
 	// Signals that a file has been rotated.
-	bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
+	bool FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
 			      double open, double close, bool terminating);
 
 	// Deletes the values as passed into Write().
diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index a284c56201..8f119d6f8f 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -18,20 +18,26 @@ namespace logging  {
 class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
 {
 public:
-	RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
+	RotationFinishedMessage(WriterFrontend* writer, const char* new_name, const char* old_name,
 				double open, double close, bool terminating)
 		: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
-		new_name(new_name), old_name(old_name), open(open),
+		new_name(copy_string(new_name)), old_name(copy_string(old_name)), open(open),
 		close(close), terminating(terminating)	{ }
 
+	virtual ~RotationFinishedMessage()
+		{
+		delete [] new_name;
+		delete [] old_name;
+		}
+
 	virtual bool Process()
 		{
 		return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating);
 		}
 
 private:
-        string new_name;
-        string old_name;
+        const char* new_name;
+        const char* old_name;
         double open;
         double close;
         bool terminating;
@@ -65,12 +71,16 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
 	{
 	int size;
 
-	if ( ! (fmt->Read(&path, "path") &&
+	string tmp_path;
+
+	if ( ! (fmt->Read(&tmp_path, "path") &&
 		fmt->Read(&rotation_base, "rotation_base") &&
 		fmt->Read(&rotation_interval, "rotation_interval") &&
 		fmt->Read(&size, "config_size")) )
 		return false;
 
+	path = copy_string(tmp_path.c_str());
+
 	config.clear();
 
 	while ( size )
@@ -81,7 +91,7 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
 		if ( ! (fmt->Read(&value, "config-value") && fmt->Read(&value, "config-key")) )
 			return false;
 
-		config.insert(std::make_pair(value, key));
+		config.insert(std::make_pair(copy_string(value.c_str()), copy_string(key.c_str())));
 		}
 
 	return true;
@@ -98,7 +108,7 @@ bool WriterBackend::WriterInfo::Write(SerializationFormat* fmt) const
 		fmt->Write(size, "config_size")) )
 		return false;
 
-	for ( config_map::const_iterator i = config.begin(); i != config.end(); ++i ) 
+	for ( config_map::const_iterator i = config.begin(); i != config.end(); ++i )
 		{
 		if ( ! (fmt->Write(i->first, "config-value") && fmt->Write(i->second, "config-key")) )
 			return false;
@@ -113,8 +123,7 @@ WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
 	fields = 0;
 	buffering = true;
 	frontend = arg_frontend;
-
-	info.path = "<path not yet set>";
+	info = new WriterInfo(frontend->Info());
 
 	SetName(frontend->Name());
 	}
@@ -128,6 +137,8 @@ WriterBackend::~WriterBackend()
 
 		delete [] fields;
 		}
+
+	delete info;
 	}
 
 void WriterBackend::DeleteVals(int num_writes, Value*** vals)
@@ -144,7 +155,7 @@ void WriterBackend::DeleteVals(int num_writes, Value*** vals)
 	delete [] vals;
 	}
 
-bool WriterBackend::FinishedRotation(string new_name, string old_name,
+bool WriterBackend::FinishedRotation(const char* new_name, const char* old_name,
 				     double open, double close, bool terminating)
 	{
 	SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating));
@@ -156,15 +167,12 @@ void WriterBackend::DisableFrontend()
 	SendOut(new DisableMessage(frontend));
 	}
 
-bool WriterBackend::Init(const WriterInfo& arg_info, int arg_num_fields, const Field* const* arg_fields, const string& frontend_name)
+bool WriterBackend::Init(int arg_num_fields, const Field* const* arg_fields)
 	{
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
-	SetName(frontend->Name());
-
-	if ( ! DoInit(arg_info, arg_num_fields, arg_fields) )
+	if ( ! DoInit(*info, arg_num_fields, arg_fields) )
 		{
 		DisableFrontend();
 		return false;
@@ -246,7 +254,7 @@ bool WriterBackend::SetBuf(bool enabled)
 	return true;
 	}
 
-bool WriterBackend::Rotate(string rotated_path, double open,
+bool WriterBackend::Rotate(const char* rotated_path, double open,
 			   double close, bool terminating)
 	{
 	if ( ! DoRotate(rotated_path, open, close, terminating) )
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index 33cde8679e..a59cd1893e 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -48,14 +48,17 @@ public:
 	 */
 	struct WriterInfo
 		{
-		typedef std::map<string, string> config_map;
+		// Structure takes ownership of these strings.
+		typedef std::map<const char*, const char*> config_map;
 
 		/**
 		 * A string left to the interpretation of the writer
 		 * implementation; it corresponds to the 'path' value configured
 		 * on the script-level for the logging filter.
+		 *
+		 * Structure takes ownership of string.
 		 */
-		string path;
+		const char* path;
 
 		/**
 		 * The rotation interval as configured for this writer.
@@ -76,9 +79,38 @@ public:
 		 * A map of key/value pairs corresponding to the relevant
 		 * filter's "config" table.
 		 */
-		std::map<string, string> config;
+		config_map config;
+
+		WriterInfo()
+			{
+			path = 0;
+			}
+
+		WriterInfo(const WriterInfo& other)
+			{
+			path = other.path ? copy_string(other.path) : 0;
+			rotation_interval = other.rotation_interval;
+			rotation_base = other.rotation_base;
+			network_time = other.network_time;
+
+			for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
+				config.insert(std::make_pair(copy_string(i->first), copy_string(i->second)));
+			}
+
+		~WriterInfo()
+			{
+			delete [] path;
+
+			for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
+				{
+				delete [] i->first;
+				delete [] i->second;
+				}
+			}
 
 		private:
+		const WriterInfo& operator=(const WriterInfo& other); // Disable.
+
 		friend class ::RemoteSerializer;
 
 		// Note, these need to be adapted when changing the struct's
@@ -90,7 +122,6 @@ public:
 	/**
 	 * One-time initialization of the writer to define the logged fields.
 	 *
-	 * @param info Meta information for the writer.
 	 * @param num_fields
 	 *
 	 * @param fields An array of size \a num_fields with the log fields.
@@ -100,7 +131,7 @@ public:
 	 *
 	 * @return False if an error occured.
 	 */
-	bool Init(const WriterInfo& info, int num_fields, const threading::Field* const* fields, const string& frontend_name);
+	bool Init(int num_fields, const threading::Field* const* fields);
 
 	/**
 	 * Writes one log entry.
@@ -146,7 +177,7 @@ public:
 	 *
 	 * @return False if an error occured.
 	 */
-	bool Rotate(string rotated_path, double open, double close, bool terminating);
+	bool Rotate(const char* rotated_path, double open, double close, bool terminating);
 
 	/**
 	 * Disables the frontend that has instantiated this backend. Once
@@ -157,7 +188,7 @@ public:
 	/**
 	 * Returns the additional writer information passed into the constructor.
 	 */
-	const WriterInfo& Info() const	{ return info; }
+	const WriterInfo& Info() const	{ return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -193,7 +224,7 @@ public:
 	 * @param terminating: True if the original rotation request occured
 	 * due to the main Bro process shutting down.
 	 */
-	bool FinishedRotation(string new_name, string old_name,
+	bool FinishedRotation(const char* new_name, const char* old_name,
 			      double open, double close, bool terminating);
 
 	/** Helper method to render an IP address as a string.
@@ -322,7 +353,7 @@ protected:
 	 * due the main Bro prcoess terminating (and not because we've
 	 * reached a regularly scheduled time for rotation).
 	 */
-	virtual bool DoRotate(string rotated_path, double open, double close,
+	virtual bool DoRotate(const char* rotated_path, double open, double close,
 			      bool terminating) = 0;
 
 	/**
@@ -351,7 +382,7 @@ private:
 	// this class, it's running in a different thread!
 	WriterFrontend* frontend;
 
-	WriterInfo info;	// Meta information as passed to Init().
+	const WriterInfo* info;	// Meta information.
 	int num_fields;	// Number of log fields.
 	const threading::Field* const*  fields;	// Log fields.
 	bool buffering;	// True if buffering is enabled.
diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc
index b816327e9c..fc237d6f6e 100644
--- a/src/logging/WriterFrontend.cc
+++ b/src/logging/WriterFrontend.cc
@@ -16,35 +16,36 @@ namespace logging  {
 class InitMessage : public threading::InputMessage<WriterBackend>
 {
 public:
-	InitMessage(WriterBackend* backend, const WriterBackend::WriterInfo& info, const int num_fields, const Field* const* fields, const string& frontend_name)
+	InitMessage(WriterBackend* backend, const int num_fields, const Field* const* fields)
 		: threading::InputMessage<WriterBackend>("Init", backend),
-		info(info), num_fields(num_fields), fields(fields),
-		frontend_name(frontend_name) { }
+		num_fields(num_fields), fields(fields)
+			{}
 
-	virtual bool Process() { return Object()->Init(info, num_fields, fields, frontend_name); }
+
+	virtual bool Process() { return Object()->Init(num_fields, fields); }
 
 private:
-	WriterBackend::WriterInfo info;
 	const int num_fields;
 	const Field * const* fields;
-	const string frontend_name;
 };
 
 class RotateMessage : public threading::InputMessage<WriterBackend>
 {
 public:
-	RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open,
+	RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const char* rotated_path, const double open,
 		      const double close, const bool terminating)
 		: threading::InputMessage<WriterBackend>("Rotate", backend),
 		frontend(frontend),
-		rotated_path(rotated_path), open(open),
+		rotated_path(copy_string(rotated_path)), open(open),
 		close(close), terminating(terminating) { }
 
+	virtual ~RotateMessage()	{ delete [] rotated_path; }
+
 	virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); }
 
 private:
 	WriterFrontend* frontend;
-	const string rotated_path;
+	const char* rotated_path;
 	const double open;
 	const double close;
 	const bool terminating;
@@ -96,7 +97,7 @@ private:
 
 using namespace logging;
 
-WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
+WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
 	{
 	stream = arg_stream;
 	writer = arg_writer;
@@ -109,7 +110,10 @@ WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool ar
 	remote = arg_remote;
 	write_buffer = 0;
 	write_buffer_pos = 0;
-	ty_name = "<not set>";
+	info = new WriterBackend::WriterInfo(arg_info);
+
+	const char* w = arg_writer->Type()->AsEnumType()->Lookup(arg_stream->InternalInt());
+	name = copy_string(fmt("%s/%s", arg_info.path, w));
 
 	if ( local )
 		{
@@ -127,14 +131,7 @@ WriterFrontend::~WriterFrontend()
 	{
 	Unref(stream);
 	Unref(writer);
-	}
-
-string WriterFrontend::Name() const
-	{
-	if ( ! info.path.size() )
-		return ty_name;
-
-	return ty_name + "/" + info.path;
+	delete info;
 	}
 
 void WriterFrontend::Stop()
@@ -143,7 +140,7 @@ void WriterFrontend::Stop()
 	SetDisable();
 	}
 
-void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num_fields, const Field* const * arg_fields)
+void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields)
 	{
 	if ( disabled )
 		return;
@@ -151,19 +148,18 @@ void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num
 	if ( initialized )
 		reporter->InternalError("writer initialize twice");
 
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
 	initialized = true;
 
 	if ( backend )
-		backend->SendIn(new InitMessage(backend, arg_info, arg_num_fields, arg_fields, Name()));
+		backend->SendIn(new InitMessage(backend, arg_num_fields, arg_fields));
 
 	if ( remote )
 		remote_serializer->SendLogCreateWriter(stream,
 						       writer,
-						       arg_info,
+						       *info,
 						       arg_num_fields,
 						       arg_fields);
 
@@ -177,7 +173,7 @@ void WriterFrontend::Write(int num_fields, Value** vals)
 	if ( remote )
 		remote_serializer->SendLogWrite(stream,
 						writer,
-						info.path,
+						info->path,
 						num_fields,
 						vals);
 
@@ -242,7 +238,7 @@ void WriterFrontend::Flush(double network_time)
 		backend->SendIn(new FlushMessage(backend, network_time));
 	}
 
-void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
+void WriterFrontend::Rotate(const char* rotated_path, double open, double close, bool terminating)
 	{
 	if ( disabled )
 		return;
diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h
index e8f3d06d6c..549d602bd5 100644
--- a/src/logging/WriterFrontend.h
+++ b/src/logging/WriterFrontend.h
@@ -31,6 +31,10 @@ public:
 	 * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The
 	 * frontend will internally instantiate a WriterBackend of the
 	 * corresponding type.
+	 *
+	 * info: The meta information struct for the writer.
+	 *
+	 * writer_name: A descriptive name for the writer's type.
 	 * 
 	 * local: If true, the writer will instantiate a local backend.
 	 *
@@ -39,7 +43,7 @@ public:
 	 *
 	 * Frontends must only be instantiated by the main thread.
 	 */
-	WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote);
+	WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote);
 
 	/**
 	 * Destructor.
@@ -68,7 +72,7 @@ public:
 	 *
 	 * This method must only be called from the main thread.
 	 */
-	void Init(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const*  fields);
+	void Init(int num_fields, const threading::Field* const*  fields);
 
 	/**
 	 * Write out a record.
@@ -130,7 +134,7 @@ public:
 	 *
 	 * This method must only be called from the main thread.
 	 */
-	void Rotate(string rotated_path, double open, double close, bool terminating);
+	void Rotate(const char* rotated_path, double open, double close, bool terminating);
 
 	/**
 	 * Finalizes writing to this tream.
@@ -175,7 +179,7 @@ public:
 	/**
 	 * Returns the additional writer information as passed into the constructor.
 	 */
-	const WriterBackend::WriterInfo& Info() const	{ return info; }
+	const WriterBackend::WriterInfo& Info() const	{ return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -188,7 +192,7 @@ public:
 	 *
 	 * This method is safe to call from any thread.
 	 */
-	string Name() const;
+	const char* Name() const	{ return name; }
 
 	/**
 	 * Returns the log fields as passed into the constructor.
@@ -210,8 +214,8 @@ protected:
 	bool local;	// True if logging locally.
 	bool remote;	// True if loggin remotely.
 
-	string ty_name;	// Name of the backend type. Set by the manager.
-	WriterBackend::WriterInfo info;	// The writer information.
+	const char* name;	// Descriptive name of the
+	WriterBackend::WriterInfo* info;	// The writer information.
 	int num_fields;	// The number of log fields.
 	const threading::Field* const*  fields;	// The log fields.
 
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 20963d1535..99fd3f3c6e 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -52,6 +52,8 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 
 Ascii::~Ascii()
 	{
+	//fprintf(stderr, "DTOR %p\n", this);
+
 	// Normally, the file will be closed here already via the Finish()
 	// message. But when we terminate abnormally, we may still have it open.
 	if ( fd )
@@ -78,7 +80,10 @@ void Ascii::CloseFile(double t)
 		return;
 
 	if ( include_meta )
-		WriteHeaderField("end", t ? Timestamp(t) : "<abnormal termination>");
+		{
+		string ts = t ? Timestamp(t) : string("<abnormal termination>");
+		WriteHeaderField("end", ts);
+		}
 
 	close(fd);
 	fd = 0;
@@ -118,6 +123,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 		if ( ! safe_write(fd, str.c_str(), str.length()) )
 			goto write_error;
 
+		string ts = Timestamp(info.network_time);
+
 		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
 		            string(set_separator, set_separator_len), false)) &&
 		        WriteHeaderField("empty_field", get_escaped_string(
@@ -125,8 +132,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 		        WriteHeaderField("unset_field", get_escaped_string(
 		            string(unset_field, unset_field_len), false)) &&
 		        WriteHeaderField("path", get_escaped_string(path, false)) &&
-			WriteHeaderField("start", Timestamp(info.network_time))) )
-		goto write_error;
+		        WriteHeaderField("start", ts)) )
+			goto write_error;
 
 		for ( int i = 0; i < num_fields; ++i )
 			{
@@ -136,8 +143,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 				types += string(separator, separator_len);
 				}
 
-			names += fields[i]->name;
-			types += fields[i]->TypeName();
+			names += string(fields[i]->name);
+			types += fields[i]->TypeName().c_str();
 			}
 
 		if ( ! (WriteHeaderField("fields", names)
@@ -229,8 +236,8 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 	case TYPE_FILE:
 	case TYPE_FUNC:
 		{
-		int size = val->val.string_val->size();
-		const char* data = val->val.string_val->data();
+		int size = val->val.string_val.length;
+		const char* data = val->val.string_val.data;
 
 		if ( ! size )
 			{
@@ -311,8 +318,7 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 		}
 
 	default:
-		Error(Fmt("unsupported field format %d for %s", val->type,
-			  field->name.c_str()));
+		Error(Fmt("unsupported field format %d for %s", val->type, field->name));
 		return false;
 	}
 
@@ -366,7 +372,7 @@ write_error:
 	return false;
 	}
 
-bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
+bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool terminating)
 	{
 	// Don't rotate special files or if there's not one currently open.
 	if ( ! fd || IsSpecial(Info().path) )
@@ -374,10 +380,10 @@ bool Ascii::DoRotate(string rotated_path, double open, double close, bool termin
 
 	CloseFile(close);
 
-	string nname = rotated_path + "." + LogExt();
+	string nname = string(rotated_path) + "." + LogExt();
 	rename(fname.c_str(), nname.c_str());
 
-	if ( ! FinishedRotation(nname, fname, open, close, terminating) )
+	if ( ! FinishedRotation(nname.c_str(), fname.c_str(), open, close, terminating) )
 		{
 		Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
 		return false;
@@ -401,19 +407,22 @@ bool Ascii::DoHeartbeat(double network_time, double current_time)
 string Ascii::LogExt()
 	{
 	const char* ext = getenv("BRO_LOG_SUFFIX");
-	if ( ! ext ) ext = "log";
+	if ( ! ext )
+		ext = "log";
+
 	return ext;
 	}
 
 string Ascii::Timestamp(double t)
 	{
-	struct tm tm;
-	char buf[128];
-	const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
 	time_t teatime = time_t(t);
 
-	localtime_r(&teatime, &tm);
-	strftime(buf, sizeof(buf), date_fmt, &tm);
+	struct tm tmbuf;
+	struct tm* tm = localtime_r(&teatime, &tmbuf);
+
+	char buf[128];
+	const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
+	strftime(buf, sizeof(buf), date_fmt, tm);
 	return buf;
 	}
 
diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h
index 371ded4344..cb82860cb7 100644
--- a/src/logging/writers/Ascii.h
+++ b/src/logging/writers/Ascii.h
@@ -24,7 +24,7 @@ protected:
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, double open,
+	virtual bool DoRotate(const char* rotated_path, double open,
 			      double close, bool terminating);
 	virtual bool DoFlush(double network_time);
 	virtual bool DoFinish(double network_time);
diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc
index 1978a8b781..7d3053e341 100644
--- a/src/logging/writers/DataSeries.cc
+++ b/src/logging/writers/DataSeries.cc
@@ -78,10 +78,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
 	case TYPE_STRING:
 	case TYPE_FILE:
 	case TYPE_FUNC:
-		if ( ! val->val.string_val->size() )
+		if ( ! val->val.string_val.length )
 			return "";
 
-		return string(val->val.string_val->data(), val->val.string_val->size());
+		return string(val->val.string_val.data, val->val.string_val.length);
 
 	case TYPE_TABLE:
 		{
@@ -302,7 +302,8 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
 
 	if( ds_dump_schema )
 		{
-		FILE* pFile = fopen ( string(info.path + ".ds.xml").c_str() , "wb" );
+		string name = string(info.path) + ".ds.xml";
+		FILE* pFile = fopen(name.c_str(), "wb" );
 
 		if( pFile )
 			{
@@ -394,17 +395,17 @@ bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
 	return true;
 }
 
-bool DataSeries::DoRotate(string rotated_path, double open, double close, bool terminating)
+bool DataSeries::DoRotate(const char* rotated_path, double open, double close, bool terminating)
 {
 	// Note that if DS files are rotated too often, the aggregate log
 	// size will be (much) larger.
 	CloseLog();
 
-	string dsname = Info().path + ".ds";
-	string nname = rotated_path + ".ds";
+	string dsname = string(Info().path) + ".ds";
+	string nname = string(rotated_path) + ".ds";
 	rename(dsname.c_str(), nname.c_str());
 
-	if ( ! FinishedRotation(nname, dsname, open, close, terminating) )
+	if ( ! FinishedRotation(nname.c_str(), dsname.c_str(), open, close, terminating) )
 		{
 		Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
 		return false;
diff --git a/src/logging/writers/DataSeries.h b/src/logging/writers/DataSeries.h
index 31d17a1a7b..9773c7ce1b 100644
--- a/src/logging/writers/DataSeries.h
+++ b/src/logging/writers/DataSeries.h
@@ -32,7 +32,7 @@ protected:
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, double open,
+	virtual bool DoRotate(const char* rotated_path, double open,
 			      double close, bool terminating);
 	virtual bool DoFlush(double network_time);
 	virtual bool DoFinish(double network_time);
diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc
index acf9355cf7..9b91b82199 100644
--- a/src/logging/writers/None.cc
+++ b/src/logging/writers/None.cc
@@ -1,4 +1,6 @@
 
+#include <algorithm>
+
 #include "None.h"
 #include "NetVar.h"
 
@@ -15,8 +17,17 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
 		std::cout << "  rotation_interval=" << info.rotation_interval << std::endl;
 		std::cout << "  rotation_base=" << info.rotation_base << std::endl;
 
-		for ( std::map<string,string>::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
-			std::cout << "  config[" << i->first << "] = " << i->second << std::endl;
+		// Output the config sorted by keys.
+
+		std::vector<std::pair<string, string> > keys;
+
+		for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
+			keys.push_back(std::make_pair(i->first, i->second));
+
+		std::sort(keys.begin(), keys.end());
+
+		for ( std::vector<std::pair<string,string> >::const_iterator i = keys.begin(); i != keys.end(); i++ )
+			std::cout << "  config[" << (*i).first << "] = " << (*i).second << std::endl;
 
 		for ( int i = 0; i < num_fields; i++ )
 			{
@@ -31,11 +42,11 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
 	return true;
 	}
 
-bool None::DoRotate(string rotated_path, double open, double close, bool terminating)
+bool None::DoRotate(const char* rotated_path, double open, double close, bool terminating)
 	{
-	if ( ! FinishedRotation(string("/dev/null"), Info().path, open, close, terminating))
+	if ( ! FinishedRotation("/dev/null", Info().path, open, close, terminating))
 		{
-		Error(Fmt("error rotating %s", Info().path.c_str()));
+		Error(Fmt("error rotating %s", Info().path));
 		return false;
 		}
 
diff --git a/src/logging/writers/None.h b/src/logging/writers/None.h
index c6d7cba56a..2a6f71a06a 100644
--- a/src/logging/writers/None.h
+++ b/src/logging/writers/None.h
@@ -24,7 +24,7 @@ protected:
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals)	{ return true; }
 	virtual bool DoSetBuf(bool enabled)	{ return true; }
-	virtual bool DoRotate(string rotated_path, double open,
+	virtual bool DoRotate(const char* rotated_path, double open,
 			      double close, bool terminating);
 	virtual bool DoFlush(double network_time)	{ return true; }
 	virtual bool DoFinish(double network_time)	{ return true; }
diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index e7fb3f4c84..af57c26939 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -12,18 +12,23 @@
 
 using namespace threading;
 
+static const int STD_FMT_BUF_LEN = 2048;
+
 uint64_t BasicThread::thread_counter = 0;
 
 BasicThread::BasicThread()
 	{
 	started = false;
 	terminating = false;
+	killed = false;
 	pthread = 0;
 
-	buf_len = 2048;
+	buf_len = STD_FMT_BUF_LEN;
 	buf = (char*) malloc(buf_len);
 
-	name = Fmt("thread-%d", ++thread_counter);
+	strerr_buffer = 0;
+
+	name = copy_string(fmt("thread-%" PRIu64, ++thread_counter));
 
 	thread_mgr->AddThread(this);
 	}
@@ -32,31 +37,41 @@ BasicThread::~BasicThread()
 	{
         if ( buf )
 		free(buf);
+
+	delete [] name;
+	delete [] strerr_buffer;
 	}
 
-void BasicThread::SetName(const string& arg_name)
+void BasicThread::SetName(const char* name)
 	{
-	// Slight race condition here with reader threads, but shouldn't matter.
-	name = arg_name;
+	delete [] name;
+	name = copy_string(name);
 	}
 
-void BasicThread::SetOSName(const string& name)
+void BasicThread::SetOSName(const char* name)
 	{
 #ifdef HAVE_LINUX
-	prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
+	prctl(PR_SET_NAME, name, 0, 0, 0);
 #endif
 
 #ifdef __APPLE__
-	pthread_setname_np(name.c_str());
+	pthread_setname_np(name);
 #endif
 
 #ifdef FREEBSD
-	pthread_set_name_np(pthread_self(), name, name.c_str());
+	pthread_set_name_np(pthread_self(), name, name);
 #endif
 	}
 
 const char* BasicThread::Fmt(const char* format, ...)
 	{
+	if ( buf_len > 10 * STD_FMT_BUF_LEN )
+		{
+		// Shrink back to normal.
+		buf = (char*) safe_realloc(buf, STD_FMT_BUF_LEN);
+		buf_len = STD_FMT_BUF_LEN;
+		}
+
 	va_list al;
 	va_start(al, format);
 	int n = safe_vsnprintf(buf, buf_len, format, al);
@@ -64,15 +79,13 @@ const char* BasicThread::Fmt(const char* format, ...)
 
 	if ( (unsigned int) n >= buf_len )
 		{ // Not enough room, grow the buffer.
-		int tmp_len = n + 32;
-		char* tmp = (char*) malloc(tmp_len);
+		buf_len = n + 32;
+		buf = (char*) safe_realloc(buf, buf_len);
 
 		// Is it portable to restart?
 		va_start(al, format);
-		n = safe_vsnprintf(tmp, tmp_len, format, al);
+		n = safe_vsnprintf(buf, buf_len, format, al);
 		va_end(al);
-
-		free(tmp);
 		}
 
 	return buf;
@@ -94,14 +107,14 @@ void BasicThread::Start()
 
 	int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
 	if ( err != 0 )
-		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), Strerror(err));
+		reporter->FatalError("Cannot create thread %s: %s", name, Strerror(err));
 
-	DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
+	DBG_LOG(DBG_THREADING, "Started thread %s", name);
 
 	OnStart();
 	}
 
-void BasicThread::Stop()
+void BasicThread::PrepareStop()
 	{
 	if ( ! started )
 		return;
@@ -109,11 +122,28 @@ void BasicThread::Stop()
 	if ( terminating )
 		return;
 
-	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
+	DBG_LOG(DBG_THREADING, "Preparing thread %s to terminate ...", name);
 
-	terminating = true;
+	OnPrepareStop();
+	}
+
+void BasicThread::Stop()
+	{
+	// XX fprintf(stderr, "stop1 %s %d %d\n", name, started, terminating);
+
+	if ( ! started )
+		return;
+
+	if ( terminating )
+		return;
+
+	// XX fprintf(stderr, "stop2 %s\n", name);
+
+	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name);
 
 	OnStop();
+
+	terminating = true;
 	}
 
 void BasicThread::Join()
@@ -123,25 +153,33 @@ void BasicThread::Join()
 
 	assert(terminating);
 
-	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
+	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name);
 
 	if ( pthread && pthread_join(pthread, 0) != 0  )
-		reporter->FatalError("Failure joining thread %s", name.c_str());
+		reporter->FatalError("Failure joining thread %s", name);
 
-	DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
+	DBG_LOG(DBG_THREADING, "Joined with thread %s", name);
 
 	pthread = 0;
 	}
 
 void BasicThread::Kill()
 	{
+	// We don't *really* kill the thread here because that leads to race
+	// conditions. Instead we set a flag that parts of the the code need
+	// to check and get out of any loops they might be in.
 	terminating = true;
+	killed = true;
+	OnKill();
+	}
 
-	if ( ! (started && pthread) )
-		return;
+void BasicThread::Done()
+	{
+	// XX fprintf(stderr, "DONE from thread %s\n", name);
+	DBG_LOG(DBG_THREADING, "Thread %s has finished", name);
 
-	pthread = 0;
-	pthread_kill(pthread, SIGTERM);
+	terminating = true;
+	killed = true;
 	}
 
 void* BasicThread::launcher(void *arg)
@@ -161,11 +199,12 @@ void* BasicThread::launcher(void *arg)
 	sigdelset(&mask_set, SIGSEGV);
 	sigdelset(&mask_set, SIGBUS);
 	int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
-	assert(res == 0);  //
+	assert(res == 0);
 
 	// Run thread's main function.
 	thread->Run();
 
+	thread->Done();
+
 	return 0;
 	}
-
diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h
index d47eb5c3c3..037420b077 100644
--- a/src/threading/BasicThread.h
+++ b/src/threading/BasicThread.h
@@ -5,7 +5,6 @@
 #include <pthread.h>
 #include <semaphore.h>
 
-#include "Queue.h"
 #include "util.h"
 
 using namespace std;
@@ -42,22 +41,25 @@ public:
 	 *
 	 * This method is safe to call from any thread.
 	 */
-	const string& Name() const { return name; }
+	const char* Name() const { return name; }
 
 	/**
 	* Sets a descriptive name for the thread. This should be a string
 	* that's useful in output presented to the user and uniquely
 	* identifies the thread.
 	*
-	* This method must be called only from the thread itself.
+	* This method must be called only from main thread at initialization
+	* time.
 	*/
-	void SetName(const string& name);
+	void SetName(const char* name);
 
 	/**
 	 * Set the name shown by the OS as the thread's description. Not
 	 * supported on all OSs.
+	 *
+	 * Must be called only from the child thread.
 	 */
-	void SetOSName(const string& name);
+	void SetOSName(const char* name);
 
 	/**
 	 * Starts the thread. Calling this methods will spawn a new OS thread
@@ -68,6 +70,18 @@ public:
 	 */
 	void Start();
 
+	/**
+	 * Signals the thread to prepare for stopping. This must be called
+	 * before Stop() and allows the thread to trigger shutting down
+	 * without yet blocking for doing so.
+	 *
+	 * Calling this method has no effect if Start() hasn't been executed
+	 * yet.
+	 *
+	 * Only Bro's main thread must call this method.
+	 */
+	void PrepareStop();
+
 	/**
 	 * Signals the thread to stop. The method lets Terminating() now
 	 * return true. It does however not force the thread to terminate.
@@ -88,6 +102,13 @@ public:
 	 */
 	bool Terminating()  const { return terminating; }
 
+	/**
+	 * Returns true if Kill() has been called.
+	 *
+	 * This method is safe to call from any thread.
+	 */
+	bool Killed()  const { return killed; }
+
 	/**
 	 * A version of fmt() that the thread can safely use.
 	 *
@@ -124,12 +145,24 @@ protected:
 	virtual void OnStart()	{}
 
 	/**
-	 * Executed with Stop(). This is a hook into stopping the thread. It
-	 * will be called from Bro's main thread after the thread has been
-	 * signaled to stop.
+	 * Executed with PrepareStop() (and before OnStop()). This is a hook
+	 * into preparing the thread for stopping. It will be called from
+	 * Bro's main thread before the thread has been signaled to stop.
+	 */
+	virtual void OnPrepareStop()	{}
+
+	/**
+	 * Executed with Stop() (and after OnPrepareStop()). This is a hook
+	 * into stopping the thread. It will be called from Bro's main thread
+	 * after the thread has been signaled to stop.
 	 */
 	virtual void OnStop()	{}
 
+	/**
+	 * Executed with Kill(). This is a hook into killing the thread.
+	 */
+	virtual void OnKill()	{}
+
 	/**
 	 * Destructor. This will be called by the manager.
 	 *
@@ -153,14 +186,18 @@ protected:
 	 */
 	void Kill();
 
+	/** Called by child thread's launcher when it's done processing. */
+	void Done();
+
 private:
 	// pthread entry function.
 	static void* launcher(void *arg);
 
-	string name;
+	const char* name;
 	pthread_t pthread;
 	bool started; 		// Set to to true once running.
 	bool terminating;	// Set to to true to signal termination.
+	bool killed;	// Set to true once forcefully killed.
 
 	// Used as a semaphore to tell the pthread thread when it may
 	// terminate.
diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc
index f1f9307b03..b997aeec47 100644
--- a/src/threading/Manager.cc
+++ b/src/threading/Manager.cc
@@ -30,6 +30,10 @@ void Manager::Terminate()
 	do Process(); while ( did_process );
 
 	// Signal all to stop.
+
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		(*i)->PrepareStop();
+
 	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
 		(*i)->Stop();
 
@@ -50,14 +54,14 @@ void Manager::Terminate()
 
 void Manager::AddThread(BasicThread* thread)
 	{
-	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
+	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name());
 	all_threads.push_back(thread);
 	idle = false;
 	}
 
 void Manager::AddMsgThread(MsgThread* thread)
 	{
-	DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str());
+	DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name());
 	msg_threads.push_back(thread);
 	}
 
@@ -114,6 +118,12 @@ void Manager::Process()
 			{
 			Message* msg = t->RetrieveOut();
 
+			if ( ! msg )
+				{
+				assert(t->Killed());
+				break;
+				}
+
 			if ( msg->Process() )
 				{
 				if ( network_time )
@@ -122,10 +132,9 @@ void Manager::Process()
 
 			else
 				{
-				string s = msg->Name() + " failed, terminating thread";
-				reporter->Error("%s", s.c_str());
+				reporter->Error("%s failed, terminating thread", msg->Name());
 				t->Stop();
-			}
+				}
 
 			delete msg;
 			}
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 3913624654..3e06a3fe1e 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -29,16 +29,6 @@ private:
 	double network_time;
 };
 
-// A dummy message that's only purpose is unblock the current read operation
-// so that the child's Run() methods can check the termination status.
-class UnblockMessage : public InputMessage<MsgThread>
-{
-public:
-	UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread)	{ }
-
-	virtual bool Process()	{ return true; }
-};
-
 /// Sends a heartbeat to the child thread.
 class HeartbeatMessage : public InputMessage<MsgThread>
 {
@@ -66,14 +56,16 @@ public:
 		INTERNAL_WARNING, INTERNAL_ERROR
 	};
 
-	ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
+	ReporterMessage(Type arg_type, MsgThread* thread, const char* arg_msg)
 		: OutputMessage<MsgThread>("ReporterMessage", thread)
-		{ type = arg_type; msg = arg_msg; }
+		{ type = arg_type; msg = copy_string(arg_msg); }
+
+	~ReporterMessage() 	 { delete [] msg; }
 
 	virtual bool Process();
 
 private:
-	string msg;
+	const char* msg;
 	Type type;
 };
 
@@ -82,18 +74,19 @@ private:
 class DebugMessage : public OutputMessage<MsgThread>
 {
 public:
-	DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
+	DebugMessage(DebugStream arg_stream, MsgThread* thread, const char* arg_msg)
 		: OutputMessage<MsgThread>("DebugMessage", thread)
-		{ stream = arg_stream; msg = arg_msg; }
+		{ stream = arg_stream; msg = copy_string(arg_msg); }
+
+	virtual ~DebugMessage()	{ delete [] msg; }
 
 	virtual bool Process()
 		{
-		string s = Object()->Name() + ": " + msg;
-		debug_logger.Log(stream, "%s", s.c_str());
+		debug_logger.Log(stream, "%s: %s", Object()->Name(), msg);
 		return true;
 		}
 private:
-	string msg;
+	const char* msg;
 	DebugStream stream;
 };
 #endif
@@ -104,41 +97,39 @@ private:
 
 Message::~Message()
 	{
+	delete [] name;
 	}
 
 bool ReporterMessage::Process()
 	{
-	string s = Object()->Name() + ": " + msg;
-	const char* cmsg = s.c_str();
-
 	switch ( type ) {
 
 	case INFO:
-		reporter->Info("%s", cmsg);
+		reporter->Info("%s: %s", Object()->Name(), msg);
 		break;
 
 	case WARNING:
-		reporter->Warning("%s", cmsg);
+		reporter->Warning("%s: %s", Object()->Name(), msg);
 		break;
 
 	case ERROR:
-		reporter->Error("%s", cmsg);
+		reporter->Error("%s: %s", Object()->Name(), msg);
 		break;
 
 	case FATAL_ERROR:
-		reporter->FatalError("%s", cmsg);
+		reporter->FatalError("%s: %s", Object()->Name(), msg);
 		break;
 
 	case FATAL_ERROR_WITH_CORE:
-		reporter->FatalErrorWithCore("%s", cmsg);
+		reporter->FatalErrorWithCore("%s: %s", Object()->Name(), msg);
 		break;
 
 	case INTERNAL_WARNING:
-		reporter->InternalWarning("%s", cmsg);
+		reporter->InternalWarning("%s: %s", Object()->Name(), msg);
 		break;
 
 	case INTERNAL_ERROR :
-		reporter->InternalError("%s", cmsg);
+		reporter->InternalError("%s: %s", Object()->Name(), msg);
 		break;
 
 	default:
@@ -148,62 +139,78 @@ bool ReporterMessage::Process()
 	return true;
 	}
 
-MsgThread::MsgThread() : BasicThread()
+MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this)
 	{
 	cnt_sent_in = cnt_sent_out = 0;
 	finished = false;
-	stopped = false;
 	thread_mgr->AddMsgThread(this);
 	}
 
 // Set by Bro's main signal handler.
 extern int signal_val;
 
-void MsgThread::OnStop()
+void MsgThread::OnPrepareStop()
 	{
-	if ( stopped )
+	if ( finished || Killed() )
 		return;
 
+	// XX fprintf(stderr, "Sending FINISH to thread %s  ...\n", Name());
+
 	// Signal thread to terminate and wait until it has acknowledged.
 	SendIn(new FinishMessage(this, network_time), true);
+	}
 
+void MsgThread::OnStop()
+	{
+	int signal_count = 0;
 	int old_signal_val = signal_val;
 	signal_val = 0;
 
 	int cnt = 0;
-	bool aborted = 0;
+	uint64_t last_size = 0;
+	uint64_t cur_size = 0;
 
-	while ( ! finished )
+	// XX fprintf(stderr, "WAITING for thread %s to stop ...\n", Name());
+
+	while ( ! (finished || Killed() ) )
 		{
                 // Terminate if we get another kill signal.
 		if ( signal_val == SIGTERM || signal_val == SIGINT )
 			{
-			// Abort all threads here so that we won't hang next
-			// on another one.
-                        fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name().c_str());
-			thread_mgr->KillThreads();
-			aborted = true;
-			break;
+			++signal_count;
+
+			if ( signal_count == 1 )
+				{
+				// Abort all threads here so that we won't hang next
+				// on another one.
+				fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name());
+				thread_mgr->KillThreads();
+				}
+			else
+				{
+				// More than one signal. Abort processing
+				// right away. on another one.
+				fprintf(stderr, "received another signal while waiting for thread %s, aborting processing\n", Name());
+				exit(1);
+				}
+
+			signal_val = 0;
 			}
 
-		if ( ++cnt % 10000 == 0 ) // Insurance against broken threads ...
-			{
-                        fprintf(stderr, "killing thread %s ...\n", Name().c_str());
-			Kill();
-			aborted = true;
-			break;
-			}
+		queue_in.WakeUp();
 
 		usleep(1000);
 		}
 
-	Finished();
-
 	signal_val = old_signal_val;
+	}
 
-	// One more message to make sure the current queue read operation unblocks.
-	if ( ! aborted )
-		SendIn(new UnblockMessage(this), true);
+void MsgThread::OnKill()
+	{
+	// Send a message to unblock the reader if its currently waiting for
+	// input. This is just an optimization to make it terminate more
+	// quickly, even without the message it will eventually time out.
+	queue_in.WakeUp();
 	}
 
 void MsgThread::Heartbeat()
@@ -213,9 +220,7 @@ void MsgThread::Heartbeat()
 
 void MsgThread::HeartbeatInChild()
 	{
-	string n = Name();
-
-	n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
+	string n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", Name(),
 		cnt_sent_in - queue_in.Size(),
 		cnt_sent_out - queue_out.Size());
 
@@ -283,7 +288,7 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force)
 		return;
 		}
 
-	DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
+	DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name(), Name());
 
 	queue_in.Put(msg);
 	++cnt_sent_in;
@@ -306,9 +311,10 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
 BasicOutputMessage* MsgThread::RetrieveOut()
 	{
 	BasicOutputMessage* msg = queue_out.Get();
-	assert(msg);
+	if ( ! msg )
+		return 0;
 
-	DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s",  msg->Name().c_str(), Name().c_str());
+	DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s",  msg->Name(), Name());
 
 	return msg;
 	}
@@ -316,10 +322,12 @@ BasicOutputMessage* MsgThread::RetrieveOut()
 BasicInputMessage* MsgThread::RetrieveIn()
 	{
 	BasicInputMessage* msg = queue_in.Get();
-	assert(msg);
+
+	if ( ! msg )
+		return 0;
 
 #ifdef DEBUG
-	string s = Fmt("Retrieved '%s' in %s",  msg->Name().c_str(), Name().c_str());
+	string s = Fmt("Retrieved '%s' in %s",  msg->Name(), Name());
 	Debug(DBG_THREADING, s.c_str());
 #endif
 
@@ -328,15 +336,18 @@ BasicInputMessage* MsgThread::RetrieveIn()
 
 void MsgThread::Run()
 	{
-	while ( ! finished )
+	while ( ! (finished || Killed() ) )
 		{
 		BasicInputMessage* msg = RetrieveIn();
 
+		if ( ! msg )
+			continue;
+
 		bool result = msg->Process();
 
 		if ( ! result )
 			{
-			string s = msg->Name() + " failed, terminating thread (MsgThread)";
+			string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
 			Error(s.c_str());
 			break;
 			}
@@ -344,7 +355,7 @@ void MsgThread::Run()
 		delete msg;
 		}
 
-    Finished();
+	Finished();
 	}
 
 void MsgThread::GetStats(Stats* stats)
diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h
index d929c1f806..1d9b17c7d9 100644
--- a/src/threading/MsgThread.h
+++ b/src/threading/MsgThread.h
@@ -228,6 +228,8 @@ protected:
 	 */
 	virtual void Run();
 	virtual void OnStop();
+	virtual void OnPrepareStop();
+	virtual void OnKill();
 
 private:
 	/**
@@ -293,7 +295,6 @@ private:
 	uint64_t cnt_sent_out;	// Counts message sent by child.
 
 	bool finished;	// Set to true by Finished message.
-	bool stopped;   // Set to true by OnStop().
 };
 
 /**
@@ -312,7 +313,7 @@ public:
 	 * what's passed into the constructor and used mainly for debugging
 	 * purposes.
 	 */
-	const string& Name() const { return name; }
+	const char* Name() const { return name; }
 
 	/**
 	 * Callback that must be overriden for processing a message.
@@ -326,10 +327,11 @@ protected:
 	 * @param arg_name A descriptive name for the type of message. Used
 	 * mainly for debugging purposes.
 	 */
-	Message(const string& arg_name)	{ name = arg_name; }
+	Message(const char* arg_name)
+		{ name = copy_string(arg_name); }
 
 private:
-	string name;
+	const char* name;
 };
 
 /**
@@ -344,7 +346,7 @@ protected:
 	 * @param name A descriptive name for the type of message. Used
 	 * mainly for debugging purposes.
 	 */
-	BasicInputMessage(const string& name) : Message(name)	{}
+	BasicInputMessage(const char* name) : Message(name)	{}
 };
 
 /**
@@ -359,7 +361,7 @@ protected:
 	 * @param name A descriptive name for the type of message. Used
 	 * mainly for debugging purposes.
 	 */
-	BasicOutputMessage(const string& name) : Message(name)	{}
+	BasicOutputMessage(const char* name) : Message(name)	{}
 };
 
 /**
@@ -384,7 +386,7 @@ protected:
 	 *
 	 * @param arg_object: An object to store with the message. 
 	 */
-	InputMessage(const string& name, O* arg_object) : BasicInputMessage(name)
+	InputMessage(const char* name, O* arg_object) : BasicInputMessage(name)
 		{ object = arg_object; }
 
 private:
@@ -413,7 +415,7 @@ protected:
 	 *
 	 * @param arg_object An object to store with the message. 
 	 */
-	OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name)
+	OutputMessage(const char* name, O* arg_object) : BasicOutputMessage(name)
 		{ object = arg_object; }
 
 private:
diff --git a/src/threading/Queue.h b/src/threading/Queue.h
index b2ccd2a0ce..29a8084352 100644
--- a/src/threading/Queue.h
+++ b/src/threading/Queue.h
@@ -1,4 +1,3 @@
-
 #ifndef THREADING_QUEUE_H
 #define THREADING_QUEUE_H
 
@@ -6,11 +5,28 @@
 #include <queue>
 #include <deque>
 #include <stdint.h>
+#include <sys/time.h>
 
 #include "Reporter.h"
+#include "BasicThread.h"
 
 #undef Queue // Defined elsewhere unfortunately.
 
+#if 1
+// We don't have pthread spinlocks on DARWIN.
+# define PTHREAD_MUTEX_T      pthread_mutex_t
+# define PTHREAD_MUTEX_LOCK(x)    pthread_mutex_lock(x)
+# define PTHREAD_MUTEX_UNLOCK(x)  pthread_mutex_unlock(x)
+# define PTHREAD_MUTEX_INIT(x)    pthread_mutex_init(x, 0)
+# define PTHREAD_MUTEX_DESTROY(x) pthread_mutex_destroy(x)
+#else
+# define PTHREAD_MUTEX_T          pthrea_spinlock_T
+# define PTHREAD_MUTEX_LOCK(x)    pthrea_spin_lock(x)
+# define PTHREAD_MUTEX_UNLOCK(x)  pthrea_spin_unlock(x)
+# define PTHREAD_MUTEX_INIT(x)    pthrea_spin_init(x, PTHREAD_PROCESS_PRIVATE)
+# define PTHREAD_MUTEX_DESTROY(x) pthrea_spin_destroy(x)
+#endif
+
 namespace threading {
 
 /**
@@ -30,8 +46,12 @@ class Queue
 public:
 	/**
 	 * Constructor.
+	 *
+	 * reader, writer: The corresponding threads. This is for checking
+	 * whether they have terminated so that we can abort I/O opeations.
+	 * Can be left null for the main thread.
 	 */
-	Queue();
+	Queue(BasicThread* arg_reader, BasicThread* arg_writer);
 
 	/**
 	 * Destructor.
@@ -39,7 +59,9 @@ public:
 	~Queue();
 
 	/**
-	 * Retrieves one elment.
+	 * Retrieves one elment. This may block for a little while of no
+	 * input is available and eventually return with a null element if
+	 * nothing shows up.
 	 */
 	T Get();
 
@@ -60,6 +82,11 @@ public:
 	 */
 	bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); }
 
+	/** Wake up the reader if it's currently blocked for input. This is
+	 primarily to give it a chance to check termination quickly.
+	**/
+	void WakeUp();
+
 	/**
 	 * Returns the number of queued items not yet retrieved.
 	 */
@@ -82,45 +109,50 @@ public:
 	void GetStats(Stats* stats);
 
 private:
-	static const int NUM_QUEUES = 8;
+	static const int NUM_QUEUES = 15;
 
-	pthread_mutex_t mutex[NUM_QUEUES];	// Mutex protected shared accesses.
+	PTHREAD_MUTEX_T mutex[NUM_QUEUES];	// Mutex protected shared accesses.
 	pthread_cond_t has_data[NUM_QUEUES];	// Signals when data becomes available
 	std::queue<T> messages[NUM_QUEUES];	// Actually holds the queued messages
 
 	int read_ptr;	// Where the next operation will read from
 	int write_ptr;	// Where the next operation will write to
 
+	BasicThread* reader;
+	BasicThread* writer;
+
 	// Statistics.
 	uint64_t num_reads;
 	uint64_t num_writes;
 };
 
-inline static void safe_lock(pthread_mutex_t* mutex)
+inline static void safe_lock(PTHREAD_MUTEX_T* mutex)
 	{
-	if ( pthread_mutex_lock(mutex) != 0 )
+	if ( PTHREAD_MUTEX_LOCK(mutex) != 0 )
 		reporter->FatalErrorWithCore("cannot lock mutex");
 	}
 
-inline static void safe_unlock(pthread_mutex_t* mutex)
+inline static void safe_unlock(PTHREAD_MUTEX_T* mutex)
 	{
-	if ( pthread_mutex_unlock(mutex) != 0 )
+	if ( PTHREAD_MUTEX_UNLOCK(mutex) != 0 )
 		reporter->FatalErrorWithCore("cannot unlock mutex");
 	}
 
 template<typename T>
-inline Queue<T>::Queue()
+inline Queue<T>::Queue(BasicThread* arg_reader, BasicThread* arg_writer)
 	{
 	read_ptr = 0;
 	write_ptr = 0;
 	num_reads = num_writes = 0;
+	reader = arg_reader;
+	writer = arg_writer;
 
 	for( int i = 0; i < NUM_QUEUES; ++i )
 		{
-		if ( pthread_cond_init(&has_data[i], NULL) != 0 )
+		if ( pthread_cond_init(&has_data[i], 0) != 0 )
 			reporter->FatalError("cannot init queue condition variable");
 
-		if ( pthread_mutex_init(&mutex[i], NULL) != 0 )
+		if ( PTHREAD_MUTEX_INIT(&mutex[i]) != 0 )
 			reporter->FatalError("cannot init queue mutex");
 		}
 	}
@@ -131,19 +163,30 @@ inline Queue<T>::~Queue()
 	for( int i = 0; i < NUM_QUEUES; ++i )
 		{
 		pthread_cond_destroy(&has_data[i]);
-		pthread_mutex_destroy(&mutex[i]);
+		PTHREAD_MUTEX_DESTROY(&mutex[i]);
 		}
 	}
 
 template<typename T>
 inline T Queue<T>::Get()
 	{
+	if ( (reader && reader->Killed()) || (writer && writer->Killed()) )
+		return 0;
+
 	safe_lock(&mutex[read_ptr]);
 
 	int old_read_ptr = read_ptr;
 
 	if ( messages[read_ptr].empty() )
-		pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]);
+		{
+		struct timespec ts;
+		ts.tv_sec = time(0) + 5;
+		ts.tv_nsec = 0;
+
+		pthread_cond_timedwait(&has_data[read_ptr], &mutex[read_ptr], &ts);
+		safe_unlock(&mutex[read_ptr]);
+		return 0;
+		}
 
 	T data = messages[read_ptr].front();
 	messages[read_ptr].pop();
@@ -222,6 +265,17 @@ inline void Queue<T>::GetStats(Stats* stats)
 		safe_unlock(&mutex[i]);
 	}
 
+template<typename T>
+inline void Queue<T>::WakeUp()
+	{
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		{
+		safe_lock(&mutex[i]);
+		pthread_cond_signal(&has_data[i]);
+		safe_unlock(&mutex[i]);
+		}
+	}
+
 }
 
 
diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc
index 4494e1b245..c0e26ccb32 100644
--- a/src/threading/SerialTypes.cc
+++ b/src/threading/SerialTypes.cc
@@ -11,23 +11,54 @@ bool Field::Read(SerializationFormat* fmt)
 	{
 	int t;
 	int st;
+	string tmp_name;
+	bool have_2nd;
 
-	bool success = (fmt->Read(&name, "name")
-			&& fmt->Read(&secondary_name, "secondary_name")
+	if ( ! fmt->Read(&have_2nd, "have_2nd") )
+		return false;
+
+	if ( have_2nd )
+		{
+		string tmp_secondary_name;
+		if ( ! fmt->Read(&tmp_secondary_name, "secondary_name") )
+			return false;
+
+		secondary_name = copy_string(tmp_secondary_name.c_str());
+		}
+	else
+		secondary_name = 0;
+
+	bool success = (fmt->Read(&tmp_name, "name")
 			&& fmt->Read(&t, "type")
 			&& fmt->Read(&st, "subtype")
 			&& fmt->Read(&optional, "optional"));
 
+	if ( ! success )
+		return false;
+
+	name = copy_string(tmp_name.c_str());
+
 	type = (TypeTag) t;
 	subtype = (TypeTag) st;
 
-	return success;
+	return true;
 	}
 
 bool Field::Write(SerializationFormat* fmt) const
 	{
+	assert(name);
+
+	if ( secondary_name )
+		{
+		if ( ! (fmt->Write(true, "have_2nd")
+			&& fmt->Write(secondary_name, "secondary_name")) )
+			return false;
+		}
+	else
+		if ( ! fmt->Write(false, "have_2nd") )
+			return false;
+
 	return (fmt->Write(name, "name")
-		&& fmt->Write(secondary_name, "secondary_name")
 		&& fmt->Write((int)type, "type")
 		&& fmt->Write((int)subtype, "subtype"),
 		fmt->Write(optional, "optional"));
@@ -51,7 +82,7 @@ Value::~Value()
 	{
 	if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
 	     && present )
-		delete val.string_val;
+		delete [] val.string_val.data;
 
 	if ( type == TYPE_TABLE && present )
 		{
@@ -224,10 +255,7 @@ bool Value::Read(SerializationFormat* fmt)
 	case TYPE_STRING:
 	case TYPE_FILE:
 	case TYPE_FUNC:
-		{
-		val.string_val = new string;
-		return fmt->Read(val.string_val, "string");
-		}
+		return fmt->Read(&val.string_val.data, &val.string_val.length, "string");
 
 	case TYPE_TABLE:
 		{
@@ -339,7 +367,7 @@ bool Value::Write(SerializationFormat* fmt) const
 	case TYPE_STRING:
 	case TYPE_FILE:
 	case TYPE_FUNC:
-		return fmt->Write(*val.string_val, "string");
+		return fmt->Write(val.string_val.data, val.string_val.length, "string");
 
 	case TYPE_TABLE:
 		{
diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h
index 283d88bf4c..60aee2411e 100644
--- a/src/threading/SerialTypes.h
+++ b/src/threading/SerialTypes.h
@@ -12,6 +12,7 @@
 using namespace std;
 
 class SerializationFormat;
+class RemoteSerializer;
 
 namespace threading {
 
@@ -19,10 +20,10 @@ namespace threading {
  * Definition of a log file, i.e., one column of a log stream.
  */
 struct Field {
-	string name;	//! Name of the field.
+	const char* name;	//! Name of the field.
 	//! Needed by input framework. Port fields have two names (one for the
 	//! port, one for the type), and this specifies the secondary name.
-	string secondary_name;
+	const char* secondary_name;
 	TypeTag type;	//! Type of the field.
 	TypeTag subtype;	//! Inner type for sets.
 	bool optional;	//! True if field is optional.
@@ -30,13 +31,24 @@ struct Field {
 	/**
 	 * Constructor.
 	 */
-	Field() 	{ subtype = TYPE_VOID; optional = false; }
+	Field(const char* name, const char* secondary_name, TypeTag type, TypeTag subtype, bool optional)
+		: name(name ? copy_string(name) : 0),
+		  secondary_name(secondary_name ? copy_string(secondary_name) : 0),
+		  type(type), subtype(subtype), optional(optional)	{ }
 
 	/**
 	 * Copy constructor.
 	 */
 	Field(const Field& other)
-		: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) {  }
+		: name(other.name ? copy_string(other.name) : 0),
+		  secondary_name(other.secondary_name ? copy_string(other.secondary_name) : 0),
+		  type(other.type), subtype(other.subtype), optional(other.optional)	{ }
+
+	~Field()
+		{
+		delete [] name;
+		delete [] secondary_name;
+		}
 
 	/**
 	 * Unserializes a field.
@@ -63,6 +75,12 @@ struct Field {
 	 * thread-safe.
 	 */
 	string TypeName() const;
+
+private:
+	friend class ::RemoteSerializer;
+
+	// Force usage of constructor above.
+	Field()	{};
 };
 
 /**
@@ -102,7 +120,11 @@ struct Value {
 		vec_t vector_val;
 		addr_t addr_val;
 		subnet_t subnet_val;
-		string* string_val;
+
+		struct {
+			char* data;
+			int length;
+		} string_val;
 	} val;
 
 	/**
@@ -147,7 +169,7 @@ struct Value {
 	static bool IsCompatibleType(BroType* t, bool atomic_only=false);
 
 private:
-friend class ::IPAddr;
+	friend class ::IPAddr;
 	Value(const Value& other)	{ } // Disabled.
 };
 
diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro
index 1f05dfc729..9298ac1c01 100644
--- a/testing/btest/istate/events.bro
+++ b/testing/btest/istate/events.bro
@@ -11,8 +11,8 @@
 # @TEST-EXEC: cat receiver/http.log $SCRIPTS/diff-remove-timestamps >receiver.http.log
 # @TEST-EXEC: cmp sender.http.log receiver.http.log
 # 
-# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
-# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
+# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' | $SCRIPTS/diff-remove-timestamps >events.snd.log
+# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' | $SCRIPTS/diff-remove-timestamps  >events.rec.log
 # @TEST-EXEC: btest-diff events.rec.log
 # @TEST-EXEC: btest-diff events.snd.log
 # @TEST-EXEC: cmp events.rec.log events.snd.log
diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index 2b029789de..9398c1cb4b 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -3,6 +3,4 @@
 # Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs).
 
 sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \
-sed 's/^#\(start\|end\).20..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' | \
-grep -v '#start' | grep -v '#end'
-
+sed 's/^#\(start\|end\).20..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

From 5cfb8d65c3a205a3a8c03dccc041a8b24d070a49 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 19 Jul 2012 18:57:15 -0700
Subject: [PATCH 055/238] Updating tests for the #start/#end change.

---
 aux/binpac                                    |  2 +-
 aux/bro-aux                                   |  2 +-
 aux/broccoli                                  |  2 +-
 aux/broctl                                    |  2 +-
 cmake                                         |  2 +-
 src/logging/writers/Ascii.cc                  |  1 +
 src/threading/BasicThread.cc                  |  8 +++--
 src/threading/BasicThread.h                   |  7 ++--
 src/threading/MsgThread.cc                    |  1 +
 testing/btest/Baseline/bifs.to_double/out     |  2 +-
 testing/btest/Baseline/core.checksums/bad.out | 20 ++++++++++++
 .../btest/Baseline/core.checksums/good.out    | 14 ++++++++
 .../core.disable-mobile-ipv6/weird.log        |  2 ++
 .../Baseline/core.expr-exception/reporter.log | 20 ++++++------
 testing/btest/Baseline/core.ipv6-frag/dns.log |  2 ++
 .../Baseline/core.print-bpf-filters/conn.log  |  2 ++
 .../Baseline/core.print-bpf-filters/output    | 16 +++++++---
 .../core.reporter-error-in-handler/output     |  2 +-
 .../Baseline/core.reporter-fmt-strings/output |  2 +-
 .../Baseline/core.reporter-parse-error/output |  2 +-
 .../core.reporter-runtime-error/output        |  2 +-
 .../core.reporter-type-mismatch/output        |  6 ++--
 .../Baseline/core.reporter/logger-test.log    | 12 +++----
 testing/btest/Baseline/core.reporter/output   |  6 ++--
 testing/btest/Baseline/core.truncation/output |  8 +++++
 .../Baseline/core.tunnels.ayiya/conn.log      |  2 ++
 .../Baseline/core.tunnels.ayiya/http.log      |  2 ++
 .../Baseline/core.tunnels.ayiya/tunnel.log    |  2 ++
 .../core.tunnels.false-teredo/dpd.log         |  2 ++
 .../core.tunnels.false-teredo/weird.log       |  2 ++
 .../Baseline/core.tunnels.teredo/conn.log     |  2 ++
 .../Baseline/core.tunnels.teredo/http.log     |  2 ++
 .../Baseline/core.tunnels.teredo/tunnel.log   |  2 ++
 .../conn.log                                  |  2 ++
 .../http.log                                  |  2 ++
 .../tunnel.log                                |  2 ++
 .../weird.log                                 |  2 ++
 .../btest/Baseline/core.vlan-mpls/conn.log    |  2 ++
 .../canonified_loaded_scripts.log             |  2 ++
 .../coverage.coverage-blacklist/output        | 10 +++---
 .../canonified_loaded_scripts.log             |  2 ++
 .../btest/Baseline/istate.broccoli/bro.log    |  6 ++--
 .../istate.events-ssl/receiver.http.log       |  4 ++-
 .../istate.events-ssl/sender.http.log         |  4 ++-
 .../Baseline/istate.events/receiver.http.log  |  4 ++-
 .../Baseline/istate.events/sender.http.log    |  4 ++-
 .../Baseline/istate.pybroccoli/bro..stdout    |  2 +-
 .../istate.pybroccoli/python..stdout.filtered |  6 ++--
 .../language.wrong-delete-field/output        |  2 +-
 .../send.log                                  | 32 ++++++++++---------
 .../ssh-new-default.log                       |  6 ++--
 .../ssh.log                                   |  2 ++
 .../ssh-filtered.log                          | 10 +++---
 .../test.log                                  |  2 ++
 .../http.log                                  |  2 ++
 .../test.log                                  |  2 ++
 .../ssh.log                                   | 10 +++---
 .../test.log                                  |  4 +--
 .../ssh.log                                   | 10 +++---
 .../test.log                                  |  2 ++
 .../ssh.log                                   |  2 ++
 .../ssh.log                                   |  2 ++
 .../ssh.ds.txt                                | 10 +++---
 .../ssh.log                                   | 12 ++++---
 .../output                                    |  4 +--
 .../ssh.log                                   |  2 ++
 .../ssh.log                                   |  4 ++-
 .../ssh.log                                   | 12 ++++---
 .../local.log                                 |  2 ++
 .../remote.log                                |  2 ++
 .../output                                    | 28 ++++++++++++----
 .../test.failure.log                          |  4 ++-
 .../test.success.log                          |  4 ++-
 .../receiver.test.log                         |  4 ++-
 .../sender.test.failure.log                   |  8 +++--
 .../sender.test.log                           | 12 ++++---
 .../sender.test.success.log                   |  6 ++--
 .../ssh.failure.log                           |  6 ++--
 .../ssh.log                                   |  8 +++--
 .../out                                       | 20 ++++++++++++
 .../out                                       | 20 ++++++++++++
 .../output                                    | 12 ++++---
 .../ssh.log                                   | 12 ++++---
 .../ssh.log                                   |  4 ++-
 .../testing.log                               |  2 ++
 .../ssh.log                                   |  2 ++
 .../manager-1.metrics.log                     |  8 +++--
 .../metrics.log                               |  8 +++--
 .../manager-1.notice.log                      |  4 ++-
 .../notice.log                                |  6 ++--
 .../manager-1.notice.log                      |  4 ++-
 .../manager-1.notice.log                      |  4 ++-
 .../notice.log                                |  4 ++-
 .../conn.log                                  |  2 ++
 .../ftp.log                                   |  2 ++
 .../conn.log                                  |  2 ++
 .../ftp.log                                   |  2 ++
 .../http.log                                  |  2 ++
 .../http.log                                  |  2 ++
 .../http.log                                  |  2 ++
 .../http.log                                  |  2 ++
 .../scripts.base.protocols.irc.basic/irc.log  |  2 ++
 .../irc.log                                   |  2 ++
 .../smtp.log                                  |  2 ++
 .../smtp_entities.log                         |  2 ++
 .../smtp_entities.log                         |  2 ++
 .../socks.log                                 |  2 ++
 .../tunnel.log                                |  2 ++
 .../socks.log                                 |  2 ++
 .../tunnel.log                                |  2 ++
 .../tunnel.log                                |  2 ++
 .../scripts.base.protocols.ssl.basic/ssl.log  |  2 ++
 .../knownhosts-all.log                        |  2 ++
 .../knownhosts-local.log                      |  2 ++
 .../knownhosts-remote.log                     |  2 ++
 .../knownservices-all.log                     |  2 ++
 .../knownservices-local.log                   |  2 ++
 .../knownservices-remote.log                  |  2 ++
 .../dns.log                                   |  2 ++
 testing/btest/istate/events-ssl.bro           |  4 +--
 testing/btest/istate/events.bro               |  4 +--
 .../base/frameworks/logging/ascii-escape.bro  |  1 +
 testing/scripts/diff-remove-timestamps        |  2 +-
 123 files changed, 442 insertions(+), 162 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index b4094cb75e..4ad8d15b63 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit b4094cb75e0a7769123f7db1f5d73f3f9f1c3977
+Subproject commit 4ad8d15b6395925c9875c9d2912a6cc3b4918e0a
diff --git a/aux/bro-aux b/aux/bro-aux
index 2038e3de04..c691c01e9c 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 2038e3de042115c3caa706426e16c830c1fd1e9e
+Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
diff --git a/aux/broccoli b/aux/broccoli
index 07866915a1..8234b8903c 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 07866915a1450ddd25b888917f494b4824b0cc3f
+Subproject commit 8234b8903cbc775f341bdb6a1c0159981d88d27b
diff --git a/aux/broctl b/aux/broctl
index 892b60edb9..d5ecd1a42c 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 892b60edb967bb456872638f22ba994e84530137
+Subproject commit d5ecd1a42c04b0dca332edc31811e5a6d0f7f2fb
diff --git a/cmake b/cmake
index 96f3d92aca..2a72c5e08e 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 96f3d92acadbe1ae64f410e974c5ff503903394b
+Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 99fd3f3c6e..d3c210ce47 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -2,6 +2,7 @@
 
 #include <string>
 #include <errno.h>
+#include <fcntl.h>
 
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index af57c26939..d4a82316e8 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -93,9 +93,11 @@ const char* BasicThread::Fmt(const char* format, ...)
 
 const char* BasicThread::Strerror(int err)
 	{
-	static char buf[128] = "<not set>";
-	strerror_r(err, buf, sizeof(buf));
-	return buf;
+	if ( ! strerr_buffer )
+		strerr_buffer = new char[256];
+
+	strerror_r(err, strerr_buffer, 256);
+	return strerr_buffer;
 	}
 
 void BasicThread::Start()
diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h
index 037420b077..e17324e948 100644
--- a/src/threading/BasicThread.h
+++ b/src/threading/BasicThread.h
@@ -120,8 +120,8 @@ public:
 	/**
 	 * A version of strerror() that the thread can safely use. This is
 	 * essentially a wrapper around strerror_r(). Note that it keeps a
-	 * single static buffer internally so the result remains valid only
-	 * until the next call.
+	 * single buffer per thread internally so the result remains valid
+	 * only until the next call.
 	 */
 	const char* Strerror(int err);
 
@@ -207,6 +207,9 @@ private:
 	char* buf;
 	unsigned int buf_len;
 
+	// For implementating Strerror().
+	char* strerr_buffer;
+
 	static uint64_t thread_counter;
 };
 
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 3e06a3fe1e..0e55b99ba1 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -5,6 +5,7 @@
 #include "Manager.h"
 
 #include <unistd.h>
+#include <signal.h>
 
 using namespace threading;
 
diff --git a/testing/btest/Baseline/bifs.to_double/out b/testing/btest/Baseline/bifs.to_double/out
index 8c2fef496a..8e172dcaa6 100644
--- a/testing/btest/Baseline/bifs.to_double/out
+++ b/testing/btest/Baseline/bifs.to_double/out
@@ -3,4 +3,4 @@
 -60.0
 3600.0
 86400.0
-1337982322.762159
+1342748947.655087
diff --git a/testing/btest/Baseline/core.checksums/bad.out b/testing/btest/Baseline/core.checksums/bad.out
index 44a27f7f0f..de4538e32b 100644
--- a/testing/btest/Baseline/core.checksums/bad.out
+++ b/testing/btest/Baseline/core.checksums/bad.out
@@ -3,81 +3,101 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-03-01
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784981.078396	-	-	-	-	-	bad_IP_checksum	-	F	bro
+#end	2012-03-26-18-03-01
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-01-25
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784885.686428	UWkUyAuUGXf	127.0.0.1	30000	127.0.0.1	80	bad_TCP_checksum	-	F	bro
+#end	2012-03-26-18-01-25
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-02-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784933.501023	UWkUyAuUGXf	127.0.0.1	30000	127.0.0.1	13000	bad_UDP_checksum	-	F	bro
+#end	2012-03-26-18-02-13
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-29-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075363.536871	UWkUyAuUGXf	192.168.1.100	8	192.168.1.101	0	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-29-23
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-06-50
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785210.013051	-	-	-	-	-	routing0_hdr	-	F	bro
 1332785210.013051	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	80	bad_TCP_checksum	-	F	bro
+#end	2012-03-26-18-06-50
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-17-23-00
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332782580.798420	-	-	-	-	-	routing0_hdr	-	F	bro
 1332782580.798420	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	13000	bad_UDP_checksum	-	F	bro
+#end	2012-03-26-17-23-00
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-25-11
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075111.800086	-	-	-	-	-	routing0_hdr	-	F	bro
 1334075111.800086	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:78:1:32::1	129	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-25-11
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-07-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785250.469132	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	80	bad_TCP_checksum	-	F	bro
+#end	2012-03-26-18-07-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-17-02-22
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332781342.923813	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	13000	bad_UDP_checksum	-	F	bro
+#end	2012-03-26-17-02-22
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-22-19
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-22-19
diff --git a/testing/btest/Baseline/core.checksums/good.out b/testing/btest/Baseline/core.checksums/good.out
index 0010974b7f..ed6c071ffc 100644
--- a/testing/btest/Baseline/core.checksums/good.out
+++ b/testing/btest/Baseline/core.checksums/good.out
@@ -3,54 +3,68 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-22-19
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-22-19
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-05-25
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785125.596793	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-03-26-18-05-25
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-17-21-48
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332782508.592037	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-03-26-17-21-48
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
diff --git a/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log b/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
index 478cfe8667..d29456f75f 100644
--- a/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
+++ b/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-05-21-56-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1333663011.602839	-	-	-	-	-	unknown_protocol_135	-	F	bro
+#end	2012-04-05-21-56-51
diff --git a/testing/btest/Baseline/core.expr-exception/reporter.log b/testing/btest/Baseline/core.expr-exception/reporter.log
index 2d0441f48a..f9e33d9718 100644
--- a/testing/btest/Baseline/core.expr-exception/reporter.log
+++ b/testing/btest/Baseline/core.expr-exception/reporter.log
@@ -3,14 +3,16 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
+#start	2011-03-18-19-06-08
 #fields	ts	level	message	location
 #types	time	enum	string	string
-1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.915940	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.916118	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.918295	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.952193	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.952228	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.915940	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.916118	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.918295	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.952193	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.952228	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/core.ipv6-frag/dns.log b/testing/btest/Baseline/core.ipv6-frag/dns.log
index 251f35d789..2003d1f253 100644
--- a/testing/btest/Baseline/core.ipv6-frag/dns.log
+++ b/testing/btest/Baseline/core.ipv6-frag/dns.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
+#start	2012-03-07-01-37-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs
 #types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]
 1331084278.438444	UWkUyAuUGXf	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51850	2607:f740:b::f93	53	udp	3903	txtpadding_323.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
 1331084293.592245	arKYeMETxOg	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51851	2607:f740:b::f93	53	udp	40849	txtpadding_3230.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
+#end	2012-03-07-01-38-18
diff --git a/testing/btest/Baseline/core.print-bpf-filters/conn.log b/testing/btest/Baseline/core.print-bpf-filters/conn.log
index b563c4a3ed..4033b64e2a 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/conn.log
+++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2005-10-07-23-23-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
+#end	2005-10-07-23-23-57
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output
index d1c2d47893..e4bc04192a 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@@ -3,30 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229717.179155	-	ip or not ip	T	T
+1342748953.570646	-	ip or not ip	T	T
+#end	<abnormal termination>
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229717.462355	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+1342748953.898675	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+#end	<abnormal termination>
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229717.733007	-	port 42	T	T
+1342748954.278211	-	port 42	T	T
+#end	<abnormal termination>
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229718.001009	-	port 56730	T	T
+1342748954.883780	-	port 56730	T	T
+#end	2005-10-07-23-23-57
diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index 3d8aa6ff54..83b310ab61 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,2 +1,2 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
 1st error printed on script level
diff --git a/testing/btest/Baseline/core.reporter-fmt-strings/output b/testing/btest/Baseline/core.reporter-fmt-strings/output
index 4842dd9fc5..bbd76f3447 100644
--- a/testing/btest/Baseline/core.reporter-fmt-strings/output
+++ b/testing/btest/Baseline/core.reporter-fmt-strings/output
@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))
diff --git a/testing/btest/Baseline/core.reporter-parse-error/output b/testing/btest/Baseline/core.reporter-parse-error/output
index 7606fe5667..76535f75d1 100644
--- a/testing/btest/Baseline/core.reporter-parse-error/output
+++ b/testing/btest/Baseline/core.reporter-parse-error/output
@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"
diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output
index 3a96954101..59bcc3ac9b 100644
--- a/testing/btest/Baseline/core.reporter-runtime-error/output
+++ b/testing/btest/Baseline/core.reporter-runtime-error/output
@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
diff --git a/testing/btest/Baseline/core.reporter-type-mismatch/output b/testing/btest/Baseline/core.reporter-type-mismatch/output
index 4c038ea8c5..23eefd13e8 100644
--- a/testing/btest/Baseline/core.reporter-type-mismatch/output
+++ b/testing/btest/Baseline/core.reporter-type-mismatch/output
@@ -1,3 +1,3 @@
-error in string and /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))
+error in string and /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))
diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log
index bc2abd142a..6f7ba1d8c7 100644
--- a/testing/btest/Baseline/core.reporter/logger-test.log
+++ b/testing/btest/Baseline/core.reporter/logger-test.log
@@ -1,6 +1,6 @@
-reporter_info|init test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output
index 185cabb1eb..2735adc931 100644
--- a/testing/btest/Baseline/core.reporter/output
+++ b/testing/btest/Baseline/core.reporter/output
@@ -1,3 +1,3 @@
-/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
diff --git a/testing/btest/Baseline/core.truncation/output b/testing/btest/Baseline/core.truncation/output
index 95d9073648..836f9170d4 100644
--- a/testing/btest/Baseline/core.truncation/output
+++ b/testing/btest/Baseline/core.truncation/output
@@ -3,30 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-11-16-01-35
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334160095.895421	-	-	-	-	-	truncated_IP	-	F	bro
+#end	2012-04-11-16-01-35
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-11-14-57-21
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334156241.519125	-	-	-	-	-	truncated_IP	-	F	bro
+#end	2012-04-11-14-57-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-21-50-48
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334094648.590126	-	-	-	-	-	truncated_IP	-	F	bro
+#end	2012-04-10-21-50-48
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-05-29-22-02-34
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1338328954.078361	-	-	-	-	-	internally_truncated_header	-	F	bro
+#end	2012-05-29-22-02-34
diff --git a/testing/btest/Baseline/core.tunnels.ayiya/conn.log b/testing/btest/Baseline/core.tunnels.ayiya/conn.log
index db54a8a475..82a3828f0d 100644
--- a/testing/btest/Baseline/core.tunnels.ayiya/conn.log
+++ b/testing/btest/Baseline/core.tunnels.ayiya/conn.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2009-11-08-04-41-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1257655301.595604	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	tcp	http	2.101052	2981	4665	S1	-	0	ShADad	10	3605	11	5329	k6kgXLOoSKl
@@ -13,3 +14,4 @@
 1257655296.585188	TEfuqmmG4bh	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff00:2	130	icmp	-	0.919988	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
 1257655296.585151	j4u32Pc5bif	fe80::216:cbff:fe9a:4cb9	131	ff02::2:f901:d225	130	icmp	-	0.719947	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
 1257655296.585034	nQcgTWjvg4c	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff9a:4cb9	130	icmp	-	4.922880	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
+#end	2009-11-08-04-41-57
diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log
index 7cef1a1b8e..4fbcd508f4 100644
--- a/testing/btest/Baseline/core.tunnels.ayiya/http.log
+++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2009-11-08-04-41-41
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1257655301.652206	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	10102	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1257655302.514424	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	2	GET	ipv6.google.com	/csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
 1257655303.603569	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	3	GET	ipv6.google.com	/gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
+#end	2009-11-08-04-41-57
diff --git a/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log b/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
index b4ef2781c6..123ea8a792 100644
--- a/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2009-11-08-04-41-33
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1257655293.629048	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::DISCOVER
 1257655296.585034	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::DISCOVER
 1257655317.464035	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::CLOSE
 1257655317.464035	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::CLOSE
+#end	2009-11-08-04-41-57
diff --git a/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log b/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
index 4949f16e62..63a0437445 100644
--- a/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
+++ b/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dpd
+#start	2009-11-18-17-59-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	analyzer	failure_reason
 #types	time	string	addr	port	addr	port	enum	string	string
 1258567191.486869	UWkUyAuUGXf	192.168.1.105	57696	192.168.1.1	53	udp	TEREDO	Teredo payload length [c\x1d\x81\x80\x00\x01\x00\x02\x00\x02\x00\x00\x04amch\x0equestionmarket\x03com\x00\x00\x01\x00...]
@@ -11,3 +12,4 @@
 1258581768.898165	TEfuqmmG4bh	192.168.1.104	50798	192.168.1.1	53	udp	TEREDO	Teredo payload length [o\xe3\x81\x80\x00\x01\x00\x02\x00\x04\x00\x04\x03www\x0fnashuatelegraph\x03com\x00\x00\x01\x00...]
 1258584478.989528	FrJExwHcSal	192.168.1.104	64963	192.168.1.1	53	udp	TEREDO	Teredo payload length [e\xbd\x81\x80\x00\x01\x00\x08\x00\x06\x00\x06\x08wellness\x05blogs\x04time\x03com\x00\x00\x01\x00...]
 1258600683.934672	5OKnoww6xl4	192.168.1.103	59838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xf0\x81\x80\x00\x01\x00\x01\x00\x02\x00\x00\x06update\x0csanasecurity\x03com\x00\x00\x01\x00...]
+#end	2009-11-19-03-18-03
diff --git a/testing/btest/Baseline/core.tunnels.false-teredo/weird.log b/testing/btest/Baseline/core.tunnels.false-teredo/weird.log
index 0ec1d0a7cf..eb4319c7eb 100644
--- a/testing/btest/Baseline/core.tunnels.false-teredo/weird.log
+++ b/testing/btest/Baseline/core.tunnels.false-teredo/weird.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2009-11-18-17-59-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1258567191.405770	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
@@ -11,3 +12,4 @@
 1258581768.568451	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
 1258584478.859853	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
 1258600683.934458	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
+#end	2009-11-19-03-18-03
diff --git a/testing/btest/Baseline/core.tunnels.teredo/conn.log b/testing/btest/Baseline/core.tunnels.teredo/conn.log
index cefc8f3e84..2342953339 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/conn.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2008-05-16-15-50-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1210953047.736921	arKYeMETxOg	192.168.2.16	1576	75.126.130.163	80	tcp	-	0.000357	0	0	SHR	-	0	fA	1	40	1	40	(empty)
@@ -26,3 +27,4 @@
 1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
 1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c
 1210953052.202579	j4u32Pc5bif	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	nQcgTWjvg4c
+#end	2008-05-16-15-51-16
diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log
index b3cf832083..c0db5fc146 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/http.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/http.log
@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2008-05-16-15-50-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1210953057.917183	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	1	POST	download913.avast.com	/cgi-bin/iavs4stats.cgi	-	Syncer/4.80 (av_pro-1169;f)	589	0	204	<empty>	-	-	-	(empty)	-	-	-	text/plain	-	-
 1210953061.585996	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1210953073.381474	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1210953074.674817	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	1	GET	www.wireshark.org	/	http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	11845	200	OK	-	-	-	(empty)	-	-	-	text/xml	-	-
+#end	2008-05-16-15-51-16
diff --git a/testing/btest/Baseline/core.tunnels.teredo/tunnel.log b/testing/btest/Baseline/core.tunnels.teredo/tunnel.log
index 9cead25be1..ab14bf68bc 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/tunnel.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2008-05-16-15-50-52
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::DISCOVER
@@ -11,3 +12,4 @@
 1210953076.058333	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::CLOSE
 1210953076.058333	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	Tunnel::TEREDO	Tunnel::CLOSE
 1210953076.058333	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	Tunnel::TEREDO	Tunnel::CLOSE
+#end	2008-05-16-15-51-16
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
index 6ceb4efcb3..7b9ff58624 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1340127577.354166	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	tcp	http	0.052829	1675	10467	S1	-	0	ShADad	10	2279	12	11191	j4u32Pc5bif
@@ -12,3 +13,4 @@
 1340127577.339015	nQcgTWjvg4c	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	k6kgXLOoSKl
 1340127577.343969	TEfuqmmG4bh	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.007778	4	4	OTH	-	0	-	1	52	1	52	UWkUyAuUGXf,j4u32Pc5bif
 1340127577.336558	arKYeMETxOg	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	UWkUyAuUGXf
+#end	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
index 869476d7db..12f0d7be7a 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1340127577.361683	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1340127577.379360	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
index 30f88ed251..1a14b3edb7 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340127577.336558	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::DISCOVER
@@ -11,3 +12,4 @@
 1340127577.406995	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::CLOSE
 1340127577.406995	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Tunnel::TEREDO	Tunnel::CLOSE
 1340127577.406995	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	Tunnel::TEREDO	Tunnel::CLOSE
+#end	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
index e01fa49d45..8b252a5819 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1340127577.346849	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Teredo_bubble_with_payload	-	F	bro
 1340127577.349292	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
+#end	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.vlan-mpls/conn.log b/testing/btest/Baseline/core.vlan-mpls/conn.log
index e165df621a..72e13ee9b4 100644
--- a/testing/btest/Baseline/core.vlan-mpls/conn.log
+++ b/testing/btest/Baseline/core.vlan-mpls/conn.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2005-10-07-23-23-55
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.102560	26	0	SH	-	0	SADF	11	470	0	0	(empty)
 1128727435.450898	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
 1278600802.069419	k6kgXLOoSKl	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
+#end	2010-07-08-14-53-22
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index 0f12ce4ead..8f90296b63 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
+#start	2012-07-20-01-49-31
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -28,3 +29,4 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/./readers/raw.bro
     scripts/base/frameworks/input/./readers/benchmark.bro
 scripts/policy/misc/loaded-scripts.bro
+#end	2012-07-20-01-49-31
diff --git a/testing/btest/Baseline/coverage.coverage-blacklist/output b/testing/btest/Baseline/coverage.coverage-blacklist/output
index 6d3d243220..c54e4283b2 100644
--- a/testing/btest/Baseline/coverage.coverage-blacklist/output
+++ b/testing/btest/Baseline/coverage.coverage-blacklist/output
@@ -1,5 +1,5 @@
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13	print cover me;
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17	print always executed;
-0	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26	print also impossible, but included in code coverage analysis;
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29	print success;
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5	print first;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13	print cover me;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17	print always executed;
+0	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26	print also impossible, but included in code coverage analysis;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29	print success;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5	print first;
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index f1f9791fc3..6bc461ed65 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
+#start	2012-07-20-01-49-33
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -108,3 +109,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/./consts.bro
     scripts/base/protocols/syslog/./main.bro
 scripts/policy/misc/loaded-scripts.bro
+#end	2012-07-20-01-49-33
diff --git a/testing/btest/Baseline/istate.broccoli/bro.log b/testing/btest/Baseline/istate.broccoli/bro.log
index 4fbbfc81ae..70bf23f95a 100644
--- a/testing/btest/Baseline/istate.broccoli/bro.log
+++ b/testing/btest/Baseline/istate.broccoli/bro.log
@@ -1,3 +1,3 @@
-ping received, seq 0, 1324314397.698781 at src, 1324314397.699240 at dest, 
-ping received, seq 1, 1324314398.698905 at src, 1324314398.699094 at dest, 
-ping received, seq 2, 1324314399.699012 at src, 1324314399.699231 at dest, 
+ping received, seq 0, 1342749173.594568 at src, 1342749173.637317 at dest, 
+ping received, seq 1, 1342749174.594948 at src, 1342749174.596551 at dest, 
+ping received, seq 2, 1342749175.595486 at src, 1342749175.596581 at dest, 
diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log
index 5a7912d23d..c9a996ef5b 100644
--- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log
+++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336588614.060989	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749182.906082	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-04
diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log
index 5a7912d23d..c9a996ef5b 100644
--- a/testing/btest/Baseline/istate.events-ssl/sender.http.log
+++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336588614.060989	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749182.906082	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-04
diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log
index 55a0189cec..566457b996 100644
--- a/testing/btest/Baseline/istate.events/receiver.http.log
+++ b/testing/btest/Baseline/istate.events/receiver.http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336587178.164598	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749191.765740	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-13
diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log
index 55a0189cec..566457b996 100644
--- a/testing/btest/Baseline/istate.events/sender.http.log
+++ b/testing/btest/Baseline/istate.events/sender.http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336587178.164598	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749191.765740	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-13
diff --git a/testing/btest/Baseline/istate.pybroccoli/bro..stdout b/testing/btest/Baseline/istate.pybroccoli/bro..stdout
index a5d20b1f2a..b73d342967 100644
--- a/testing/btest/Baseline/istate.pybroccoli/bro..stdout
+++ b/testing/btest/Baseline/istate.pybroccoli/bro..stdout
@@ -1,7 +1,7 @@
 ==== atomic
 -10
 2
-1336411585.166009
+1342749196.619505
 2.0 mins
 F
 1.5
diff --git a/testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered b/testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered
index a44a95bd69..2f2a5978d8 100644
--- a/testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered
+++ b/testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered
@@ -1,7 +1,7 @@
 ==== atomic a 1 ====
 -4L -4
 42 42
-1336411585.1711
+1342749196.6624
 60.0
 True True
 3.14
@@ -14,7 +14,7 @@ True True
 ==== atomic a 2 ====
 -10L -10
 2 2
-1336411585.1660
+1342749196.6195
 120.0
 False False
 1.5
@@ -27,7 +27,7 @@ False False
 ==== atomic b 2 ====
 -10L -10
 <broccoli.count instance at > 2
-<broccoli.time instance at > 1336411585.1660
+<broccoli.time instance at > 1342749196.6195
 <broccoli.interval instance at > 120.0
 False False
 1.5
diff --git a/testing/btest/Baseline/language.wrong-delete-field/output b/testing/btest/Baseline/language.wrong-delete-field/output
index c2aae8aae3..1eefa1d2fe 100644
--- a/testing/btest/Baseline/language.wrong-delete-field/output
+++ b/testing/btest/Baseline/language.wrong-delete-field/output
@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)
+error in /da/home/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
index 94e0403238..7e21ff86b7 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
@@ -3,20 +3,22 @@
 #empty_field	(empty)
 #unset_field	-
 #path	communication
+#start	2012-07-20-01-49-40
 #fields	ts	peer	src_name	connected_peer_desc	connected_peer_addr	connected_peer_port	level	message
 #types	time	string	string	string	addr	port	string	string
-1340904724.781527	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
-1340904724.784954	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
-1340904724.786168	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
-1340904724.786168	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
-1340904724.786168	bro	script	-	-	-	info	connection established
-1340904724.786168	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
-1340904724.786168	bro	script	-	-	-	info	accepting state
-1340904724.787645	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
-1340904724.787645	bro	parent	-	-	-	info	warning: no events to request
-1340904724.788857	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
-1340904724.829480	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer supports keep-in-cache; using that
-1340904724.829480	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: running
-1340904724.829480	bro	parent	-	-	-	info	terminating...
-1340904724.832952	bro	child	-	-	-	info	terminating
-1340904724.834082	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
+1342748980.737451	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
+1342748980.747149	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
+1342748980.748489	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
+1342748980.748489	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
+1342748980.750749	bro	script	-	-	-	info	connection established
+1342748980.750749	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
+1342748980.750749	bro	script	-	-	-	info	accepting state
+1342748980.752225	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
+1342748980.752225	bro	parent	-	-	-	info	warning: no events to request
+1342748980.753384	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
+1342748980.793108	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer supports keep-in-cache; using that
+1342748980.793108	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: running
+1342748980.793108	bro	parent	-	-	-	info	terminating...
+1342748980.796454	bro	child	-	-	-	info	terminating
+1342748980.797536	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
+#end	2012-07-20-01-49-40
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
index 485bfe3eba..a0359c2d70 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh-new-default
+#start	2012-07-20-01-49-19
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314313.140603	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314313.140603	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748959.430282	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748959.430282	1.2.3.4	1234	2.3.4.5	80	failure	US
+#end	2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
index 144a7a6426..0c826f9694 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
@@ -3,8 +3,10 @@
 #empty_field|(empty)
 #unset_field|-
 #path|ssh
+#start|2012-07-20-01-49-19
 #fields|data|data2
 #types|string|string
 abc\x0a\xffdef|DATA2
 abc\x7c\xffdef|DATA2
 abc\xff\x7cdef|DATA2
+#end|2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log
index a2610bb522..b6e4889a21 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log
@@ -5,8 +5,8 @@ PREFIX<>unset_field|NOT-SET
 PREFIX<>path|ssh
 PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
 PREFIX<>types|time|addr|port|addr|port|string|string|bool
-1342126762.852986|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
-1342126762.852986|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
-1342126762.852986|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
-1342126762.852986|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
-1342126762.852986|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
index c9e69994fc..b1a4ba52e2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2012-07-20-01-49-19
 #fields	x	y	z
 #types	string	string	string
 \x2d	-	(empty)
+#end	2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
index 97744b7df8..683f149317 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2011-09-12-03-57-36
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	1	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	0	346	404	Not Found	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2011-09-12-03-57-37
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
index b88627c806..a03c6f954b 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2012-07-20-01-49-19
 #fields	ss
 #types	table[string]
 CC,AA,\x2c,\x2c\x2c
+#end	2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
index 0ef81128d3..0c6a266de0 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
@@ -5,8 +5,8 @@
 #path||ssh
 #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country
 #types||time||addr||port||addr||port||string||string
-1324314313.899736||1.2.3.4||1234||2.3.4.5||80||success||unknown
-1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||US
-1324314313.899736||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
-1324314313.899736||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
-1324314313.899736||1.2.3.4||1234||2.3.4.5||80||failure||MX
+1342759749.586006||1.2.3.4||1234||2.3.4.5||80||success||unknown
+1342759749.586006||1.2.3.4||1234||2.3.4.5||80||failure||US
+1342759749.586006||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
+1342759749.586006||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
+1342759749.586006||1.2.3.4||1234||2.3.4.5||80||failure||MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
index 72df0d73d4..21b81abf95 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2012-07-12-21-00-27
+#start	2012-07-20-01-49-22
 #fields	data	c
 #types	string	count
 Test1	42
 \x23Kaputt	42
 Test2	42
-#end	2012-07-12-21-00-27
+#end	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log
index f66dec7160..6e3263673a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-options/ssh.log
@@ -1,5 +1,5 @@
-1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|unknown
-1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|US
-1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|UK
-1324314313.990741|1.2.3.4|1234|2.3.4.5|80|success|BR
-1324314313.990741|1.2.3.4|1234|2.3.4.5|80|failure|MX
+1342748960.098729|1.2.3.4|1234|2.3.4.5|80|success|unknown
+1342748960.098729|1.2.3.4|1234|2.3.4.5|80|failure|US
+1342748960.098729|1.2.3.4|1234|2.3.4.5|80|failure|UK
+1342748960.098729|1.2.3.4|1234|2.3.4.5|80|success|BR
+1342748960.098729|1.2.3.4|1234|2.3.4.5|80|failure|MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
index 00ab6c8ca0..5fba268afa 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2012-07-20-01-49-20
 #fields	data
 #types	time
 1234567890.000000
@@ -13,3 +14,4 @@
 1234567890.000010
 1234567890.000001
 1234567890.000000
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
index 5acaa7b2fc..7d3bbc0774 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-20
 #fields	status	country	a1	b1	b2
 #types	string	string	count	count	count
 success	unknown	1	3	4
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
index 086a4836fe..c3163dba6f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-20
 #fields	status	country
 #types	string	string
 success	unknown
@@ -10,3 +11,4 @@ failure	US
 failure	UK
 success	BR
 failure	MX
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt
index e9640dfd9d..e6abc3f1f6 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt
@@ -27,8 +27,8 @@
 
 # Extent, type='ssh'
 t id.orig_h id.orig_p id.resp_h id.resp_p status country
-1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success unknown
-1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure US
-1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure UK
-1337216256.956476 1.2.3.4 1234 2.3.4.5 80 success BR
-1337216256.956476 1.2.3.4 1234 2.3.4.5 80 failure MX
+1342748962.493341 1.2.3.4 1234 2.3.4.5 80 success unknown
+1342748962.493341 1.2.3.4 1234 2.3.4.5 80 failure US
+1342748962.493341 1.2.3.4 1234 2.3.4.5 80 failure UK
+1342748962.493341 1.2.3.4 1234 2.3.4.5 80 success BR
+1342748962.493341 1.2.3.4 1234 2.3.4.5 80 failure MX
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
index 16ba17c62c..42f945bf0c 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-20
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314314.443785	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314314.443785	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314314.443785	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1324314314.443785	1.2.3.4	1234	2.3.4.5	80	success	BR
-1324314314.443785	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342748960.468458	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748960.468458	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748960.468458	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342748960.468458	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342748960.468458	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output
index 5da27764a5..6bd153946e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.events/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.events/output
@@ -1,2 +1,2 @@
-[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown]
-[t=1324314314.738385, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US]
+[t=1342748960.593451, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=unknown]
+[t=1342748960.593451, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US]
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
index 4ccf4c836a..3fe01ff913 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-20
 #fields	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	port	addr	port	string	string
 1234	2.3.4.5	80	success	unknown
@@ -10,3 +11,4 @@
 1234	2.3.4.5	80	failure	UK
 1234	2.3.4.5	80	success	BR
 1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
index 4aa3d8f0a7..205f37243f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-20
 #fields	t	f
 #types	time	file
-1324314314.940195	Foo.log
+1342748960.757056	Foo.log
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
index 00242d65c1..cafacf9c4e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-20
 #fields	t	id.orig_h
 #types	time	addr
-1324314315.040480	1.2.3.4
-1324314315.040480	1.2.3.4
-1324314315.040480	1.2.3.4
-1324314315.040480	1.2.3.4
-1324314315.040480	1.2.3.4
+1342748960.796093	1.2.3.4
+1342748960.796093	1.2.3.4
+1342748960.796093	1.2.3.4
+1342748960.796093	1.2.3.4
+1342748960.796093	1.2.3.4
+#end	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
index c2c69f3153..3240e9f824 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	local
+#start	2011-03-18-19-06-13
 #fields	ts	id.orig_h
 #types	time	addr
 1300475168.859163	141.142.220.118
@@ -35,3 +36,4 @@
 1300475168.902195	141.142.220.118
 1300475168.894787	141.142.220.118
 1300475168.901749	141.142.220.118
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
index b396c3fc2d..84980836c4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	remote
+#start	2011-03-18-19-06-13
 #fields	ts	id.orig_h
 #types	time	addr
 1300475169.780331	173.192.163.128
 1300475167.097012	fe80::217:f2ff:fed7:cf65
 1300475171.675372	fe80::3074:17d5:2052:c324
 1300475173.116749	fe80::3074:17d5:2052:c324
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
index a6b8a4e090..1c67ff52b6 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
@@ -10,54 +10,68 @@ static-prefix-2-UK.log
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-0-BR
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	success	BR
+#end	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-0-MX3
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	failure	MX3
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	MX3
+#end	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-0-unknown
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	success	unknown
+#end	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-1-MX
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-1-US
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	US
+#end	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-2-MX2
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	failure	MX2
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	MX2
+#end	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-2-UK
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.385189	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	UK
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
index 733bb02847..96dede8965 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.498365	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748961.488370	1.2.3.4	1234	2.3.4.5	80	failure	US
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
index 0261caeb06..85b5ca9f45 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314315.498365	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748961.488370	1.2.3.4	1234	2.3.4.5	80	success	unknown
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
index d9bd34309a..aa18822daf 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
@@ -3,6 +3,8 @@
 #empty_field	EMPTY
 #unset_field	-
 #path	test
+#start	1970-01-01-00-00-00
 #fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve
 #types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table[count]	table[string]	table[string]	vector[count]	vector[string]
-T	-42	Test::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1324314315.880694	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY
+T	-42	Test::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1342749004.579242	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY
+#end	2012-07-20-01-50-05
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
index 6cb58bf4ac..36b88e496d 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
+#start	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
index f5b79ee2c4..22d354fce4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	success	BR
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
index c40e56af93..888dc424b5 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
+#start	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314321.061516	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+#end	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
index cb3d4aafb8..5a23ad2066 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh.failure
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314328.196443	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314328.196443	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	UK
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
index 38a5bb660c..cea1069748 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314328.196443	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314328.196443	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1324314328.196443	1.2.3.4	1234	2.3.4.5	80	failure	BR
+1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	BR
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
index 915915f43e..91b6f5de7a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
@@ -19,11 +19,31 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, pat
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
 #empty_field	(empty)
+#end	2011-03-07-03-59-55
+#end	2011-03-07-04-00-05
+#end	2011-03-07-04-59-55
+#end	2011-03-07-05-00-05
+#end	2011-03-07-05-59-55
+#end	2011-03-07-06-00-05
+#end	2011-03-07-06-59-55
+#end	2011-03-07-07-00-05
+#end	2011-03-07-07-59-55
+#end	2011-03-07-08-00-05
+#end	2011-03-07-08-59-55
+#end	2011-03-07-09-00-05
+#end	2011-03-07-09-59-55
+#end	2011-03-07-10-00-05
+#end	2011-03-07-10-59-55
+#end	2011-03-07-11-00-05
+#end	2011-03-07-11-59-55
+#end	2011-03-07-12-00-05
+#end	2011-03-07-12-59-55
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #path	test
 #path	test2
 #separator \x09
 #set_separator	,
+#start	2011-03-07-03-00-05
 #types	time	addr	port	addr	port
 #unset_field	-
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
index c335b5eeb9..4764ff23d0 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
@@ -14,97 +14,117 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
 1299470395.000000	10.0.0.2	20	10.0.0.3	0
+#end	2011-03-07-04-00-05
 > test.2011-03-07-04-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299470405.000000	10.0.0.1	20	10.0.0.2	1025
 1299473995.000000	10.0.0.2	20	10.0.0.3	1
+#end	2011-03-07-05-00-05
 > test.2011-03-07-05-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299474005.000000	10.0.0.1	20	10.0.0.2	1026
 1299477595.000000	10.0.0.2	20	10.0.0.3	2
+#end	2011-03-07-06-00-05
 > test.2011-03-07-06-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299477605.000000	10.0.0.1	20	10.0.0.2	1027
 1299481195.000000	10.0.0.2	20	10.0.0.3	3
+#end	2011-03-07-07-00-05
 > test.2011-03-07-07-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299481205.000000	10.0.0.1	20	10.0.0.2	1028
 1299484795.000000	10.0.0.2	20	10.0.0.3	4
+#end	2011-03-07-08-00-05
 > test.2011-03-07-08-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299484805.000000	10.0.0.1	20	10.0.0.2	1029
 1299488395.000000	10.0.0.2	20	10.0.0.3	5
+#end	2011-03-07-09-00-05
 > test.2011-03-07-09-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299488405.000000	10.0.0.1	20	10.0.0.2	1030
 1299491995.000000	10.0.0.2	20	10.0.0.3	6
+#end	2011-03-07-10-00-05
 > test.2011-03-07-10-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299492005.000000	10.0.0.1	20	10.0.0.2	1031
 1299495595.000000	10.0.0.2	20	10.0.0.3	7
+#end	2011-03-07-11-00-05
 > test.2011-03-07-11-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299495605.000000	10.0.0.1	20	10.0.0.2	1032
 1299499195.000000	10.0.0.2	20	10.0.0.3	8
+#end	2011-03-07-12-00-05
 > test.2011-03-07-12-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299499205.000000	10.0.0.1	20	10.0.0.2	1033
 1299502795.000000	10.0.0.2	20	10.0.0.3	9
+#end	2011-03-07-12-59-55
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
index 09afe2031c..110cef054a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	/dev/stdout
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314328.844271	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314328.844271	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314328.844271	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1324314328.844271	1.2.3.4	1234	2.3.4.5	80	success	BR
-1324314328.844271	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342748961.732599	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748961.732599	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748961.732599	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342748961.732599	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342748961.732599	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
index 53292324af..c9191b666e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314328.950525	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314328.950525	1.2.3.4	1234	2.3.4.5	80	failure	US
-1324314328.950525	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1324314328.950525	1.2.3.4	1234	2.3.4.5	80	success	BR
-1324314328.950525	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342748961.748481	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748961.748481	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748961.748481	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342748961.748481	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342748961.748481	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#end	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
index 74aa0312a1..1fc29dbb4e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
@@ -3,6 +3,8 @@
 #empty_field	EMPTY
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-22
 #fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
 #types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table[count]	table[string]	table[string]	vector[count]	vector[string]	func
-T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1324314329.051618	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1342748962.114672	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+#end	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
index 7956ad11a0..b4089aeee8 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	testing
+#start	2012-07-20-01-49-22
 #fields	a.val1	a.val2	b
 #types	count	count	count
 -	-	6
 1	2	3
+#end	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
index 65ab5592bf..ae5d6d246e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
+#start	2012-07-20-01-49-22
 #fields	vec
 #types	vector[string]
 -,2,-,-,5
+#end	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index a22deb26e4..a3f476c1fb 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
+#start	2012-07-20-01-50-41
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1328303679.867377	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1328303679.867377	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
-1328303679.867377	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+1342749041.601712	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1342749041.601712	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1342749041.601712	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+#end	2012-07-20-01-50-49
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index 4bfb6964ea..b497da5194 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
+#start	2012-07-20-01-49-22
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1328303763.333948	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
-1328303763.333948	TEST_METRIC	foo-bar	7.2.1.5	-	-	1
-1328303763.333948	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
+1342748962.841548	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
+1342748962.841548	TEST_METRIC	foo-bar	7.2.1.5	-	-	1
+1342748962.841548	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
+#end	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
index 59d70896fb..8f3a9dc70c 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
+#start	2012-07-20-01-50-59
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1325633225.777902	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
+1342749059.978651	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
+#end	2012-07-20-01-51-08
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
index 58346b79e6..5a214b26cc 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
+#start	2012-07-20-01-49-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1325633274.875473	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
-1325633274.875473	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	6.5.4.3	-	-
+1342748963.085888	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
+1342748963.085888	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	6.5.4.3	-	-
+#end	2012-07-20-01-49-23
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
index 10888b21ec..4903ec0c01 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
+#start	2012-07-20-01-51-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1325633122.490990	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+1342749078.270791	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+#end	2012-07-20-01-51-27
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
index 5deac88071..bd77a90c86 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
+#start	2012-07-20-01-51-36
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
-1325633150.723248	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+1342749096.545663	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+#end	2012-07-20-01-51-45
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
index 1d168d7613..5a3cdfa69f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
+#start	2012-07-20-01-49-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double
-1325633207.922993	-	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-
+1342748963.685754	-	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-
+#end	2012-07-20-01-49-23
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log
index 4a20ec39b4..316056fa8c 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2012-02-21-16-53-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562	(empty)
@@ -10,3 +11,4 @@
 1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164	(empty)
 1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164	(empty)
 1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458	(empty)
+#end	2012-02-21-16-53-20
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log
index debc093771..cee57182ed 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ftp
+#start	2012-02-21-16-53-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	user	password	command	arg	mime_type	mime_desc	file_size	reply_code	reply_msg	tags	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	count	string	table[string]	file
 1329843179.926563	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	ASCII text	77	226	Transfer complete.	-	-
 1329843197.727769	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	ASCII text, with CRLF line terminators	77	226	Transfer complete.	-	-
+#end	2012-02-21-16-53-20
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log
index 9d19ffaf85..299bdbc4ba 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2012-02-15-17-43-15
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1329327783.316897	arKYeMETxOg	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49186	2001:470:4867:99::21	57086	tcp	ftp-data	0.219721	0	342	SF	-	0	ShAdfFa	5	372	4	642	(empty)
@@ -11,3 +12,4 @@
 1329327795.571921	j4u32Pc5bif	2001:470:4867:99::21	55785	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49189	tcp	ftp-data	0.109813	77	0	SF	-	0	ShADFaf	5	449	4	300	(empty)
 1329327777.822004	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	tcp	ftp	26.658219	310	3448	SF	-	0	ShAdDfFa	57	4426	34	5908	(empty)
 1329327800.017649	TEfuqmmG4bh	2001:470:4867:99::21	55647	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49190	tcp	ftp-data	0.109181	342	0	SF	-	0	ShADFaf	5	714	4	300	(empty)
+#end	2012-02-15-17-43-24
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log
index 8bc2ef2cb7..096b91df65 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ftp
+#start	2012-02-15-17-43-07
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	user	password	command	arg	mime_type	mime_desc	file_size	reply_code	reply_msg	tags	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	count	string	table[string]	file
 1329327787.396984	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	anonymous	test	RETR	ftp://[2001:470:4867:99::21]/robots.txt	-	-	77	226	Transfer complete.	-	-
 1329327795.463946	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	anonymous	test	RETR	ftp://[2001:470:4867:99::21]/robots.txt	-	-	77	226	Transfer complete.	-	-
+#end	2012-02-15-17-43-24
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
index ddcea2e9c7..c457f9b64b 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2009-03-19-05-21-36
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1237440095.634312	UWkUyAuUGXf	192.168.3.103	54102	128.146.216.51	80	1	POST	www.osu.edu	/	-	curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3	2001	60731	200	OK	100	Continue	-	(empty)	-	-	-	text/html	-	-
+#end	2009-03-19-05-21-36
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
index cec098a50b..46ae431fc2 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2005-10-07-23-23-56
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1128727435.634189	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat
+#end	2005-10-07-23-23-57
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
index d4e5679da1..69e6613a3c 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2009-11-18-20-58-04
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	FAKE_MIME	-	-
@@ -10,3 +11,4 @@
 1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	FAKE_MIME	-	-
 1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	image/png	e0029eea80812e9a8e57b8d05d52938a	-
 1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	image/png	30aa926344f58019d047e85ba049ca1e	-
+#end	2009-11-18-20-58-32
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
index dfaf34acbf..6e7eb96454 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2009-11-18-20-58-04
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	file
 1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	-	-
@@ -10,3 +11,4 @@
 1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	-	-
 1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	-	-
 1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	-	-
+#end	2009-11-18-20-58-32
diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
index b5c137bcf8..fe18751420 100644
--- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
+++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
+#start	2011-07-20-19-12-44
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	file
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-
 1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	-
+#end	2011-07-20-19-15-42
diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
index 7513bfb9b8..8bd6bd8394 100644
--- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
+++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
+#start	2011-07-20-19-12-44
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	dcc_mime_type	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	string	file
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-	-
 1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
+#end	2011-07-20-19-15-42
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
index 2c1380cb44..eca41f7d09 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	smtp
+#start	2009-10-05-06-06-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
 #types	time	string	addr	port	addr	port	count	string	string	table[string]	string	string	table[string]	string	string	string	string	addr	string	string	string	vector[addr]	string
 1254722768.219663	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	GP	<gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	Mon, 5 Oct 2009 11:36:07 +0530	"Gurpartap Singh" <gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	-	<000301ca4581$ef9e57f0$cedb07d0$@in>	-	SMTP	-	-	-	250 OK id=1Mugho-0003Dg-Un	74.53.140.153,10.10.1.4	Microsoft Office Outlook 12.0
+#end	2009-10-05-06-06-16
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
index 453b55932e..9bae222897 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	smtp_entities
+#start	2009-10-05-06-06-10
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	file	string
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	79	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat	(empty)
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	1918	FAKE_MIME	-	-	(empty)
 1254722770.692804	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	NEWS.txt	10823	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat	(empty)
+#end	2009-10-05-06-06-16
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
index 2b471782d5..5cb4bb15ef 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	smtp_entities
+#start	2009-10-05-06-06-10
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	file	string
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	79	FAKE_MIME	92bca2e6cdcde73647125da7dccbdd07	-	(empty)
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	1918	FAKE_MIME	-	-	(empty)
 1254722770.692804	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	NEWS.txt	10823	FAKE_MIME	a968bb0f9f9d95835b2e74c845877e87	-	(empty)
+#end	2009-10-05-06-06-16
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log
index 08d31fdb69..960ea71720 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	socks
+#start	2012-06-20-17-23-38
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	user	status	request.host	request.name	request_p	bound.host	bound.name	bound_p
 #types	time	string	addr	port	addr	port	count	string	string	addr	string	port	addr	string	port
 1340213015.276495	UWkUyAuUGXf	10.0.0.55	53994	60.190.189.214	8124	5	-	succeeded	-	www.osnews.com	80	192.168.0.31	-	2688
+#end	2012-06-20-17-28-10
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log
index a7068cd0da..d914b3074e 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2012-06-20-17-23-35
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340213015.276495	-	10.0.0.55	0	60.190.189.214	8124	Tunnel::SOCKS	Tunnel::DISCOVER
+#end	2012-06-20-17-28-10
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log
index 8fd109f3a4..ef07cc31a5 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	socks
+#start	2012-06-19-13-41-02
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	user	status	request.host	request.name	request_p	bound.host	bound.name	bound_p
 #types	time	string	addr	port	addr	port	count	string	string	addr	string	port	addr	string	port
 1340113261.914619	UWkUyAuUGXf	10.0.0.50	59580	85.194.84.197	1080	5	-	succeeded	-	www.google.com	443	0.0.0.0	-	443
+#end	2012-06-19-13-41-05
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log
index 5eac3ae7ad..10f079b888 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2012-06-19-13-41-01
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340113261.914619	-	10.0.0.50	0	85.194.84.197	1080	Tunnel::SOCKS	Tunnel::DISCOVER
+#end	2012-06-19-13-41-05
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log
index 4723cb99c4..4299e302ce 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2008-04-15-22-43-49
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1208299429.265774	-	127.0.0.1	0	127.0.0.1	1080	Tunnel::SOCKS	Tunnel::DISCOVER
+#end	2008-04-15-22-43-49
diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
index 74156362e5..b77925e498 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
+#start	2012-04-27-14-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string
 1335538392.319381	UWkUyAuUGXf	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-
+#end	2012-04-27-14-53-16
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
index 0799292857..6951e4d51f 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
+#start	2011-03-18-19-06-08
 #fields	ts	host
 #types	time	addr
 1300475168.783842	141.142.220.118
 1300475168.783842	208.80.152.118
 1300475168.915940	208.80.152.3
 1300475168.962628	208.80.152.2
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
index 6fdba24d39..b70a701448 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
+#start	2011-03-18-19-06-08
 #fields	ts	host
 #types	time	addr
 1300475168.783842	141.142.220.118
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
index 9ef6ee47b7..8e9d8c6c79 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
+#start	2011-03-18-19-06-08
 #fields	ts	host
 #types	time	addr
 1300475168.783842	208.80.152.118
 1300475168.915940	208.80.152.3
 1300475168.962628	208.80.152.2
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
index d53da6f693..25198e92d5 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
+#start	2011-06-24-15-51-31
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table[string]
 1308930691.049431	172.16.238.131	22	tcp	SSH
@@ -10,3 +11,4 @@
 1308930716.462556	74.125.225.81	80	tcp	HTTP
 1308930718.361665	172.16.238.131	21	tcp	FTP
 1308930726.872485	141.142.192.39	22	tcp	SSH
+#end	2011-06-24-15-52-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
index ef1722d6a1..598f49fa65 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
+#start	2011-06-24-15-51-31
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table[string]
 1308930691.049431	172.16.238.131	22	tcp	SSH
 1308930694.550308	172.16.238.131	80	tcp	HTTP
 1308930718.361665	172.16.238.131	21	tcp	FTP
+#end	2011-06-24-15-52-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
index 3fc68cdb91..c248b18146 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
+#start	2011-06-24-15-51-56
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table[string]
 1308930716.462556	74.125.225.81	80	tcp	HTTP
 1308930726.872485	141.142.192.39	22	tcp	SSH
+#end	2011-06-24-15-52-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
index f636093677..fb024db6d2 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
+#start	1999-06-28-23-40-27
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	auth	addl
 #types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	table[string]	table[string]
 930613226.529070	UWkUyAuUGXf	212.180.42.100	25000	131.243.64.3	53	tcp	34798	-	-	-	-	-	0	NOERROR	F	F	F	T	0	4.3.2.1	31337.000000	-	-
+#end	1999-06-28-23-40-27
diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro
index afbee3f6d9..e09bf112fd 100644
--- a/testing/btest/istate/events-ssl.bro
+++ b/testing/btest/istate/events-ssl.bro
@@ -7,8 +7,8 @@
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
 # 
-# @TEST-EXEC: cat sender/http.log   $SCRIPTS/diff-remove-timestamps >sender.http.log
-# @TEST-EXEC: cat receiver/http.log $SCRIPTS/diff-remove-timestamps >receiver.http.log
+# @TEST-EXEC: cat sender/http.log   | $SCRIPTS/diff-remove-timestamps >sender.http.log
+# @TEST-EXEC: cat receiver/http.log | $SCRIPTS/diff-remove-timestamps >receiver.http.log
 # @TEST-EXEC: cmp sender.http.log receiver.http.log
 # 
 # @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' | $SCRIPTS/diff-remove-timestamps >events.snd.log
diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro
index 9298ac1c01..70726a9f20 100644
--- a/testing/btest/istate/events.bro
+++ b/testing/btest/istate/events.bro
@@ -7,8 +7,8 @@
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
 # 
-# @TEST-EXEC: cat sender/http.log   $SCRIPTS/diff-remove-timestamps >sender.http.log
-# @TEST-EXEC: cat receiver/http.log $SCRIPTS/diff-remove-timestamps >receiver.http.log
+# @TEST-EXEC: cat sender/http.log   | $SCRIPTS/diff-remove-timestamps >sender.http.log
+# @TEST-EXEC: cat receiver/http.log | $SCRIPTS/diff-remove-timestamps >receiver.http.log
 # @TEST-EXEC: cmp sender.http.log receiver.http.log
 # 
 # @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g' | $SCRIPTS/diff-remove-timestamps >events.snd.log
diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
index f2c370a27a..1d0742216d 100644
--- a/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
@@ -1,5 +1,6 @@
 #
 # @TEST-EXEC: bro -b %INPUT 
+# @TEST-EXEC: cat ssh.log | egrep -v '#start|#end' >ssh.log.tmp && mv ssh.log.tmp ssh.log
 # @TEST-EXEC: btest-diff ssh.log
 
 redef LogAscii::separator = "||";
diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index 9398c1cb4b..cbb5aa5c0e 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -3,4 +3,4 @@
 # Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs).
 
 sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \
-sed 's/^#\(start\|end\).20..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
+sed 's/^#\(start\|end\).\(19\|20\)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

From 7fde1165e931ae5007b3d2071fcd1a2e4a8f9b60 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 19 Jul 2012 09:41:44 -0700
Subject: [PATCH 056/238] Give configure a --disable-perftools option.

This disables Perftools support even if found. Linking in tcmalloc can
make debugging memory problems quite a bit hard (including confusing
valgrind).
---
 CMakeLists.txt | 4 +++-
 configure      | 5 +++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 28b702ab01..bea83b0de6 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -91,7 +91,9 @@ endif ()
 set(USE_PERFTOOLS false)
 set(USE_PERFTOOLS_DEBUG false)
 
-find_package(GooglePerftools)
+if (NOT DISABLE_PERFTOOLS)
+   find_package(GooglePerftools)
+endif ()
 
 if (GOOGLEPERFTOOLS_FOUND)
     include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
diff --git a/configure b/configure
index 3258d4abfc..2de4be62c4 100755
--- a/configure
+++ b/configure
@@ -33,6 +33,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
+    --disable-perftools    don't try to build python with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
 
@@ -105,6 +106,7 @@ append_cache_entry INSTALL_BROCCOLI     BOOL   true
 append_cache_entry INSTALL_BROCTL       BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
+append_cache_entry DISABLE_PERFTOOLS    BOOL   false
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -156,6 +158,9 @@ while [ $# -ne 0 ]; do
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS    BOOL   false
             ;;
+        --disable-perftools)
+            append_cache_entry DISABLE_PERFTOOLS    BOOL   true
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;

From db3d89d290203a0adb7ba23885198c48bb8ea026 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 20 Jul 2012 08:51:39 -0400
Subject: [PATCH 057/238] Some documentation updates for elasticsearch plugin.

---
 .../frameworks/logging/writers/elasticsearch.bro  | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index adc675e487..2a58f95ae9 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -1,7 +1,16 @@
+##! Log writer for sending logs to an ElasticSearch server.
+##!
+##! Note: This module is in testing and is not yet considered stable!
+##!
+##! There is one known memory issue.  If your elasticsearch server is 
+##! running slowly and taking too long to return from bulk insert 
+##! requests, the message queue to the writer thread will continue 
+##! growing larger and larger giving the appearance of a memory leak.
+
 module LogElasticSearch;
 
 export {
-        ## Name of the ES cluster
+	## Name of the ES cluster
 	const cluster_name = "elasticsearch" &redef;
 
 	## ES Server
@@ -18,16 +27,16 @@ export {
 	const type_prefix = "" &redef;
 
 	## The time before an ElasticSearch transfer will timeout.
+	## This is not working!
 	const transfer_timeout = 2secs;
 
 	## The batch size is the number of messages that will be queued up before 
 	## they are sent to be bulk indexed.
-	## Note: this is mainly a memory usage parameter.
 	const max_batch_size = 1000 &redef;
 	
 	## The maximum amount of wall-clock time that is allowed to pass without
 	## finishing a bulk log send.  This represents the maximum delay you 
-	## would like to have with your logs before they show up in ElasticSearch.
+	## would like to have with your logs before they are sent to ElasticSearch.
 	const max_batch_interval = 1min &redef;
 	
 	## The maximum byte size for a buffered JSON string to send to the bulk

From 0a681367b70e03fbb938146ec497546aa01d4ec8 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Jul 2012 06:58:39 -0700
Subject: [PATCH 058/238] Revert "Fixing calc_next_rotate to use UTC based time
 functions."

This reverts commit 6335dbb5e1cf694afea3c306012a258614d13880.
---
 src/util.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/util.cc b/src/util.cc
index abbea3e906..3cfa5fca1c 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1112,9 +1112,9 @@ double calc_next_rotate(double current, double interval, double base)
 	time_t teatime = time_t(current);
 
 	struct tm t;
-	t = *gmtime_r(&teatime, &t);
+	t = *localtime_r(&teatime, &t);
 	t.tm_hour = t.tm_min = t.tm_sec = 0;
-	double startofday = timegm(&t);
+	double startofday = mktime(&t);
 
 	if ( base < 0 )
 		// No base time given. To get nice timestamps, we round

From 2efebcd8bea8dbbc446de02054814b0f0f9da39b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Jul 2012 07:04:37 -0700
Subject: [PATCH 059/238] Updating submodule(s).

 [nomail]
---
 aux/broctl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/broctl b/aux/broctl
index d5ecd1a42c..231358f166 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit d5ecd1a42c04b0dca332edc31811e5a6d0f7f2fb
+Subproject commit 231358f166f61cc32201a8ac3671ea0c0f5c324e

From 7bd8367076eeba1e3ef4a8c7d4d29f22355d518f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 20 Jul 2012 11:02:09 -0400
Subject: [PATCH 060/238] More documentation updates.

---
 doc/logging-elasticsearch.rst | 80 ++++++++++++++++-------------------
 1 file changed, 37 insertions(+), 43 deletions(-)

diff --git a/doc/logging-elasticsearch.rst b/doc/logging-elasticsearch.rst
index b6d22cf5fa..7571c68219 100644
--- a/doc/logging-elasticsearch.rst
+++ b/doc/logging-elasticsearch.rst
@@ -1,28 +1,38 @@
 
-========================================
+=========================================
 Indexed Logging Output with ElasticSearch
-========================================
+=========================================
 
 .. rst-class:: opening
 
    Bro's default ASCII log format is not exactly the most efficient
-   way for storing and searching large volumes of data. ElasticSearch
-   is a new and exciting technology for dealing with tons of data.
-   ElasticSearch is a search engine built on top of Apache's Lucene
+   way for searching large volumes of data. ElasticSearch
+   is a new data storage technology for dealing with tons of data.
+   It's also a search engine built on top of Apache's Lucene
    project. It scales very well, both for distributed indexing and 
    distributed searching.
 
 .. contents::
 
+Warning
+-------
+
+This writer plugin is still in testing and is not yet recommended for
+production use!  The approach to how logs are handled in the plugin is "fire
+and forget" at this time, there is no error handling if the server fails to
+respond successfully to the insertion request.
+
 Installing ElasticSearch
 ------------------------
 
-ElasticSearch requires a JRE to run. Please download the latest version
-from: <http://www.elasticsearch.org/download/>. Once extracted, start
-ElasticSearch with::
+Download the latest version from: <http://www.elasticsearch.org/download/>.
+Once extracted, start ElasticSearch with::
 
 # ./bin/elasticsearch
 
+For more detailed information, refer to the ElasticSearch installation
+documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html
+
 Compiling Bro with ElasticSearch Support
 ----------------------------------------
 
@@ -41,49 +51,32 @@ First, ensure that you have libcurl installed the run configure.::
 Activating ElasticSearch
 ------------------------
 
-The direct way to use ElasticSearch is to switch *all* log files over to
-ElasticSearch. To do that, just add ``redef
-Log::default_writer=Log::WRITER_ELASTICSEARCH;`` to your ``local.bro``.
-For testing, you can also just pass that on the command line::
+The easiest way to enable ElasticSearch output is to load the tuning/logs-to-
+elasticsearch.bro script.  If you are using BroControl, the following line in
+local.bro will enable it.
 
-    bro -r trace.pcap Log::default_writer=Log::WRITER_ELASTICSEARCH
+.. console::
 
-With that, Bro will now write all its output into ElasticSearch. You can 
-inspect these using ElasticSearch's REST-ful interface. For more
-information, see: <http://www.elasticsearch.org/guide/reference/api/>.
+    @load tuning/logs-to-elasticsearch
 
-There is also a rudimentary web interface to ElasticSearch, available at:
-<http://mobz.github.com/elasticsearch-head/>.
+With that, Bro will now write most of its logs into ElasticSearch in addition
+to maintaining the Ascii logs like it would do by default.  That script has
+some tunable options for choosing which logs to send to ElasticSearch, refer
+to the autogenerated script documentation for those options.
 
-You can also switch only individual files over to ElasticSearch by adding
-code like this to your ``local.bro``::
+There is an interface being written specifically to integrate with the data
+that Bro outputs into ElasticSearch named Brownian.  It can be found here::
 
-.. code::bro
+    https://github.com/grigorescu/Brownian
 
-    event bro_init()
-        {
-        local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
-        f$writer = Log::WRITER_ELASTICSEARCH;               # Change writer type.
-        Log::add_filter(Conn::LOG, f);                   # Replace filter with adapted version.
-        }
+Tuning
+------
 
-Configuring ElasticSearch
--------------------------
+A common problem encountered with ElasticSearch is too many files being held
+open.  The ElasticSearch website has some suggestions on how to increase the
+open file limit.
 
-Bro's ElasticSearch writer comes with a few configuration options::
-
-- cluster_name: Currently unused.
-
-- server_host:  Where to send the data. Default localhost.
-
-- server_port:  What port to send the data to. Default 9200.
-
-- index_prefix:   ElasticSearch indexes are like databases in a standard DB model. 
-  This is the name of the index to which to send the data. Default bro.
-
-- type_prefix:  ElasticSearch types are like tables in a standard DB model. This is a prefix that gets prepended to Bro log names. Example: type_prefix = "bro_" would create types "bro_dns", "bro_http", etc. Default: none.
-
-- batch_size:   How many messages to buffer before sending to ElasticSearch. This is mainly a memory optimization - changing this doesn't seem to affect indexing performance that much. Default: 10,000.
+  - http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html
 
 TODO
 ----
@@ -93,3 +86,4 @@ Lots.
 - Perform multicast discovery for server.
 - Better error detection.
 - Better defaults (don't index loaded-plugins, for instance).
+- 

From c5d1aebbfe8c49ba89dd9d0c906f5ae38669497b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Jul 2012 09:01:25 -0700
Subject: [PATCH 061/238] Temporarily removing tuning/logs-to-elasticsearch.bro
 from the test-all-policy.

Loading it in there can lead to some tests not terminating. We need to
fix that, it let's the coverage test fail.
---
 scripts/test-all-policy.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index a7c43b14b3..c4acece25d 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -60,5 +60,5 @@
 @load tuning/defaults/__load__.bro
 @load tuning/defaults/packet-fragments.bro
 @load tuning/defaults/warnings.bro
-@load tuning/logs-to-elasticsearch.bro
+# @load tuning/logs-to-elasticsearch.bro
 @load tuning/track-all-assets.bro

From ce4b8dd4aca99c4e1013b5c843df30bfedc54cfd Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Jul 2012 09:57:38 -0700
Subject: [PATCH 062/238] Changing HTTP DPD port 3138 to 3128.

Addresses #857.
---
 scripts/base/protocols/http/main.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro
index f4377e03de..21b4fb6113 100644
--- a/scripts/base/protocols/http/main.bro
+++ b/scripts/base/protocols/http/main.bro
@@ -114,7 +114,7 @@ event bro_init() &priority=5
 
 # DPD configuration.
 const ports = {
-	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
+	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
 	8000/tcp, 8080/tcp, 8888/tcp,
 };
 redef dpd_config += { 

From 5ef83900d8b30a44fe86eb97501ba8cc53c06194 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Jul 2012 12:28:34 -0700
Subject: [PATCH 063/238] Sed usage in canonifier script didn't work on
 non-Linux systems.

---
 testing/scripts/diff-remove-timestamps | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index cbb5aa5c0e..e235746f93 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -2,5 +2,13 @@
 #
 # Replace anything which looks like timestamps with XXXs (including the #start/end markers in logs).
 
+# Get us "modern" regexps with sed.
+if [ `uname` == "Linux" ]; then
+   sed="sed"
+else
+   sed="sed -E"
+fi
+
+# The first sed uses a "basic" regexp, the 2nd a "modern:.
 sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \
-sed 's/^#\(start\|end\).\(19\|20\)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
+$sed 's/^#(start|end).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

From 58e2b70fc806621a833d13a88fbee4562f6753ba Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Fri, 20 Jul 2012 14:37:14 -0700
Subject: [PATCH 064/238] make version_ok return true for TLSv12

I think it is a bug that this was missing...
---
 src/ssl-analyzer.pac | 1 +
 src/ssl-defs.pac     | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/ssl-analyzer.pac b/src/ssl-analyzer.pac
index d1ac470284..3d9564eaab 100644
--- a/src/ssl-analyzer.pac
+++ b/src/ssl-analyzer.pac
@@ -93,6 +93,7 @@ function version_ok(vers : uint16) : bool
 	case SSLv30:
 	case TLSv10:
 	case TLSv11:
+	case TLSv12:
 		return true;
 
 	default:
diff --git a/src/ssl-defs.pac b/src/ssl-defs.pac
index b13b7c4881..4f715bbddd 100644
--- a/src/ssl-defs.pac
+++ b/src/ssl-defs.pac
@@ -22,5 +22,6 @@ enum SSLVersions {
 	SSLv20		= 0x0002,
 	SSLv30		= 0x0300,
 	TLSv10		= 0x0301,
-	TLSv11		= 0x0302
+	TLSv11		= 0x0302,
+	TLSv12		= 0x0303
 };

From 053b307e24ee247137dcef031caaeadf681f126d Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sun, 22 Jul 2012 13:42:31 -0700
Subject: [PATCH 065/238] Bug fix for BasicThread.

---
 src/threading/BasicThread.cc | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index d4a82316e8..9c113fb7ec 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -42,24 +42,24 @@ BasicThread::~BasicThread()
 	delete [] strerr_buffer;
 	}
 
-void BasicThread::SetName(const char* name)
+void BasicThread::SetName(const char* arg_name)
 	{
 	delete [] name;
-	name = copy_string(name);
+	name = copy_string(arg_name);
 	}
 
-void BasicThread::SetOSName(const char* name)
+void BasicThread::SetOSName(const char* arg_name)
 	{
 #ifdef HAVE_LINUX
-	prctl(PR_SET_NAME, name, 0, 0, 0);
+	prctl(PR_SET_NAME, arg_name, 0, 0, 0);
 #endif
 
 #ifdef __APPLE__
-	pthread_setname_np(name);
+	pthread_setname_np(arg_name);
 #endif
 
 #ifdef FREEBSD
-	pthread_set_name_np(pthread_self(), name, name);
+	pthread_set_name_np(pthread_self(), arg_name, arg_name);
 #endif
 	}
 

From 71fc2a1728d430b10610b324ed92379b1bad3875 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sun, 22 Jul 2012 15:50:12 -0700
Subject: [PATCH 066/238] Another small change to MsgThread API.

Threads will now reliably get a call to DoFinish() no matter how the
thread terminates. This will always be called from within the thread,
whereas the destructor is called from the main thread after the child
thread has already terminated.

Also removing debugging code.

However, two problems remain with the ASCII writer (seeing them only
on MacOS):

    - the #start/#end timestamps contain only dummy values right now.
      The odd thing is that once I enable strftime() to print actual
      timestamps, I get crashes (even though strftime() is supposed to
      be thread-safe).

    - occassionally, there's still output missing in tests. In those
      cases, the file descriptor apparently goes bad: a write() will
      suddently return EBADF for reasons I don't understand yet.
---
 src/logging/writers/Ascii.cc    | 31 ++++++++++++++++++++-----------
 src/threading/BasicThread.cc    |  8 ++------
 src/threading/MsgThread.cc      | 11 ++++++++---
 src/threading/MsgThread.h       | 11 +++++------
 src/util.cc                     |  3 +++
 testing/scripts/diff-canonifier |  2 +-
 6 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index d3c210ce47..87fa5dfb3c 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -53,12 +53,11 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 
 Ascii::~Ascii()
 	{
-	//fprintf(stderr, "DTOR %p\n", this);
-
-	// Normally, the file will be closed here already via the Finish()
-	// message. But when we terminate abnormally, we may still have it open.
-	if ( fd )
-		CloseFile(0);
+	if ( ! ascii_done )
+		{
+		fprintf(stderr, "internal error: finish missing\n");
+		abort();
+		}
 
 	delete [] separator;
 	delete [] set_separator;
@@ -77,7 +76,7 @@ bool Ascii::WriteHeaderField(const string& key, const string& val)
 
 void Ascii::CloseFile(double t)
 	{
-	if ( ! fd)
+	if ( ! fd )
 		return;
 
 	if ( include_meta )
@@ -170,7 +169,7 @@ bool Ascii::DoFinish(double network_time)
 	{
 	if ( ascii_done )
 		{
-		fprintf(stderr, "duplicate finish message\n");
+		fprintf(stderr, "internal error: duplicate finish\n");
 		abort();
 		}
 
@@ -353,6 +352,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 		// It would so escape the first character.
 		char buf[16];
 		snprintf(buf, sizeof(buf), "\\x%02x", bytes[0]);
+
 		if ( ! safe_write(fd, buf, strlen(buf)) )
 			goto write_error;
 
@@ -416,14 +416,23 @@ string Ascii::LogExt()
 
 string Ascii::Timestamp(double t)
 	{
+#if 1
+	return "2012-01-01-00-00-00";
+#else
+    // Using the version below leads to occasional crashes at least on Mac OS.
+    // Not sure why, all the function should be thread-safe ...
+
 	time_t teatime = time_t(t);
 
 	struct tm tmbuf;
 	struct tm* tm = localtime_r(&teatime, &tmbuf);
 
-	char buf[128];
+        char tmp[128];
 	const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
-	strftime(buf, sizeof(buf), date_fmt, tm);
-	return buf;
+	strftime(tmp, sizeof(tmp), date_fmt, tm);
+
+	return tmp;
+#endif
 	}
 
+
diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc
index 9c113fb7ec..c708bb79ef 100644
--- a/src/threading/BasicThread.cc
+++ b/src/threading/BasicThread.cc
@@ -35,7 +35,7 @@ BasicThread::BasicThread()
 
 BasicThread::~BasicThread()
 	{
-        if ( buf )
+	if ( buf )
 		free(buf);
 
 	delete [] name;
@@ -50,6 +50,7 @@ void BasicThread::SetName(const char* arg_name)
 
 void BasicThread::SetOSName(const char* arg_name)
 	{
+
 #ifdef HAVE_LINUX
 	prctl(PR_SET_NAME, arg_name, 0, 0, 0);
 #endif
@@ -131,16 +132,12 @@ void BasicThread::PrepareStop()
 
 void BasicThread::Stop()
 	{
-	// XX fprintf(stderr, "stop1 %s %d %d\n", name, started, terminating);
-
 	if ( ! started )
 		return;
 
 	if ( terminating )
 		return;
 
-	// XX fprintf(stderr, "stop2 %s\n", name);
-
 	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name);
 
 	OnStop();
@@ -177,7 +174,6 @@ void BasicThread::Kill()
 
 void BasicThread::Done()
 	{
-	// XX fprintf(stderr, "DONE from thread %s\n", name);
 	DBG_LOG(DBG_THREADING, "Thread %s has finished", name);
 
 	terminating = true;
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 0e55b99ba1..121bec265c 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -155,8 +155,6 @@ void MsgThread::OnPrepareStop()
 	if ( finished || Killed() )
 		return;
 
-	// XX fprintf(stderr, "Sending FINISH to thread %s  ...\n", Name());
-
 	// Signal thread to terminate and wait until it has acknowledged.
 	SendIn(new FinishMessage(this, network_time), true);
 	}
@@ -356,7 +354,14 @@ void MsgThread::Run()
 		delete msg;
 		}
 
-	Finished();
+	// In case we haven't send the finish method yet, do it now. Reading
+	// global network_time here should be fine, it isn't changing
+	// anymore.
+	if ( ! finished )
+		{
+		OnFinish(network_time);
+		Finished();
+		}
 	}
 
 void MsgThread::GetStats(Stats* stats)
diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h
index 1d9b17c7d9..da505de6be 100644
--- a/src/threading/MsgThread.h
+++ b/src/threading/MsgThread.h
@@ -197,10 +197,6 @@ protected:
 	 */
 	virtual void Heartbeat();
 
-	/** Flags that the child process has finished processing. Called from child.
-	 */
-	void Finished();
-
 	/** Internal heartbeat processing. Called from child.
 	 */
 	void HeartbeatInChild();
@@ -217,8 +213,7 @@ protected:
 	virtual bool OnHeartbeat(double network_time, double current_time) = 0;
 
 	/** Triggered for execution in the child thread just before shutting threads down.
-	 *  The child thread should finish its operations and then *must*
-	 *  call this class' implementation.
+	 *  The child thread should finish its operations.
 	 */
 	virtual bool OnFinish(double network_time) = 0;
 
@@ -288,6 +283,10 @@ private:
 	 */
 	bool MightHaveOut() { return queue_out.MaybeReady(); }
 
+	/** Flags that the child process has finished processing. Called from child.
+	 */
+	void Finished();
+
 	Queue<BasicInputMessage *> queue_in;
 	Queue<BasicOutputMessage *> queue_out;
 
diff --git a/src/util.cc b/src/util.cc
index 553944c69c..cd367cf825 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1301,6 +1301,9 @@ bool safe_write(int fd, const char* data, int len)
 			if ( errno == EINTR )
 				continue;
 
+            fprintf(stderr, "safe_write error: %d\n", errno);
+            abort();
+
 			return false;
 			}
 
diff --git a/testing/scripts/diff-canonifier b/testing/scripts/diff-canonifier
index 4d04b3372c..3cb213a3f7 100755
--- a/testing/scripts/diff-canonifier
+++ b/testing/scripts/diff-canonifier
@@ -2,4 +2,4 @@
 #
 # Default canonifier used with the tests in testing/btest/*.
 
-`dirname $0`/diff-remove-timestamps | grep -v XXX
+`dirname $0`/diff-remove-timestamps

From 775961ee1525c9d245ec27d7a82816a3fc0c34b2 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sun, 22 Jul 2012 15:57:26 -0700
Subject: [PATCH 067/238] Updating test base line.

(Due to removing the debugging helper in canonification script.)
---
 .../out                                       | 22 ++-----------------
 1 file changed, 2 insertions(+), 20 deletions(-)

diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
index 91b6f5de7a..e2b8a8b377 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
@@ -19,31 +19,13 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, pat
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
 #empty_field	(empty)
-#end	2011-03-07-03-59-55
-#end	2011-03-07-04-00-05
-#end	2011-03-07-04-59-55
-#end	2011-03-07-05-00-05
-#end	2011-03-07-05-59-55
-#end	2011-03-07-06-00-05
-#end	2011-03-07-06-59-55
-#end	2011-03-07-07-00-05
-#end	2011-03-07-07-59-55
-#end	2011-03-07-08-00-05
-#end	2011-03-07-08-59-55
-#end	2011-03-07-09-00-05
-#end	2011-03-07-09-59-55
-#end	2011-03-07-10-00-05
-#end	2011-03-07-10-59-55
-#end	2011-03-07-11-00-05
-#end	2011-03-07-11-59-55
-#end	2011-03-07-12-00-05
-#end	2011-03-07-12-59-55
+#end	2012-01-01-00-00-00
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #path	test
 #path	test2
 #separator \x09
 #set_separator	,
-#start	2011-03-07-03-00-05
+#start	2012-01-01-00-00-00
 #types	time	addr	port	addr	port
 #unset_field	-
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024

From f2e60a76a81360a64fec78d3693bea2a22ec389a Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sun, 22 Jul 2012 21:04:59 -0700
Subject: [PATCH 068/238] Script fix for Linux.

---
 testing/scripts/diff-remove-timestamps | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index e235746f93..84bd21aa60 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -4,7 +4,7 @@
 
 # Get us "modern" regexps with sed.
 if [ `uname` == "Linux" ]; then
-   sed="sed"
+   sed="sed -r"
 else
    sed="sed -E"
 fi

From 336990e234e2903d9e5a596fc1b53f000181cef8 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 23 Jul 2012 11:27:08 -0700
Subject: [PATCH 069/238] make reading ascii logfiles work when the input
 separator is different from \t.

(Wrong escape character was used for reading header fields).
---
 src/input/readers/Ascii.cc                            | 4 ++--
 testing/btest/scripts/base/frameworks/input/event.bro | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 73821d7cb6..297f8a7136 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -144,7 +144,7 @@ bool Ascii::ReadHeader(bool useCached)
 		pos++;
 		}
 
-	//printf("Updating fields from description %s\n", line.c_str());
+	// printf("Updating fields from description %s\n", line.c_str());
 	columnMap.clear();
 
 	for ( int i = 0; i < NumFields(); i++ )
@@ -199,7 +199,7 @@ bool Ascii::GetLine(string& str)
 		if ( str[0] != '#' )
 			return true;
 
-		if ( str.compare(0,8, "#fields\t") == 0 )
+		if ( ( str.compare(0,7, "#fields") == 0 ) && ( str[7] == separator[0] ) )
 			{
 			str = str.substr(8);
 			return true;
diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro
index d275cee59c..f07ca0c43e 100644
--- a/testing/btest/scripts/base/frameworks/input/event.bro
+++ b/testing/btest/scripts/base/frameworks/input/event.bro
@@ -48,7 +48,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: b
 event bro_init()
 	{
 	try = 0;
-    outfile = open("../out");
+	outfile = open("../out");
 	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line]);
 	Input::remove("input");
 	}

From 8e453663dd4d9540789614582ddce84f877a8b50 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 23 Jul 2012 12:43:42 -0700
Subject: [PATCH 070/238] Input framework now accepts escaped ascii values as
 input.

I managed to completely forget to add unescaping to the input framework -
this should fix it. It now works with the exact same escaping that is
used by the writers (\x##).

Includes one testcase that seems to work - everything else still passes.
---
 src/input/readers/Ascii.cc |  2 ++
 src/util.cc                | 70 ++++++++++++++++++++++++++++++--------
 src/util.h                 |  1 +
 3 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 297f8a7136..aaa124f0c1 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -438,6 +438,8 @@ bool Ascii::DoUpdate()
 			if ( ! getline(splitstream, s, separator[0]) )
 				break;
 
+			s = get_unescaped_string(s);
+
 			stringfields[pos] = s;
 			pos++;
 			}
diff --git a/src/util.cc b/src/util.cc
index cd367cf825..544ba1b573 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -42,6 +42,46 @@
 #include "Net.h"
 #include "Reporter.h"
 
+/**
+ * Takes a string, unescapes all characters that are escaped as hex codes
+ * (\x##) and turns them into the equivalent ascii-codes. Returns a string
+ * containing no escaped values
+ *
+ * @param str string to unescape
+ * @return A str::string without escaped characters.
+ */
+std::string get_unescaped_string(const std::string& str)
+	{
+	char* buf = new char [str.length() + 1]; // it will at most have the same length as str.
+	char* bufpos = buf;
+	size_t pos = 0;
+
+	while ( pos < str.length() )
+		{
+		if ( str[pos] == '\\' && str[pos+1] == 'x' && 
+		     isxdigit(str[pos+2]) && isxdigit(str[pos+3]) ) 
+			{
+				*bufpos = (decode_hex(str[pos+2]) << 4) +
+					decode_hex(str[pos+3]);
+
+				pos += 4;
+				bufpos++;
+			}
+		else 
+			{
+			*bufpos = str[pos];
+			bufpos++;
+			pos++;
+			}
+		}
+
+	*bufpos = 0;
+
+	string outstring (buf, bufpos - buf);
+	delete [] buf;
+	return outstring;
+	}
+
 /**
  * Takes a string, escapes characters into equivalent hex codes (\x##), and
  * returns a string containing all escaped values.
@@ -53,25 +93,25 @@
  * @return A std::string containing a list of escaped hex values of the form
  * \x## */
 std::string get_escaped_string(const std::string& str, bool escape_all)
-{
-    char tbuf[16];
-    string esc = "";
+	{
+	char tbuf[16];
+	string esc = "";
 
-    for ( size_t i = 0; i < str.length(); ++i )
-        {
-	char c = str[i];
+	for ( size_t i = 0; i < str.length(); ++i )
+        	{
+		char c = str[i];
 
-	if ( escape_all || isspace(c) || ! isascii(c) || ! isprint(c) )
-		{
-		snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]);
-		esc += tbuf;
+		if ( escape_all || isspace(c) || ! isascii(c) || ! isprint(c) )
+			{
+			snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]);
+			esc += tbuf;
+			}
+		else
+			esc += c;
 		}
-	else
-		esc += c;
-	}
 
-    return esc;
-}
+	return esc;
+	}
 
 char* copy_string(const char* s)
 	{
diff --git a/src/util.h b/src/util.h
index a695c6df6a..fc4b60792b 100644
--- a/src/util.h
+++ b/src/util.h
@@ -90,6 +90,7 @@ void delete_each(T* t)
 		delete *it;
 	}
 
+std::string get_unescaped_string(const std::string& str);
 std::string get_escaped_string(const std::string& str, bool escape_all);
 
 extern char* copy_string(const char* s);

From 3163e8462928a0294605d690ed176ed528a64813 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 23 Jul 2012 12:46:09 -0700
Subject: [PATCH 071/238] and like nearly always - forgot the baseline.

---
 .../btest/Baseline/scripts.base.frameworks.input.binary/out | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.binary/out

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.binary/out b/testing/btest/Baseline/scripts.base.frameworks.input.binary/out
new file mode 100644
index 0000000000..deab902925
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.binary/out
@@ -0,0 +1,6 @@
+abc^J\xffdef
+DATA2
+abc|\xffdef
+DATA2
+abc\xff|def
+DATA2

From 90735c3164019bd124b26b14f522d4bc16e71f50 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 23 Jul 2012 12:51:07 -0700
Subject: [PATCH 072/238] and just to be a little bit careful - add check if
 the field description is long enough. Otherwise there might possibly be an
 access of uninitialized memory, when someone reads a file that contains just
 #fields without any following field descriptions.

---
 src/input/readers/Ascii.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index aaa124f0c1..fd936b07b6 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -199,7 +199,7 @@ bool Ascii::GetLine(string& str)
 		if ( str[0] != '#' )
 			return true;
 
-		if ( ( str.compare(0,7, "#fields") == 0 ) && ( str[7] == separator[0] ) )
+		if ( ( str.length() > 8 ) && ( str.compare(0,7, "#fields") == 0 ) && ( str[7] == separator[0] ) )
 			{
 			str = str.substr(8);
 			return true;

From 9b0fe744f2805555d5ab0312b2098b1049f2ed31 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 23 Jul 2012 16:47:44 -0500
Subject: [PATCH 073/238] Fix WriterBackend::WriterInfo serialization, reenable
 ascii start/end tags.

Instantiations of WriterInfo in RemoteSerializer::ProcessLogCreateWriter()
would leave the network_time member uninitialized which could later
cause localtime_r() calls in Ascii::Timestamp() to return a null pointer
due to the bizarre input and giving that to strftime() causes it to segfault.
---
 src/logging/WriterBackend.cc | 2 ++
 src/logging/WriterBackend.h  | 4 ++--
 src/logging/writers/Ascii.cc | 9 +--------
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index 8f119d6f8f..87db8e4437 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -76,6 +76,7 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
 	if ( ! (fmt->Read(&tmp_path, "path") &&
 		fmt->Read(&rotation_base, "rotation_base") &&
 		fmt->Read(&rotation_interval, "rotation_interval") &&
+		fmt->Read(&network_time, "network_time") &&
 		fmt->Read(&size, "config_size")) )
 		return false;
 
@@ -105,6 +106,7 @@ bool WriterBackend::WriterInfo::Write(SerializationFormat* fmt) const
 	if ( ! (fmt->Write(path, "path") &&
 		fmt->Write(rotation_base, "rotation_base") &&
 		fmt->Write(rotation_interval, "rotation_interval") &&
+		fmt->Write(network_time, "network_time") &&
 		fmt->Write(size, "config_size")) )
 		return false;
 
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index a59cd1893e..1ca5650057 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -81,9 +81,9 @@ public:
 		 */
 		config_map config;
 
-		WriterInfo()
+		WriterInfo() : path(0), rotation_interval(0.0), rotation_base(0.0),
+		               network_time(0.0)
 			{
-			path = 0;
 			}
 
 		WriterInfo(const WriterInfo& other)
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 87fa5dfb3c..c77e680a92 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -416,23 +416,16 @@ string Ascii::LogExt()
 
 string Ascii::Timestamp(double t)
 	{
-#if 1
-	return "2012-01-01-00-00-00";
-#else
-    // Using the version below leads to occasional crashes at least on Mac OS.
-    // Not sure why, all the function should be thread-safe ...
-
 	time_t teatime = time_t(t);
 
 	struct tm tmbuf;
 	struct tm* tm = localtime_r(&teatime, &tmbuf);
 
-        char tmp[128];
+	char tmp[128];
 	const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
 	strftime(tmp, sizeof(tmp), date_fmt, tm);
 
 	return tmp;
-#endif
 	}
 
 

From 5d33e22b4d6516da6060b5b1fb12e804c3255600 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 23 Jul 2012 16:20:59 -0700
Subject: [PATCH 074/238] Updating NEWS.

---
 CHANGES | 29 +++++++++++++++++++++++++++++
 NEWS    | 28 +++++++++++++++++++++-------
 VERSION |  2 +-
 3 files changed, 51 insertions(+), 8 deletions(-)

diff --git a/CHANGES b/CHANGES
index 950a2abad6..ed5e58c206 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,33 @@
 
+2.0-844 | 2012-07-23 16:20:59 -0700
+
+  * Reworking parts of the internal threading/logging/input APIs for
+    thread-safety. (Robin Sommer)
+
+  * Bugfix for SSL version check. (Bernhard Amann)
+    
+  * Changing a HTTP DPD from port 3138 to 3128. Addresses #857. (Robin
+    Sommer)
+
+  * ElasticSearch logging writer. See logging-elasticsearch.rst for
+    more information. (Vlad Grigorescu and Seth Hall).
+
+  * Give configure a --disable-perftools option to disable Perftools
+    support even if found. (Robin Sommer)
+
+  * The ASCII log writer now includes "#start <timestamp>" and "#end
+    <timestamp> lines in the each file. (Robin Sommer)
+
+  * Renamed ASCII logger "header" options to "meta". (Robin Sommer)
+
+  * ASCII logs now escape '#' at the beginning of log lines. Addresses
+    #763. (Robin Sommer)
+
+  * Fix bug, where in dns.log rcode always was set to 0/NOERROR when
+    no reply package was seen. (Bernhard Amann)
+
+  * Updating to Mozilla's current certificate bundle. (Seth Hall)
+    
 2.0-769 | 2012-07-13 16:17:33 -0700
 
   * Fix some Info:Record field documentation. (Vlad Grigorescu)
diff --git a/NEWS b/NEWS
index 0798920d8a..00aeb62132 100644
--- a/NEWS
+++ b/NEWS
@@ -56,13 +56,6 @@ New Functionality
   "reader plugins" that make it easy to interface to different data
   sources. We will add more in the future.
 
-- Bro's default ASCII log format is not exactly the most efficient way
-  for storing and searching large volumes of data. An an alternative,
-  Bro now comes with experimental support for DataSeries output, an
-  efficient binary format for recording structured bulk data.
-  DataSeries is developed and maintained at HP Labs. See
-  doc/logging-dataseries for more information.
-
 - BroControl now has built-in support for host-based load-balancing
   when using either PF_RING, Myricom cards, or individual interfaces.
   Instead of adding a separate worker entry in node.cfg for each Bro
@@ -78,6 +71,24 @@ New Functionality
   "lb_method=interfaces" to specify which interfaces to load-balance
   on).
 
+- Bro's default ASCII log format is not exactly the most efficient way
+  for storing and searching large volumes of data. An alternatives,
+  Bro now comes with experimental support for two alternative output
+  formats:
+
+    * DataSeries: an efficient binary format for recording structured
+      bulk data. DataSeries is developed and maintained at HP Labs.
+      See doc/logging-dataseries for more information.
+
+    * ElasticSearch: a distributed RESTful, storage engine and search
+      engine built on top of Apache Lucene. It scales very well, both
+      for distributed indexing and distributed searching.
+
+  Note that at this point, we consider Bro's support for these two
+  formats as prototypes for collecting experience with alternative
+  outputs. We do not yet recommend them for production (but welcome
+  feedback!)
+
 
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~
@@ -146,6 +157,9 @@ the full set.
   renamed to LogAscii::meta_prefix and LogAscii::include_meta,
   respectively.
 
+- The ASCII writers "header_*" options have been renamed to "meta_*"
+  (because there's now also a footer).
+
 Bro 2.0
 -------
 
diff --git a/VERSION b/VERSION
index 99ff0a1495..7868a9d201 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0-769
+2.0-844

From c6c2d4d5d610c8df37dd1817a3fed314d95361a0 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 23 Jul 2012 16:59:51 -0700
Subject: [PATCH 075/238] Baseline update.

---
 .../out                                       | 22 +++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
index e2b8a8b377..91b6f5de7a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
@@ -19,13 +19,31 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, pat
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
 #empty_field	(empty)
-#end	2012-01-01-00-00-00
+#end	2011-03-07-03-59-55
+#end	2011-03-07-04-00-05
+#end	2011-03-07-04-59-55
+#end	2011-03-07-05-00-05
+#end	2011-03-07-05-59-55
+#end	2011-03-07-06-00-05
+#end	2011-03-07-06-59-55
+#end	2011-03-07-07-00-05
+#end	2011-03-07-07-59-55
+#end	2011-03-07-08-00-05
+#end	2011-03-07-08-59-55
+#end	2011-03-07-09-00-05
+#end	2011-03-07-09-59-55
+#end	2011-03-07-10-00-05
+#end	2011-03-07-10-59-55
+#end	2011-03-07-11-00-05
+#end	2011-03-07-11-59-55
+#end	2011-03-07-12-00-05
+#end	2011-03-07-12-59-55
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #path	test
 #path	test2
 #separator \x09
 #set_separator	,
-#start	2012-01-01-00-00-00
+#start	2011-03-07-03-00-05
 #types	time	addr	port	addr	port
 #unset_field	-
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024

From f887535f1c706a727f683c2450114d4c5e322808 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 23 Jul 2012 17:28:27 -0700
Subject: [PATCH 076/238] fix problem with possible access to unititialized
 memory (thanks robin :) )

---
 src/util.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/util.cc b/src/util.cc
index 544ba1b573..da046133a6 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -50,13 +50,14 @@
  * @param str string to unescape
  * @return A str::string without escaped characters.
  */
-std::string get_unescaped_string(const std::string& str)
+std::string get_unescaped_string(const std::string& arg_str)
 	{
-	char* buf = new char [str.length() + 1]; // it will at most have the same length as str.
+	const char* str = arg_str.c_str();
+	char* buf = new char [arg_str.length() + 1]; // it will at most have the same length as str.
 	char* bufpos = buf;
 	size_t pos = 0;
 
-	while ( pos < str.length() )
+	while ( pos < arg_str.length() )
 		{
 		if ( str[pos] == '\\' && str[pos+1] == 'x' && 
 		     isxdigit(str[pos+2]) && isxdigit(str[pos+3]) ) 

From 3f21764d0029b0cae72e2613f914ea69569d8ad4 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 24 Jul 2012 09:09:08 -0700
Subject: [PATCH 077/238] Updating submodule(s).

 [nomail]
---
 aux/binpac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/binpac b/aux/binpac
index 4ad8d15b63..4f01ea4081 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 4ad8d15b6395925c9875c9d2912a6cc3b4918e0a
+Subproject commit 4f01ea40817ad232a96535c64fce7dc16d4e2fff

From 3f4b4c88a6e4fc7f14c4620fe9093a11f9b7dd61 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 24 Jul 2012 11:18:32 -0500
Subject: [PATCH 078/238] Fix initialization of WriterFrontend names.

The string representation of the writer looked up based on the stream's
enum value instead of the writer's enum value, often causing this
component of the name to be "(null)" since a null pointer was returned
from the lookup.
---
 src/logging/WriterFrontend.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc
index fc237d6f6e..7c8f6861cf 100644
--- a/src/logging/WriterFrontend.cc
+++ b/src/logging/WriterFrontend.cc
@@ -112,7 +112,7 @@ WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVa
 	write_buffer_pos = 0;
 	info = new WriterBackend::WriterInfo(arg_info);
 
-	const char* w = arg_writer->Type()->AsEnumType()->Lookup(arg_stream->InternalInt());
+	const char* w = arg_writer->Type()->AsEnumType()->Lookup(arg_writer->InternalInt());
 	name = copy_string(fmt("%s/%s", arg_info.path, w));
 
 	if ( local )

From 13952154a109a69f665a1549b613721384f2599f Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 24 Jul 2012 09:19:20 -0700
Subject: [PATCH 079/238] add comparator functor to the info maps of
 readerbackend and readerwriteend.

This is required, because after the recent changes the info map containst a
char* as key. Without the comparator the map will compare the char addresses
for all operations - which is not really what we want.
---
 src/input/ReaderBackend.h   |  2 +-
 src/logging/WriterBackend.h |  2 +-
 src/util.h                  | 11 +++++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h
index 7626cc25ed..8ee14c808a 100644
--- a/src/input/ReaderBackend.h
+++ b/src/input/ReaderBackend.h
@@ -74,7 +74,7 @@ public:
 	struct ReaderInfo
 		{
 		// Structure takes ownership of the strings.
-		typedef std::map<const char*, const char*> config_map;
+		typedef std::map<const char*, const char*, CompareString> config_map;
 
 		/**
 		 * A string left to the interpretation of the reader
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index 1ca5650057..d5f2be225e 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -49,7 +49,7 @@ public:
 	struct WriterInfo
 		{
 		// Structure takes ownership of these strings.
-		typedef std::map<const char*, const char*> config_map;
+		typedef std::map<const char*, const char*, CompareString> config_map;
 
 		/**
 		 * A string left to the interpretation of the writer
diff --git a/src/util.h b/src/util.h
index a695c6df6a..030a704092 100644
--- a/src/util.h
+++ b/src/util.h
@@ -345,4 +345,15 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al
 extern void get_memory_usage(unsigned int* total,
 			     unsigned int* malloced);
 
+// class to be used as a third argument for stl maps to be able to use
+// char*'s as keys. Otherwise the pointer values will be compared instead
+// of the actual string values.
+struct CompareString 
+	{
+	bool operator()(char const *a, char const *b) const
+		{
+		return std::strcmp(a, b) < 0;
+		}
+	};
+
 #endif

From cfa8769a422fa9ec1eeb4592f8b2eea6ef5a2a58 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 24 Jul 2012 11:22:51 -0500
Subject: [PATCH 080/238] Fix memory leak when processing a thread's input
 message fails.

The message is reclaimed in both success/fail cases now.
---
 src/threading/MsgThread.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index b7a8f4922c..48c7253885 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -342,14 +342,14 @@ void MsgThread::Run()
 
 		bool result = msg->Process();
 
+		delete msg;
+
 		if ( ! result )
 			{
 			string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
 			Error(s.c_str());
 			break;
 			}
-
-		delete msg;
 		}
 
 	// In case we haven't send the finish method yet, do it now. Reading

From 0d748c117d73351daa6157d634d493e1691251eb Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 24 Jul 2012 11:06:16 -0700
Subject: [PATCH 081/238] Adding missing include needed on some systems.

---
 CHANGES                      | 4 ++++
 VERSION                      | 2 +-
 src/logging/writers/Ascii.cc | 1 +
 src/logging/writers/None.cc  | 2 +-
 src/threading/MsgThread.cc   | 1 +
 5 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 8fbd067fbc..e5be483e77 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.0-849 | 2012-07-24 11:06:16 -0700
+
+  * Adding missing include needed on some systems. (Robin Sommer)
+    
 2.0-846 | 2012-07-23 16:36:37 -0700
 
   * Fix WriterBackend::WriterInfo serialization, reenable ascii
diff --git a/VERSION b/VERSION
index 500ff3b4cd..3ccfd995f9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0-846
+2.0-849
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index c77e680a92..3866c48b64 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -3,6 +3,7 @@
 #include <string>
 #include <errno.h>
 #include <fcntl.h>
+#include <unistd.h>
 
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc
index 9b91b82199..cf383899a1 100644
--- a/src/logging/writers/None.cc
+++ b/src/logging/writers/None.cc
@@ -39,7 +39,7 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
 		std::cout << std::endl;
 		}
 
-	return true;
+	return false;
 	}
 
 bool None::DoRotate(const char* rotated_path, double open, double close, bool terminating)
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index b7a8f4922c..fea9c2a532 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -346,6 +346,7 @@ void MsgThread::Run()
 			{
 			string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
 			Error(s.c_str());
+			Kill();
 			break;
 			}
 

From 43752b3d9f4efadebda9d342921e186ee09bcbde Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 24 Jul 2012 11:16:03 -0700
Subject: [PATCH 082/238] Reverting accidentally committed changes.

Thanks, Bernhard!
---
 src/logging/writers/None.cc | 2 +-
 src/threading/MsgThread.cc  | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc
index cf383899a1..9b91b82199 100644
--- a/src/logging/writers/None.cc
+++ b/src/logging/writers/None.cc
@@ -39,7 +39,7 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
 		std::cout << std::endl;
 		}
 
-	return false;
+	return true;
 	}
 
 bool None::DoRotate(const char* rotated_path, double open, double close, bool terminating)
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index fea9c2a532..b7a8f4922c 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -346,7 +346,6 @@ void MsgThread::Run()
 			{
 			string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
 			Error(s.c_str());
-			Kill();
 			break;
 			}
 

From c36a449c76cc442f64b97d1a7c11febf454304d9 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 24 Jul 2012 15:04:14 -0700
Subject: [PATCH 083/238] New built-in function to_double(s: string).

Closes #859.
---
 CHANGES                                       |  4 ++++
 VERSION                                       |  2 +-
 src/bro.bif                                   | 23 +++++++++++++++++++
 .../Baseline/bifs.to_double_from_string/error |  2 ++
 .../bifs.to_double_from_string/output         |  5 ++++
 testing/btest/bifs/to_double_from_string.bro  | 16 +++++++++++++
 6 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/bifs.to_double_from_string/error
 create mode 100644 testing/btest/Baseline/bifs.to_double_from_string/output
 create mode 100644 testing/btest/bifs/to_double_from_string.bro

diff --git a/CHANGES b/CHANGES
index e5be483e77..87a537c1e9 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.0-851 | 2012-07-24 15:04:14 -0700
+
+  * New built-in function to_double(s: string). (Scott Campbell)
+    
 2.0-849 | 2012-07-24 11:06:16 -0700
 
   * Adding missing include needed on some systems. (Robin Sommer)
diff --git a/VERSION b/VERSION
index 3ccfd995f9..c4f46b78c9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0-849
+2.0-851
diff --git a/src/bro.bif b/src/bro.bif
index f18d3ba1b5..2c22626c99 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -2604,6 +2604,29 @@ function to_subnet%(sn: string%): subnet
 	return ret;
 	%}
 
+## Converts a :bro:type:`string` to a :bro:type:`double`.
+##
+## str: The :bro:type:`string` to convert.
+##
+## Returns: The :bro:type:`string` *str* as double, or 0 if *str* has
+##          an invalid format.
+##
+function to_double%(str: string%): double
+	%{
+	const char* s = str->CheckString();
+	char* end_s;
+	
+	double d = strtod(s, &end_s);
+	
+	if ( s[0] == '\0' || end_s[0] != '\0' )
+		{       
+		builtin_error("bad conversion to count", @ARG@[0]);
+		d = 0;
+		}     
+	
+	return new Val(d, TYPE_DOUBLE);
+	%}
+
 ## Converts a :bro:type:`count` to an :bro:type:`addr`.
 ##
 ## ip: The :bro:type:`count` to convert.
diff --git a/testing/btest/Baseline/bifs.to_double_from_string/error b/testing/btest/Baseline/bifs.to_double_from_string/error
new file mode 100644
index 0000000000..5ba5997101
--- /dev/null
+++ b/testing/btest/Baseline/bifs.to_double_from_string/error
@@ -0,0 +1,2 @@
+error in /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 7 and /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 15: bad conversion to count (to_double(d) and NotADouble)
+error in /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 7 and /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 16: bad conversion to count (to_double(d) and )
diff --git a/testing/btest/Baseline/bifs.to_double_from_string/output b/testing/btest/Baseline/bifs.to_double_from_string/output
new file mode 100644
index 0000000000..661d2b1479
--- /dev/null
+++ b/testing/btest/Baseline/bifs.to_double_from_string/output
@@ -0,0 +1,5 @@
+to_double(3.14) = 3.14 (SUCCESS)
+to_double(-3.14) = -3.14 (SUCCESS)
+to_double(0) = 0.0 (SUCCESS)
+to_double(NotADouble) = 0.0 (SUCCESS)
+to_double() = 0.0 (SUCCESS)
diff --git a/testing/btest/bifs/to_double_from_string.bro b/testing/btest/bifs/to_double_from_string.bro
new file mode 100644
index 0000000000..88af6758f0
--- /dev/null
+++ b/testing/btest/bifs/to_double_from_string.bro
@@ -0,0 +1,16 @@
+# @TEST-EXEC: bro -b %INPUT >output 2>error
+# @TEST-EXEC: btest-diff output
+# @TEST-EXEC: btest-diff error
+
+function test_to_double(d: string, expect: double)
+	{
+	local result = to_double(d);
+	print fmt("to_double(%s) = %s (%s)", d, result,
+	          result == expect ? "SUCCESS" : "FAILURE");
+	}
+
+test_to_double("3.14", 3.14);
+test_to_double("-3.14", -3.14);
+test_to_double("0", 0);
+test_to_double("NotADouble", 0);
+test_to_double("", 0);

From b9a76d7ed0f16390a7cfd4da7e3a21cc404c9c5b Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 24 Jul 2012 17:21:30 -0500
Subject: [PATCH 084/238] Fix file permissions of log files

A recent commit was erroneously causing new log files to be created with
execute permissions.
---
 src/logging/writers/Ascii.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 3866c48b64..4d2f59ea72 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -101,7 +101,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 
 	fname = IsSpecial(path) ? path : path + "." + LogExt();
 
-	fd = open(fname.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0777);
+	fd = open(fname.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
 
 	if ( fd < 0 )
 		{

From 3a8f812f1c11be204b2e8451bb24a47eb02db7bf Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 24 Jul 2012 17:32:04 -0500
Subject: [PATCH 085/238] Correct a typo

---
 configure | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure b/configure
index 2de4be62c4..bfe54123f0 100755
--- a/configure
+++ b/configure
@@ -33,7 +33,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
-    --disable-perftools    don't try to build python with Google Perftools
+    --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
 

From 5af131e3035a7057b0c5f321b5e1007a102548f8 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 24 Jul 2012 16:10:52 -0700
Subject: [PATCH 086/238] Compile fix.

---
 src/util.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/util.h b/src/util.h
index 4435b830dd..048ec384e3 100644
--- a/src/util.h
+++ b/src/util.h
@@ -353,7 +353,7 @@ struct CompareString
 	{
 	bool operator()(char const *a, char const *b) const
 		{
-		return std::strcmp(a, b) < 0;
+		return strcmp(a, b) < 0;
 		}
 	};
 

From 91522e78365491ac9c784c8eaa146011fb9e4610 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 25 Jul 2012 12:10:47 -0500
Subject: [PATCH 087/238] Fix tests and error message for to_double BIF

---
 src/bro.bif                                             | 2 +-
 testing/btest/Baseline/bifs.to_double_from_string/error | 4 ++--
 testing/btest/bifs/to_double_from_string.bro            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/bro.bif b/src/bro.bif
index 2c22626c99..2a37429ad6 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -2620,7 +2620,7 @@ function to_double%(str: string%): double
 	
 	if ( s[0] == '\0' || end_s[0] != '\0' )
 		{       
-		builtin_error("bad conversion to count", @ARG@[0]);
+		builtin_error("bad conversion to double", @ARG@[0]);
 		d = 0;
 		}     
 	
diff --git a/testing/btest/Baseline/bifs.to_double_from_string/error b/testing/btest/Baseline/bifs.to_double_from_string/error
index 5ba5997101..d6c6c0c75b 100644
--- a/testing/btest/Baseline/bifs.to_double_from_string/error
+++ b/testing/btest/Baseline/bifs.to_double_from_string/error
@@ -1,2 +1,2 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 7 and /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 15: bad conversion to count (to_double(d) and NotADouble)
-error in /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 7 and /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 16: bad conversion to count (to_double(d) and )
+error in /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 7 and /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 15: bad conversion to double (to_double(d) and NotADouble)
+error in /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 7 and /da/home/robin/bro/master/testing/btest/.tmp/bifs.to_double_from_string/to_double_from_string.bro, line 16: bad conversion to double (to_double(d) and )
diff --git a/testing/btest/bifs/to_double_from_string.bro b/testing/btest/bifs/to_double_from_string.bro
index 88af6758f0..781261084f 100644
--- a/testing/btest/bifs/to_double_from_string.bro
+++ b/testing/btest/bifs/to_double_from_string.bro
@@ -1,6 +1,6 @@
 # @TEST-EXEC: bro -b %INPUT >output 2>error
 # @TEST-EXEC: btest-diff output
-# @TEST-EXEC: btest-diff error
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff error
 
 function test_to_double(d: string, expect: double)
 	{

From 2fafadd9300b2abdf9195f7270071d9549850084 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 25 Jul 2012 12:20:12 -0500
Subject: [PATCH 088/238] Fix differing log filters of streams from writing to
 same writer/path.

Since WriterFrontend objects are looked up internally by writer type and
path, and they also expect to write consistent field arguments, it could
be the case that more than one filter of a given stream attempts to
write to the same path (derived either from $path or $path_func fields
of the filter) with the same writer type.  This won't work, so now
WriterFrontend objects are bound to the filter that instantiated them so
that we can warn about other filters attempting to write to the
conflicting writer/path and the write can be skipped.  Remote logs don't
appear to suffer the same issue due to pre-filtering.

Addresses #842.
---
 src/logging/Manager.cc                        | 17 ++++++++++++--
 src/logging/Manager.h                         |  2 +-
 src/logging/WriterBackend.cc                  |  5 ++--
 .../http.log                                  | 23 +++++++++++++++++++
 .../reporter.log                              | 23 +++++++++++++++++++
 .../logging/writer-path-conflict.bro          | 14 +++++++++++
 6 files changed, 78 insertions(+), 6 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log
 create mode 100755 testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
 create mode 100644 testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro

diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index c4245680a6..3499d55f74 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -86,6 +86,7 @@ struct Manager::WriterInfo {
 	Func* postprocessor;
 	WriterFrontend* writer;
 	WriterBackend::WriterInfo* info;
+	string instantiating_filter;
 	};
 
 struct Manager::Stream {
@@ -764,8 +765,18 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 		WriterFrontend* writer = 0;
 
 		if ( w != stream->writers.end() )
+			{
+			if ( w->second->instantiating_filter != filter->name )
+				{
+				reporter->Warning("Skipping write to filter '%s' on path '%s'"
+				  " because filter '%s' has already instantiated the same"
+				  " writer type for that path", filter->name.c_str(),
+				  filter->path.c_str(), w->second->instantiating_filter.c_str());
+				continue;
+				}
 			// We know this writer already.
 			writer = w->second->writer;
+			}
 
 		else
 			{
@@ -800,7 +811,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 			writer = CreateWriter(stream->id, filter->writer,
 					      info, filter->num_fields,
-					      arg_fields, filter->local, filter->remote);
+					      arg_fields, filter->local, filter->remote, filter->name);
 
 			if ( ! writer )
 				{
@@ -999,7 +1010,8 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 	}
 
 WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
-				int num_fields, const threading::Field* const*  fields, bool local, bool remote)
+				int num_fields, const threading::Field* const*  fields, bool local, bool remote,
+				const string& instantiating_filter)
 	{
 	Stream* stream = FindStream(id);
 
@@ -1023,6 +1035,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken
 	winfo->interval = 0;
 	winfo->postprocessor = 0;
 	winfo->info = info;
+	winfo->instantiating_filter = instantiating_filter;
 
 	// Search for a corresponding filter for the writer/path pair and use its
 	// rotation settings.  If no matching filter is found, fall back on
diff --git a/src/logging/Manager.h b/src/logging/Manager.h
index ae7a1796ba..d2041592c1 100644
--- a/src/logging/Manager.h
+++ b/src/logging/Manager.h
@@ -165,7 +165,7 @@ protected:
 	// Takes ownership of fields and info.
 	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
 				int num_fields, const threading::Field* const* fields,
-				bool local, bool remote);
+				bool local, bool remote, const string& instantiating_filter="");
 
 	// Takes ownership of values..
 	bool Write(EnumVal* id, EnumVal* writer, string path,
diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index 87db8e4437..2933062eff 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -201,7 +201,6 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
 		return false;
 		}
 
-#ifdef DEBUG
 	// Double-check all the types match.
 	for ( int j = 0; j < num_writes; j++ )
 		{
@@ -209,17 +208,17 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
 			{
 			if ( vals[j][i]->type != fields[i]->type )
 				{
+#ifdef DEBUG
 				const char* msg = Fmt("Field type doesn't match in WriterBackend::Write() (%d vs. %d)",
 						      vals[j][i]->type, fields[i]->type);
 				Debug(DBG_LOGGING, msg);
-
+#endif
 				DisableFrontend();
 				DeleteVals(num_writes, vals);
 				return false;
 				}
 			}
 		}
-#endif
 
 	bool success = true;
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log
new file mode 100644
index 0000000000..9ac9b6304c
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#start	2011-03-18-19-06-08
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
+1300475168.784020	j4u32Pc5bif	141.142.220.118	48649	208.80.152.118	80	1	GET	bits.wikimedia.org	/skins-1.5/monobook/main.css	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.916018	VW0XPVINV8a	141.142.220.118	49997	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/6/63/Wikipedia-logo.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.916183	3PKsZ2Uye21	141.142.220.118	49996	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.918358	GSxOnSLghOa	141.142.220.118	49998	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/b/bd/Bookshelf-40x201_6.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.952307	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.952296	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.954820	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	1	GET	upload.wikimedia.org	/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.962687	i2rO3KD1Syg	141.142.220.118	35642	208.80.152.2	80	1	GET	meta.wikimedia.org	/images/wikimedia-button.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.975934	VW0XPVINV8a	141.142.220.118	49997	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.976436	3PKsZ2Uye21	141.142.220.118	49996	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475168.979264	GSxOnSLghOa	141.142.220.118	49998	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475169.014619	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475169.014593	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+1300475169.014927	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
new file mode 100755
index 0000000000..7a4225d718
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#start	2011-03-18-19-06-08
+#fields	ts	level	message	location
+#types	time	enum	string	string
+1300475168.843894	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475168.975800	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475168.976327	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475168.979160	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.012666	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.012730	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.014860	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.022665	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.036294	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.036798	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.039923	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.074793	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.074938	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475169.075065	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro b/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro
new file mode 100644
index 0000000000..be6c0e9e9e
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro
@@ -0,0 +1,14 @@
+# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-diff reporter.log
+# @TEST-EXEC: btest-diff http.log
+
+@load base/protocols/http
+
+event bro_init()
+	{
+	# Both the default filter for the http stream and this new one will
+	# attempt to have the same writer write to path "http", which will
+	# be reported as a warning and the write skipped.
+	local filter: Log::Filter = [$name="host-only", $include=set("host")];
+	Log::add_filter(HTTP::LOG, filter);
+	}

From 4abcfa1f66b2dc9f82b0a40d591ecb39bfaa1fd7 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 25 Jul 2012 12:42:46 -0500
Subject: [PATCH 089/238] Fix complaint from valgrind about uninitialized
 memory usage.

---
 src/util.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/util.cc b/src/util.cc
index a34f41dadb..be560928d6 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -691,6 +691,7 @@ void init_random_seed(uint32 seed, const char* read_file, const char* write_file
 	{
 	static const int bufsiz = 16;
 	uint32 buf[bufsiz];
+	memset(buf, 0, sizeof(buf));
 	int pos = 0;	// accumulates entropy
 	bool seeds_done = false;
 

From 7e228f1d6b8cbd0f1b096c77953fb4339a895d7e Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 25 Jul 2012 13:58:08 -0700
Subject: [PATCH 090/238] Silencing compiler warnings.

---
 src/input/Manager.cc | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index f38613a6f8..90d7eae2f4 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1816,7 +1816,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 
 	case TYPE_ADDR:
 		{
-		int length;
+		int length = 0;
 		switch ( val->val.addr_val.family ) {
 		case IPv4:
 			length = sizeof(val->val.addr_val.in.in4);
@@ -1837,7 +1837,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 
 	case TYPE_SUBNET:
 		{
-		int length;
+		int length = 0;
 		switch ( val->val.subnet_val.prefix.family ) {
 		case IPv4:
 			length = sizeof(val->val.addr_val.in.in4);
@@ -1968,7 +1968,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 
 	case TYPE_ADDR:
 		{
-		IPAddr* addr;
+		IPAddr* addr = 0;
 		switch ( val->val.addr_val.family ) {
 		case IPv4:
 			addr = new IPAddr(val->val.addr_val.in.in4);
@@ -1989,7 +1989,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 
 	case TYPE_SUBNET:
 		{
-		IPAddr* addr;
+		IPAddr* addr = 0;
 		switch ( val->val.subnet_val.prefix.family ) {
 		case IPv4:
 			addr = new IPAddr(val->val.subnet_val.prefix.in.in4);

From a33e9a69417a1ae4a8e54d1bc929967d4cd1f0df Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 25 Jul 2012 13:58:23 -0700
Subject: [PATCH 091/238] Fixing FreeBSD compiler error.

---
 src/logging/writers/ElasticSearch.cc |  5 +++--
 src/util.h                           | 13 +++++++------
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index b7edcf6aa6..2da79ed7b9 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -3,16 +3,17 @@
 // This is experimental code that is not yet ready for production usage.
 //
 
+
 #include "config.h"
 
 #ifdef USE_ELASTICSEARCH
 
+#include "util.h" // Needs to come first for stdint.h
+
 #include <string>
 #include <errno.h>
 
-#include "util.h"
 #include "BroString.h"
-
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
 
diff --git a/src/util.h b/src/util.h
index 048ec384e3..5d1bdf188a 100644
--- a/src/util.h
+++ b/src/util.h
@@ -3,6 +3,13 @@
 #ifndef util_h
 #define util_h
 
+// Expose C99 functionality from inttypes.h, which would otherwise not be
+// available in C++.
+#define __STDC_FORMAT_MACROS
+#define __STDC_LIMIT_MACROS
+#include <inttypes.h>
+#include <stdint.h>
+
 #include <string>
 #include <stdio.h>
 #include <stdlib.h>
@@ -10,12 +17,6 @@
 #include <stdarg.h>
 #include "config.h"
 
-// Expose C99 functionality from inttypes.h, which would otherwise not be
-// available in C++.
-#define __STDC_FORMAT_MACROS
-#define __STDC_LIMIT_MACROS
-#include <inttypes.h>
-
 #if __STDC__
 #define myattribute __attribute__
 #else

From f2a0afad3c6dbb274e5631680fe238ec841ed37f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 25 Jul 2012 17:01:47 -0400
Subject: [PATCH 092/238] Fixes to elasticsearch plugin to make libcurl handle
 http responses correctly.

---
 src/logging/writers/ElasticSearch.cc | 4 ++--
 src/logging/writers/ElasticSearch.h  | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 2da79ed7b9..cc6f8b1c4f 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -359,10 +359,10 @@ CURL* ElasticSearch::HTTPSetup()
 	return handle;
 	}
 
-bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
+size_t ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
 	{
 	//TODO: Do some verification on the result?
-	return true;
+	return size;
 	}
 
 bool ElasticSearch::HTTPSend(CURL *handle)
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index 0d863f2f19..0e88bf3e88 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -45,7 +45,7 @@ private:
 	bool UpdateIndex(double now, double rinterval, double rbase);
 
 	CURL* HTTPSetup();
-	bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
+	size_t HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
 	bool HTTPSend(CURL *handle);
 
 	// Buffers, etc.

From c3aba199f6f6d580392b28f87f93f7aa6c2d2e9f Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 25 Jul 2012 17:40:21 -0500
Subject: [PATCH 093/238] Fix build warnings

---
 scripts/base/frameworks/logging/writers/elasticsearch.bro | 2 +-
 scripts/policy/tuning/logs-to-elasticsearch.bro           | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index a6a485226a..b0e8fac40e 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -23,7 +23,7 @@ export {
 	const index_prefix = "bro" &redef;
 
 	## The ES type prefix comes before the name of the related log.
-	## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.
+	## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
 	const type_prefix = "" &redef;
 
 	## The time before an ElasticSearch transfer will timeout.
diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro
index c3cc9d5002..b4d16a19a1 100644
--- a/scripts/policy/tuning/logs-to-elasticsearch.bro
+++ b/scripts/policy/tuning/logs-to-elasticsearch.bro
@@ -6,13 +6,13 @@ export {
 	## An elasticsearch specific rotation interval.
 	const rotation_interval = 24hr &redef;
 
-	## Optionally ignore any :bro:enum:`Log::ID` from being sent to
+	## Optionally ignore any :bro:type:`Log::ID` from being sent to
 	## ElasticSearch with this script.
 	const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
 
-	## If you want to explicitly only send certain :bro:enum:`Log::ID` 
+	## If you want to explicitly only send certain :bro:type:`Log::ID` 
 	## streams, add them to this set.  If the set remains empty, all will 
-	## be sent.  The :bro:id:`excluded_log_ids` option will remain in 
+	## be sent.  The :bro:id:`LogElasticSearch::excluded_log_ids` option will remain in 
 	## effect as well.
 	const send_logs: set[string] = set() &redef;
 }
@@ -42,4 +42,4 @@ event bro_init() &priority=-5
 		{
 		Log::add_filter(id, filter);
 		}
-	}
\ No newline at end of file
+	}

From c48a16664b521bbcaa0fa60e37ae65b49202b168 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 25 Jul 2012 18:05:42 -0500
Subject: [PATCH 094/238] Fix double close() in FilerSerializer class.

---
 src/Serializer.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Serializer.cc b/src/Serializer.cc
index 06bbf73f48..97ee8f743c 100644
--- a/src/Serializer.cc
+++ b/src/Serializer.cc
@@ -742,9 +742,10 @@ FileSerializer::~FileSerializer()
 		io->Flush();
 
 	delete [] file;
-	delete io;
 
-	if ( fd >= 0 )
+	if ( io )
+		delete io;  // destructor will call close() on fd
+	else if ( fd >= 0 )
 		close(fd);
 	}
 
@@ -808,7 +809,7 @@ void FileSerializer::CloseFile()
 	if ( io )
 		io->Flush();
 
-	if ( fd >= 0 )
+	if ( fd >= 0 && ! io ) // destructor of io calls close() on fd
 		close(fd);
 	fd = -1;
 

From 84399c5d7dae83ae252c08b7a2766f3bb212c1e4 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Thu, 26 Jul 2012 08:58:12 -0700
Subject: [PATCH 095/238] add testcase for subrecords to input framework tests

---
 .../out                                       | 14 ++++
 .../base/frameworks/input/subrecord.bro       | 70 +++++++++++++++++++
 2 files changed, 84 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.subrecord/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/subrecord.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.subrecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.subrecord/out
new file mode 100644
index 0000000000..c7e46dfacd
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.subrecord/out
@@ -0,0 +1,14 @@
+{
+[-42] = [sub=[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, two=[a=1.2.3.4, d=3.14]], t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/subrecord.bro b/testing/btest/scripts/base/frameworks/input/subrecord.bro
new file mode 100644
index 0000000000..8c845a1842
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/subrecord.bro
@@ -0,0 +1,70 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	sub.b	i	sub.e	sub.c	sub.p	sub.sn	sub.two.a	sub.two.d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type SubVal2: record {
+	a: addr;
+	d: double;
+};
+
+type SubVal: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	two: SubVal2;
+};
+
+type Val: record {
+	sub: SubVal;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From 734e5f68d377679df9106e534e20f923cffaf99c Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 26 Jul 2012 12:40:12 -0500
Subject: [PATCH 096/238] Add more error handling for close() calls.

---
 src/ChunkedIO.cc             |  6 +++---
 src/FlowSrc.cc               |  2 +-
 src/RemoteSerializer.cc      | 22 ++++++++++++----------
 src/Serializer.cc            |  4 ++--
 src/logging/writers/Ascii.cc |  2 +-
 src/util.cc                  | 27 +++++++++++++++++++++++++--
 src/util.h                   |  3 +++
 7 files changed, 47 insertions(+), 19 deletions(-)

diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc
index f5bcb4b7c1..2c766c7eb1 100644
--- a/src/ChunkedIO.cc
+++ b/src/ChunkedIO.cc
@@ -76,7 +76,7 @@ void ChunkedIO::DumpDebugData(const char* basefnname, bool want_reads)
 		ChunkedIOFd io(fd, "dump-file");
 		io.Write(*i);
 		io.Flush();
-		close(fd);
+		safe_close(fd);
 		}
 
 	l->clear();
@@ -127,7 +127,7 @@ ChunkedIOFd::~ChunkedIOFd()
 
 	delete [] read_buffer;
 	delete [] write_buffer;
-	close(fd);
+	safe_close(fd);
 
 	if ( partial )
 		{
@@ -686,7 +686,7 @@ ChunkedIOSSL::~ChunkedIOSSL()
 		ssl = 0;
 		}
 
-	close(socket);
+	safe_close(socket);
 	}
 
 
diff --git a/src/FlowSrc.cc b/src/FlowSrc.cc
index fe6998ea79..59ce3fd6a4 100644
--- a/src/FlowSrc.cc
+++ b/src/FlowSrc.cc
@@ -58,7 +58,7 @@ void FlowSrc::Process()
 
 void FlowSrc::Close()
 	{
-	close(selectable_fd);
+	safe_close(selectable_fd);
 	}
 
 
diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc
index 7ed8b9318e..4e9ccb7dd2 100644
--- a/src/RemoteSerializer.cc
+++ b/src/RemoteSerializer.cc
@@ -647,7 +647,7 @@ void RemoteSerializer::Fork()
 			exit(1); // FIXME: Better way to handle this?
 			}
 
-		close(pipe[1]);
+		safe_close(pipe[1]);
 
 		return;
 		}
@@ -664,12 +664,12 @@ void RemoteSerializer::Fork()
 			}
 
 		child.SetParentIO(io);
-		close(pipe[0]);
+		safe_close(pipe[0]);
 
 		// Close file descriptors.
-		close(0);
-		close(1);
-		close(2);
+		safe_close(0);
+		safe_close(1);
+		safe_close(2);
 
 		// Be nice.
 		setpriority(PRIO_PROCESS, 0, 5);
@@ -4001,7 +4001,7 @@ bool SocketComm::Connect(Peer* peer)
 		if ( connect(sockfd, res->ai_addr, res->ai_addrlen) < 0 )
 			{
 			Error(fmt("connect failed: %s", strerror(errno)), peer);
-			close(sockfd);
+			safe_close(sockfd);
 			sockfd = -1;
 			continue;
 			}
@@ -4174,16 +4174,18 @@ bool SocketComm::Listen()
 			{
 			Error(fmt("can't bind to %s:%s, %s", l_addr_str.c_str(),
 			          port_str, strerror(errno)));
-			close(fd);
 
 			if ( errno == EADDRINUSE )
 				{
 				// Abandon completely this attempt to set up listening sockets,
 				// try again later.
+				safe_close(fd);
 				CloseListenFDs();
 				listen_next_try = time(0) + bind_retry_interval;
 				return false;
 				}
+
+			safe_close(fd);
 			continue;
 			}
 
@@ -4191,7 +4193,7 @@ bool SocketComm::Listen()
 			{
 			Error(fmt("can't listen on %s:%s, %s", l_addr_str.c_str(),
 			          port_str, strerror(errno)));
-			close(fd);
+			safe_close(fd);
 			continue;
 			}
 
@@ -4227,7 +4229,7 @@ bool SocketComm::AcceptConnection(int fd)
 		{
 		Error(fmt("accept fail, unknown address family %d",
 		          client.ss.ss_family));
-		close(clientfd);
+		safe_close(clientfd);
 		return false;
 		}
 
@@ -4298,7 +4300,7 @@ const char* SocketComm::MakeLogString(const char* msg, Peer* peer)
 void SocketComm::CloseListenFDs()
 	{
 	for ( size_t i = 0; i < listen_fds.size(); ++i )
-		close(listen_fds[i]);
+		safe_close(listen_fds[i]);
 
 	listen_fds.clear();
 	}
diff --git a/src/Serializer.cc b/src/Serializer.cc
index 97ee8f743c..fc6d00d06c 100644
--- a/src/Serializer.cc
+++ b/src/Serializer.cc
@@ -746,7 +746,7 @@ FileSerializer::~FileSerializer()
 	if ( io )
 		delete io;  // destructor will call close() on fd
 	else if ( fd >= 0 )
-		close(fd);
+		safe_close(fd);
 	}
 
 bool FileSerializer::Open(const char* file, bool pure)
@@ -810,7 +810,7 @@ void FileSerializer::CloseFile()
 		io->Flush();
 
 	if ( fd >= 0 && ! io ) // destructor of io calls close() on fd
-		close(fd);
+		safe_close(fd);
 	fd = -1;
 
 	delete [] file;
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 4d2f59ea72..0ccdd1f569 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -86,7 +86,7 @@ void Ascii::CloseFile(double t)
 		WriteHeaderField("end", ts);
 		}
 
-	close(fd);
+	safe_close(fd);
 	fd = 0;
 	}
 
diff --git a/src/util.cc b/src/util.cc
index be560928d6..171fcdce37 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -722,7 +722,7 @@ void init_random_seed(uint32 seed, const char* read_file, const char* write_file
 			{
 			int amt = read(fd, buf + pos,
 					sizeof(uint32) * (bufsiz - pos));
-			close(fd);
+			safe_close(fd);
 
 			if ( amt > 0 )
 				pos += amt / sizeof(uint32);
@@ -1204,7 +1204,7 @@ void _set_processing_status(const char* status)
 		len -= n;
 		}
 
-	close(fd);
+	safe_close(fd);
 
 	errno = old_errno;
 	}
@@ -1353,6 +1353,29 @@ bool safe_write(int fd, const char* data, int len)
 	return true;
 	}
 
+void safe_close(int fd)
+	{
+	/*
+	 * Failure cases of close(2) are ...
+	 * EBADF: Indicative of programming logic error that needs to be fixed, we
+	 *        should always be attempting to close a valid file descriptor.
+	 * EINTR: Ignore signal interruptions, most implementations will actually
+	 *        reclaim the open descriptor and POSIX standard doesn't leave many
+	 *        options by declaring the state of the descriptor as "unspecified".
+	 *        Attempting to inspect actual state or re-attempt close() is not
+	 *        thread safe.
+	 * EIO:   Again the state of descriptor is "unspecified", but don't recover
+	 *        from an I/O error, safe_write() won't either.
+	 */
+	if ( close(fd) < 0 && errno != EINTR )
+		{
+		char buf[128];
+		strerror_r(errno, buf, sizeof(buf));
+		fprintf(stderr, "safe_close error %d: %s\n", errno, buf);
+		abort();
+		}
+	}
+
 void out_of_memory(const char* where)
 	{
 	reporter->FatalError("out of memory in %s.\n", where);
diff --git a/src/util.h b/src/util.h
index 5d1bdf188a..e69167abce 100644
--- a/src/util.h
+++ b/src/util.h
@@ -297,6 +297,9 @@ inline size_t pad_size(size_t size)
 // thread-safe as long as no two threads write to the same descriptor.
 extern bool safe_write(int fd, const char* data, int len);
 
+// Wraps close(2) to emit error messages and abort on unrecoverable errors.
+extern void safe_close(int fd);
+
 extern void out_of_memory(const char* where);
 
 inline void* safe_realloc(void* ptr, size_t size)

From 1a49363bbec6e7b6576fc16b780a0728dc99a7c4 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Thu, 26 Jul 2012 12:12:54 -0700
Subject: [PATCH 097/238] add testcase for subrecords and events add missing
 binary testcase (Baseline is in master, testcase is missing for some reason)
 make error output for nonmatching event types much more verbose

---
 src/input/Manager.cc                          |  6 +-
 .../out                                       | 12 +++
 .../scripts/base/frameworks/input/binary.bro  | 56 ++++++++++++++
 .../base/frameworks/input/subrecord-event.bro | 77 +++++++++++++++++++
 4 files changed, 150 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.subrecord-event/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/binary.bro
 create mode 100644 testing/btest/scripts/base/frameworks/input/subrecord-event.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 90d7eae2f4..40e3c413bb 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -443,7 +443,11 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 		if ( !same_type((*args)[2], fields ) )
 			{
-			reporter->Error("Incompatible type for event");
+			ODesc desc1;
+			ODesc desc2;
+			(*args)[2]->Describe(&desc1);
+			fields->Describe(&desc2);
+			reporter->Error("Incompatible type '%s':%s for event which needs type '%s':%s\n", type_name((*args)[2]->Tag()), desc1.Bytes(), type_name(fields->Tag()), desc2.Bytes());
 			return false;
 			}
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.subrecord-event/out b/testing/btest/Baseline/scripts.base.frameworks.input.subrecord-event/out
new file mode 100644
index 0000000000..197cb54df9
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.subrecord-event/out
@@ -0,0 +1,12 @@
+[sub=[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, two=[a=1.2.3.4, d=3.14]], t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro
new file mode 100644
index 0000000000..86e02196b5
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/binary.bro
@@ -0,0 +1,56 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+redef InputAscii::separator = "|";
+redef InputAscii::set_separator = ",";
+redef InputAscii::empty_field = "(empty)";
+redef InputAscii::unset_field = "-";
+
+@TEST-START-FILE input.log
+#separator |
+#set_separator|,
+#empty_field|(empty)
+#unset_field|-
+#path|ssh
+#start|2012-07-20-01-49-19
+#fields|data|data2
+#types|string|string
+abc\x0a\xffdef|DATA2
+abc\x7c\xffdef|DATA2
+abc\xff\x7cdef|DATA2
+#end|2012-07-20-01-49-19
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+type Val: record {
+		data: string;
+		data2: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, a: string, b: string)
+	{
+	print outfile, a;
+	print outfile, b;
+	try = try + 1;
+	if ( try == 3 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
+	}
diff --git a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro
new file mode 100644
index 0000000000..244eefbc3b
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro
@@ -0,0 +1,77 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	sub.b	i	sub.e	sub.c	sub.p	sub.sn	sub.two.a	sub.two.d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type SubVal2: record {
+	a: addr;
+	d: double;
+};
+
+type SubVal: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	two: SubVal2;
+};
+
+type Val: record {
+	sub: SubVal;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+
+
+event line(description: Input::EventDescription, tpe: Input::Event, value: Val)
+	{
+	print outfile, value;
+	try = try + 1;
+	if ( try == 7 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;	
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_event([$source="../input.log", $name="ssh", $fields=Val, $ev=line, $want_record=T]);
+	Input::remove("ssh");
+	print "Hi";
+	}

From 8633d91c4021194334bbd06a05483e5ec6ab82db Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Thu, 26 Jul 2012 12:15:06 -0700
Subject: [PATCH 098/238] and remove superflous print.

Yes, I know, look at the diff before committing...
---
 .../btest/scripts/base/frameworks/input/subrecord-event.bro   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro
index 244eefbc3b..4e7dc1690a 100644
--- a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro
+++ b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro
@@ -59,7 +59,7 @@ event line(description: Input::EventDescription, tpe: Input::Event, value: Val)
 	{
 	print outfile, value;
 	try = try + 1;
-	if ( try == 7 )
+	if ( try == 1 )
 		{
 		close(outfile);
 		terminate();
@@ -70,8 +70,6 @@ event bro_init()
 	{
 	try = 0;	
 	outfile = open("../out");
-	# first read in the old stuff into the table...
 	Input::add_event([$source="../input.log", $name="ssh", $fields=Val, $ev=line, $want_record=T]);
 	Input::remove("ssh");
-	print "Hi";
 	}

From 63e8bf72edad62d4118e22be1e61e32404d03f30 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 26 Jul 2012 16:55:49 -0500
Subject: [PATCH 099/238] Change path conflicts between log filters to be
 auto-corrected.

This change makes it so when differing logging filters on the same
stream attempt to write to the same writer/path combination, the path
of the filter doing the later write will be automatically adjusted so
that it does not conflict with the other.  The path is adjusted by
appending "-N", where N is the smallest integer greater or equal to 2
required to resolve the path name conflict.

Addresses #842.
---
 scripts/base/frameworks/logging/main.bro      | 11 ++++-
 src/logging/Manager.cc                        | 41 ++++++++++++++-----
 .../http-2-2.log                              | 23 +++++++++++
 .../http-2.log                                | 23 +++++++++++
 .../http-3.log                                | 23 +++++++++++
 .../reporter.log                              | 17 ++------
 .../logging/writer-path-conflict.bro          | 12 +++++-
 7 files changed, 124 insertions(+), 26 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log

diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro
index cc0d341605..79c9884f9d 100644
--- a/scripts/base/frameworks/logging/main.bro
+++ b/scripts/base/frameworks/logging/main.bro
@@ -96,6 +96,12 @@ export {
 		## file name. Generally, filenames are expected to given
 		## without any extensions; writers will add appropiate
 		## extensions automatically.
+		##
+		## If this path is found to conflict with another filter's
+		## for the same writer type, it is automatically corrected
+		## by appending "-N", where N is the smallest integer greater
+		## or equal to 2 that allows the corrected path name to not
+		## conflict with another filter's.
 		path: string &optional;
 
 		## A function returning the output path for recording entries
@@ -115,7 +121,10 @@ export {
 		## rec: An instance of the streams's ``columns`` type with its
 		##      fields set to the values to be logged.
 		##
-		## Returns: The path to be used for the filter.
+		## Returns: The path to be used for the filter, which will be subject
+		##          to the same automatic correction rules as the *path*
+		##          field of :bro:type:`Log::Filter` in the case of conflicts
+		##          with other filters trying to use the same writer/path pair.
 		path_func: function(id: ID, path: string, rec: any): string &optional;
 
 		## Subset of column names to record. If not given, all
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 3499d55f74..b1b289a478 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -758,22 +758,43 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 #endif
 			}
 
+		Stream::WriterPathPair wpp(filter->writer->AsEnum(), path);
+
 		// See if we already have a writer for this path.
-		Stream::WriterMap::iterator w =
-			stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path));
+		Stream::WriterMap::iterator w = stream->writers.find(wpp);
+
+		if ( w != stream->writers.end() &&
+		     w->second->instantiating_filter != filter->name )
+			{
+			// Auto-correct path due to conflict with another filter over the
+			// same writer/path pair
+			string instantiator = w->second->instantiating_filter;
+			string new_path;
+			unsigned int i = 2;
+
+			do {
+				char num[32];
+				snprintf(num, sizeof(num), "-%u", i++);
+				new_path = path + num;
+				wpp.second = new_path;
+				w = stream->writers.find(wpp);
+			} while ( w != stream->writers.end());
+
+			Unref(filter->path_val);
+			filter->path_val = new StringVal(new_path.c_str());
+
+			reporter->Warning("Write using filter '%s' on path '%s' changed to"
+			  " use new path '%s' to avoid conflict with filter '%s'",
+			  filter->name.c_str(), path.c_str(), new_path.c_str(),
+			  instantiator.c_str());
+
+			path = filter->path = filter->path_val->AsString()->CheckString();
+			}
 
 		WriterFrontend* writer = 0;
 
 		if ( w != stream->writers.end() )
 			{
-			if ( w->second->instantiating_filter != filter->name )
-				{
-				reporter->Warning("Skipping write to filter '%s' on path '%s'"
-				  " because filter '%s' has already instantiated the same"
-				  " writer type for that path", filter->name.c_str(),
-				  filter->path.c_str(), w->second->instantiating_filter.c_str());
-				continue;
-				}
 			// We know this writer already.
 			writer = w->second->writer;
 			}
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log
new file mode 100644
index 0000000000..1e41aca795
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http-2-2
+#start	2011-03-18-19-06-08
+#fields	status_code
+#types	count
+304
+304
+304
+304
+304
+304
+304
+304
+304
+304
+304
+304
+304
+304
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log
new file mode 100644
index 0000000000..4d3622c7a0
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http-2
+#start	2011-03-18-19-06-08
+#fields	host
+#types	string
+bits.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+meta.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+upload.wikimedia.org
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log
new file mode 100644
index 0000000000..727a6c02fa
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log
@@ -0,0 +1,23 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http-3
+#start	2011-03-18-19-06-08
+#fields	uri
+#types	string
+/skins-1.5/monobook/main.css
+/wikipedia/commons/6/63/Wikipedia-logo.png
+/wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png
+/wikipedia/commons/b/bd/Bookshelf-40x201_6.png
+/wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png
+/wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png
+/wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png
+/images/wikimedia-button.png
+/wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png
+/wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png
+/wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png
+/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png
+/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png
+/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png
+#end	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
index 7a4225d718..3514ca5134 100755
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
@@ -6,18 +6,7 @@
 #start	2011-03-18-19-06-08
 #fields	ts	level	message	location
 #types	time	enum	string	string
-1300475168.843894	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475168.975800	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475168.976327	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475168.979160	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.012666	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.012730	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.014860	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.022665	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.036294	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.036798	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.039923	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.074793	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.074938	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
-1300475169.075065	Reporter::WARNING	Skipping write to filter 'host-only' on path 'http' because filter 'default' has already instantiated the same writer type for that path	(empty)
+1300475168.843894	Reporter::WARNING	Write using filter 'host-only' on path 'http' changed to use new path 'http-2' to avoid conflict with filter 'default'	(empty)
+1300475168.843894	Reporter::WARNING	Write using filter 'uri-only' on path 'http' changed to use new path 'http-3' to avoid conflict with filter 'default'	(empty)
+1300475168.843894	Reporter::WARNING	Write using filter 'status-only' on path 'http-2' changed to use new path 'http-2-2' to avoid conflict with filter 'host-only'	(empty)
 #end	2011-03-18-19-06-13
diff --git a/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro b/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro
index be6c0e9e9e..908fb43c72 100644
--- a/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro
+++ b/testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro
@@ -1,6 +1,9 @@
 # @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT
 # @TEST-EXEC: btest-diff reporter.log
 # @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff http-2.log
+# @TEST-EXEC: btest-diff http-3.log
+# @TEST-EXEC: btest-diff http-2-2.log
 
 @load base/protocols/http
 
@@ -8,7 +11,14 @@ event bro_init()
 	{
 	# Both the default filter for the http stream and this new one will
 	# attempt to have the same writer write to path "http", which will
-	# be reported as a warning and the write skipped.
+	# be reported as a warning and the path auto-corrected to "http-2"
 	local filter: Log::Filter = [$name="host-only", $include=set("host")];
+	# Same deal here, but should be auto-corrected to "http-3".
+	local filter2: Log::Filter = [$name="uri-only", $include=set("uri")];
+	# Conflict between auto-correct paths needs to be corrected, too, this
+	# time it will be "http-2-2".
+	local filter3: Log::Filter = [$path="http-2", $name="status-only", $include=set("status_code")];
 	Log::add_filter(HTTP::LOG, filter);
+	Log::add_filter(HTTP::LOG, filter2);
+	Log::add_filter(HTTP::LOG, filter3);
 	}

From 412bebb7031d7954a1ce20deef3d6a2f2face192 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 26 Jul 2012 15:24:27 -0700
Subject: [PATCH 100/238] Tweaking the custom-rotate test to produce stable
 output.

There seems to be a race condition in capturing the external shell's
stdout output reliably. As far as I can tell, Bro's doing everything
correctly though, the log postprocessors gets executed as expected. So
I rewrote the test to capture the output in a separate file first, and
that seems to solve the test failures.
---
 .../btest/scripts/base/frameworks/logging/rotate-custom.bro  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
index 8a7f16d182..07fc8cef7c 100644
--- a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
+++ b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
@@ -1,5 +1,6 @@
 #
-#@TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out
+# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out.tmp
+# @TEST-EXEC: cat out.tmp pp.log | sort >out
 # @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
@@ -19,7 +20,7 @@ export {
 }
 
 redef Log::default_rotation_interval = 1hr;
-redef Log::default_rotation_postprocessor_cmd = "echo 1st";
+redef Log::default_rotation_postprocessor_cmd = "echo 1st >>pp.log";
 
 function custom_rotate(info: Log::RotationInfo) : bool
 {

From ef3b75129f393728f74ba039a71183063b313b02 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 26 Jul 2012 15:38:12 -0700
Subject: [PATCH 101/238] Updating baseline for custom-rotate test.

---
 .../.stderr                                            | 10 ----------
 .../scripts.base.frameworks.logging.rotate-custom/out  | 10 ++++++++++
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr
index e1958d67ad..e69de29bb2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/.stderr
@@ -1,10 +0,0 @@
-1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii
-1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii
-1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii
-1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii
-1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii
-1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii
-1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii
-1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii
-1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii
-1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
index 91b6f5de7a..19354f8df2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
@@ -1,3 +1,13 @@
+1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0 ascii
+1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0 ascii
+1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0 ascii
+1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0 ascii
+1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0 ascii
+1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0 ascii
+1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0 ascii
+1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0 ascii
+1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0 ascii
+1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_03.00.05.log, path=test2, open=1299466805.0, close=1299470395.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_03.59.55.log, path=test2, open=1299470395.0, close=1299470405.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_04.00.05.log, path=test2, open=1299470405.0, close=1299473995.0, terminating=F]

From 743fc1680dc9d4c04f38ca80c7ef4e5b88e8f4cb Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 26 Jul 2012 16:31:20 -0700
Subject: [PATCH 102/238] Improving error handling for threads.

If a thread command fails (like the input framework not finding a
file), that now (1) no longer hangs Bro, and (2) even allows for
propagating error messages back before the thread is stops.

(Actually, the thread doesn't really "stop"; the thread manager keeps
threads around independent of their success; but it no longer polls
them for input.)

Closes #858.
---
 src/threading/Manager.cc   | 14 +++++++++++---
 src/threading/Manager.h    | 17 ++++++++++-------
 src/threading/MsgThread.cc | 22 ++++++++++++++++++----
 3 files changed, 39 insertions(+), 14 deletions(-)

diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc
index 53c11f2ee9..cfc44596e1 100644
--- a/src/threading/Manager.cc
+++ b/src/threading/Manager.cc
@@ -80,8 +80,10 @@ double Manager::NextTimestamp(double* network_time)
 
 	for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ )
 		{
-			if ( (*i)->MightHaveOut() )
-				return timer_mgr->Time();
+		MsgThread* t = *i;
+
+		if ( (*i)->MightHaveOut() && ! t->Killed() )
+			return timer_mgr->Time();
 		}
 
 	return -1.0;
@@ -95,6 +97,12 @@ void Manager::KillThreads()
 		(*i)->Kill();
         }
 
+void Manager::KillThread(BasicThread* thread)
+	{
+	DBG_LOG(DBG_THREADING, "Killing thread %s ...", thread->Name());
+	thread->Kill();
+	}
+
 void Manager::Process()
 	{
 	bool do_beat = false;
@@ -114,7 +122,7 @@ void Manager::Process()
 		if ( do_beat )
 			t->Heartbeat();
 
-		while ( t->HasOut() )
+		while ( t->HasOut() && ! t->Killed() )
 			{
 			Message* msg = t->RetrieveOut();
 
diff --git a/src/threading/Manager.h b/src/threading/Manager.h
index be81c69ba0..b46a06a46e 100644
--- a/src/threading/Manager.h
+++ b/src/threading/Manager.h
@@ -74,6 +74,16 @@ public:
 	 */
 	void ForceProcessing() { Process(); }
 
+	/**
+	 * Signals a specific threads to terminate immediately.
+	 */
+	void KillThread(BasicThread* thread);
+
+	/**
+	 * Signals all threads to terminate immediately.
+	 */
+	void KillThreads();
+
 protected:
 	friend class BasicThread;
 	friend class MsgThread;
@@ -106,13 +116,6 @@ protected:
 	 */
 	virtual double NextTimestamp(double* network_time);
 
-	/**
-	 * Kills all thread immediately. Note that this may cause race conditions
-	 * if a child thread currently holds a lock that might block somebody
-	 * else.
-	 */
-	virtual void KillThreads();
-
 	/**
 	 * Part of the IOSource interface.
 	 */
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index 48c7253885..e0f3fd8b0c 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -70,6 +70,16 @@ private:
 	Type type;
 };
 
+// A message from the the child to the main process, requesting suicide.
+class KillMeMessage : public OutputMessage<MsgThread>
+{
+public:
+	KillMeMessage(MsgThread* thread)
+		: OutputMessage<MsgThread>("ReporterMessage", thread) 	{}
+
+	virtual bool Process()	{ thread_mgr->KillThread(Object()); return true; }
+};
+
 #ifdef DEBUG
 // A debug message from the child to be passed on to the DebugLogger.
 class DebugMessage : public OutputMessage<MsgThread>
@@ -346,16 +356,20 @@ void MsgThread::Run()
 
 		if ( ! result )
 			{
-			string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
-			Error(s.c_str());
-			break;
+			Error("terminating thread");
+
+			// This will eventually kill this thread, but only
+			// after all other outgoing messages (in particular
+			// error messages have been processed by then main
+			// thread).
+			SendOut(new KillMeMessage(this));
 			}
 		}
 
 	// In case we haven't send the finish method yet, do it now. Reading
 	// global network_time here should be fine, it isn't changing
 	// anymore.
-	if ( ! finished )
+	if ( ! finished && ! Killed() )
 		{
 		OnFinish(network_time);
 		Finished();

From 86ae7d8b7c6500cde05fd478ea4f011168c25aec Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 26 Jul 2012 16:38:03 -0700
Subject: [PATCH 103/238] Test for input framework failing to find a file.

The output isn't the nicest yet ...
---
 .../bro..stderr                               |  5 ++++
 .../base/frameworks/input/missing-file.bro    | 30 +++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr
 create mode 100644 testing/btest/scripts/base/frameworks/input/missing-file.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr
new file mode 100644
index 0000000000..4380007b93
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr
@@ -0,0 +1,5 @@
+error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat
+error: does-not-exist.dat/Input::READER_ASCII: Init failed
+warning: Stream input is already queued for removal. Ignoring remove.
+error: does-not-exist.dat/Input::READER_ASCII: terminating thread
+received termination signal
diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro
new file mode 100644
index 0000000000..269e287acc
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro
@@ -0,0 +1,30 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff bro/.stderr
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	i: int;
+	b: bool;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool)
+	{
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="does-not-exist.dat", $name="input", $fields=Val, $ev=line]);
+	Input::remove("input");
+	}

From f5862fb01408884079b84467cf139aad6046e3f1 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 26 Jul 2012 17:15:10 -0700
Subject: [PATCH 104/238] Preventing writers/readers from receiving further
 messages after a failure.

Once a writer/reader Do* method has returned false, no further ones
will be executed anymore. This is primarily a safety mechanism to make
it easier for writer/reader authors as otherwise they would often need
to track the failure state themselves (because with the now delayed
termination from the earlier commit, furhter messages can now still
arrive for a little bit).
---
 CHANGES                      | 13 +++++++++++++
 VERSION                      |  2 +-
 src/input/ReaderBackend.cc   | 13 ++++++++++++-
 src/logging/WriterBackend.cc | 29 +++++++++++++++++++++++++----
 src/logging/WriterBackend.h  |  2 ++
 src/logging/writers/Ascii.cc |  2 +-
 src/threading/MsgThread.cc   |  2 ++
 src/threading/MsgThread.h    |  7 +++++++
 8 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/CHANGES b/CHANGES
index 3fe0fa2b73..44a3edc3c6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,17 @@
 
+2.0-891 | 2012-07-26 17:15:10 -0700
+
+  * Reader/writer API: preventing plugins from receiving further
+    messages after a failure. (Robin Sommer)
+
+  * New test for input framework that fails to find a file. (Robin
+    Sommer)
+    
+  * Improving error handling for threads. (Robin Sommer)
+    
+  * Tweaking the custom-rotate test to produce stable output. (Robin
+    Sommer)
+
 2.0-884 | 2012-07-26 14:33:21 -0700
 
   * Add comprehensive error handling for close() calls. (Jon Siwek)
diff --git a/VERSION b/VERSION
index ced5c78870..b97bde7b8d 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0-884
+2.0-891
diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc
index 88a78c3cd7..81060be7d5 100644
--- a/src/input/ReaderBackend.cc
+++ b/src/input/ReaderBackend.cc
@@ -191,6 +191,9 @@ void ReaderBackend::SendEntry(Value* *vals)
 bool ReaderBackend::Init(const int arg_num_fields,
 		         const threading::Field* const* arg_fields)
 	{
+	if ( Failed() )
+		return true;
+
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
@@ -210,7 +213,9 @@ bool ReaderBackend::Init(const int arg_num_fields,
 
 bool ReaderBackend::OnFinish(double network_time)
 	{
-	DoClose();
+	if ( ! Failed() )
+		DoClose();
+
 	disabled = true; // frontend disables itself when it gets the Close-message.
 	SendOut(new ReaderClosedMessage(frontend));
 
@@ -231,6 +236,9 @@ bool ReaderBackend::Update()
 	if ( disabled )
 		return false;
 
+	if ( Failed() )
+		return true;
+
 	bool success = DoUpdate();
 	if ( ! success )
 		DisableFrontend();
@@ -248,6 +256,9 @@ void ReaderBackend::DisableFrontend()
 
 bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
 	{
+	if ( Failed() )
+		return true;
+
 	return DoHeartbeat(network_time, current_time);
 	}
 
diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index 2933062eff..afdc4b99c5 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -174,6 +174,9 @@ bool WriterBackend::Init(int arg_num_fields, const Field* const* arg_fields)
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
+	if ( Failed() )
+		return true;
+
 	if ( ! DoInit(*info, arg_num_fields, arg_fields) )
 		{
 		DisableFrontend();
@@ -222,12 +225,15 @@ bool WriterBackend::Write(int arg_num_fields, int num_writes, Value*** vals)
 
 	bool success = true;
 
-	for ( int j = 0; j < num_writes; j++ )
+	if ( ! Failed() )
 		{
-		success = DoWrite(num_fields, fields, vals[j]);
+		for ( int j = 0; j < num_writes; j++ )
+			{
+			success = DoWrite(num_fields, fields, vals[j]);
 
-		if ( ! success )
-			break;
+			if ( ! success )
+				break;
+			}
 		}
 
 	DeleteVals(num_writes, vals);
@@ -244,6 +250,9 @@ bool WriterBackend::SetBuf(bool enabled)
 		// No change.
 		return true;
 
+	if ( Failed() )
+		return true;
+
 	buffering = enabled;
 
 	if ( ! DoSetBuf(enabled) )
@@ -258,6 +267,9 @@ bool WriterBackend::SetBuf(bool enabled)
 bool WriterBackend::Rotate(const char* rotated_path, double open,
 			   double close, bool terminating)
 	{
+	if ( Failed() )
+		return true;
+
 	if ( ! DoRotate(rotated_path, open, close, terminating) )
 		{
 		DisableFrontend();
@@ -269,6 +281,9 @@ bool WriterBackend::Rotate(const char* rotated_path, double open,
 
 bool WriterBackend::Flush(double network_time)
 	{
+	if ( Failed() )
+		return true;
+
 	if ( ! DoFlush(network_time) )
 		{
 		DisableFrontend();
@@ -280,11 +295,17 @@ bool WriterBackend::Flush(double network_time)
 
 bool WriterBackend::OnFinish(double network_time)
 	{
+	if ( Failed() )
+		return true;
+
 	return DoFinish(network_time);
 	}
 
 bool WriterBackend::OnHeartbeat(double network_time, double current_time)
 	{
+	if ( Failed() )
+		return true;
+
 	SendOut(new FlushWriteBufferMessage(frontend));
 	return DoHeartbeat(network_time, current_time);
 	}
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index d5f2be225e..77dbe71f45 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -182,6 +182,8 @@ public:
 	/**
 	 * Disables the frontend that has instantiated this backend. Once
 	 * disabled,the frontend will not send any further message over.
+	 *
+	 * TODO: Do we still need this method (and the corresponding message)?
 	 */
 	void DisableFrontend();
 
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 0ccdd1f569..c471b3db0c 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -92,7 +92,7 @@ void Ascii::CloseFile(double t)
 
 bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
 	{
-    assert(! fd);
+	assert(! fd);
 
 	string path = info.path;
 
diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc
index e0f3fd8b0c..6c63c5a287 100644
--- a/src/threading/MsgThread.cc
+++ b/src/threading/MsgThread.cc
@@ -154,6 +154,7 @@ MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this)
 	{
 	cnt_sent_in = cnt_sent_out = 0;
 	finished = false;
+	failed = false;
 	thread_mgr->AddMsgThread(this);
 	}
 
@@ -363,6 +364,7 @@ void MsgThread::Run()
 			// error messages have been processed by then main
 			// thread).
 			SendOut(new KillMeMessage(this));
+			failed = true;
 			}
 		}
 
diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h
index da505de6be..e3e7c8500f 100644
--- a/src/threading/MsgThread.h
+++ b/src/threading/MsgThread.h
@@ -201,6 +201,12 @@ protected:
 	 */
 	void HeartbeatInChild();
 
+	/** Returns true if a child command has reported a failure. In that case, we'll
+	  * be in the process of killing this thread and no further activity
+	  * should carried out. To be called only from this child thread.
+	  */
+	bool Failed() const { return failed; }
+
 	/**
 	 * Regulatly triggered for execution in the child thread.
 	 *
@@ -294,6 +300,7 @@ private:
 	uint64_t cnt_sent_out;	// Counts message sent by child.
 
 	bool finished;	// Set to true by Finished message.
+	bool failed;	// Set to true when a command failed.
 };
 
 /**

From 76ea1823877677612e159c54edf1958898e7ceb2 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Thu, 26 Jul 2012 21:13:49 -0700
Subject: [PATCH 105/238] make want_record=T the default for events

---
 scripts/base/frameworks/input/main.bro                     | 2 +-
 testing/btest/scripts/base/frameworks/input/binary.bro     | 2 +-
 testing/btest/scripts/base/frameworks/input/event.bro      | 2 +-
 testing/btest/scripts/base/frameworks/input/executeraw.bro | 2 +-
 testing/btest/scripts/base/frameworks/input/raw.bro        | 2 +-
 testing/btest/scripts/base/frameworks/input/rereadraw.bro  | 2 +-
 testing/btest/scripts/base/frameworks/input/streamraw.bro  | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index c31f92dba5..7f015402bc 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -84,7 +84,7 @@ export {
 
 		## If want_record if false (default), the event receives each value in fields as a seperate argument.
 		## If it is set to true, the event receives all fields in a signle record value.
-		want_record: bool &default=F;
+		want_record: bool &default=T;
 
 		## The event that is rised each time a new line is received from the reader.
 		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro
index 86e02196b5..ce7f66a01d 100644
--- a/testing/btest/scripts/base/frameworks/input/binary.bro
+++ b/testing/btest/scripts/base/frameworks/input/binary.bro
@@ -51,6 +51,6 @@ event bro_init()
 	{
 	try = 0;
 	outfile = open("../out");
-	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::remove("input");
 	}
diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro
index f07ca0c43e..d0088472e7 100644
--- a/testing/btest/scripts/base/frameworks/input/event.bro
+++ b/testing/btest/scripts/base/frameworks/input/event.bro
@@ -49,6 +49,6 @@ event bro_init()
 	{
 	try = 0;
 	outfile = open("../out");
-	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::remove("input");
 	}
diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro
index 222b4256d1..626b9cdfd2 100644
--- a/testing/btest/scripts/base/frameworks/input/executeraw.bro
+++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro
@@ -37,6 +37,6 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string)
 event bro_init()
 	{
 	outfile = open("../out.tmp");
-	Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::remove("input");
 	}
diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro
index cb19213173..d15aec22bb 100644
--- a/testing/btest/scripts/base/frameworks/input/raw.bro
+++ b/testing/btest/scripts/base/frameworks/input/raw.bro
@@ -44,6 +44,6 @@ event bro_init()
 	{
 	try = 0;
 	outfile = open("../out");
-	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::remove("input");
 	}
diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro
index 1051351c2b..2fdcdc8f9e 100644
--- a/testing/btest/scripts/base/frameworks/input/rereadraw.bro
+++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro
@@ -44,7 +44,7 @@ event bro_init()
 	{
 	try = 0;
 	outfile = open("../out");
-	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::force_update("input");
 	Input::remove("input");
 	}
diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro
index a6aba88c5f..3bc06f7dea 100644
--- a/testing/btest/scripts/base/frameworks/input/streamraw.bro
+++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro
@@ -58,5 +58,5 @@ event bro_init()
 	{
 	outfile = open("../out");
 	try = 0;
-	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
 	}

From f02ed65878b81dfde81c2483887223bab99ad2e8 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Thu, 26 Jul 2012 21:51:29 -0700
Subject: [PATCH 106/238] Fix crash when encountering an InterpreterException
 in a predicate in logging or input Framework.

Inputframework: did not contain any error handling for this case.

Logging framework: tried to catch the interpreter-exception. However the exception already was caught
by the call-function and not propagated. Instead, call returns a 0-pointer in this case, which
lead to a segmentation fault.
---
 src/input/Manager.cc   |  9 ++++++---
 src/logging/Manager.cc | 21 ++++++---------------
 2 files changed, 12 insertions(+), 18 deletions(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 40e3c413bb..d278933125 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1544,7 +1544,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals)
 
 bool Manager::CallPred(Func* pred_func, const int numvals, ...)
 	{
-	bool result;
+	bool result = false;
 	val_list vl(numvals);
 
 	va_list lP;
@@ -1555,8 +1555,11 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...)
 	va_end(lP);
 
 	Val* v = pred_func->Call(&vl);
-	result = v->AsBool();
-	Unref(v);
+	if ( v ) 
+		{
+		result = v->AsBool();
+		Unref(v);
+		}
 
 	return(result);
 	}
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index b1b289a478..6729ec24d2 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -686,16 +686,13 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 			int result = 1;
 
-			try
+			Val* v = filter->pred->Call(&vl);
+			if ( v ) 
 				{
-				Val* v = filter->pred->Call(&vl);
 				result = v->AsBool();
 				Unref(v);
 				}
 
-			catch ( InterpreterException& e )
-				{ /* Already reported. */ }
-
 			if ( ! result )
 				continue;
 			}
@@ -726,12 +723,9 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 
 			Val* v = 0;
 
-			try
-				{
-				v = filter->path_func->Call(&vl);
-				}
+			v = filter->path_func->Call(&vl);
 
-			catch ( InterpreterException& e )
+			if ( !v )
 				{
 				return false;
 				}
@@ -1381,16 +1375,13 @@ bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, con
 
 	int result = 0;
 
-	try
+	Val* v = func->Call(&vl);
+	if ( v ) 
 		{
-		Val* v = func->Call(&vl);
 		result = v->AsBool();
 		Unref(v);
 		}
 
-	catch ( InterpreterException& e )
-		{ /* Already reported. */ }
-
 	return result;
 	}
 

From a3798070da5dbfd95469c784a6fcae5efdf8203a Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Fri, 27 Jul 2012 07:33:04 -0700
Subject: [PATCH 107/238] update input framework documentation to reflect
 want_record change.

---
 scripts/base/frameworks/input/main.bro | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index 7f015402bc..55da6ae7ec 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -82,11 +82,11 @@ export {
 		## Record describing the fields to be retrieved from the source input.
 		fields: any;
 
-		## If want_record if false (default), the event receives each value in fields as a seperate argument.
-		## If it is set to true, the event receives all fields in a signle record value.
+		## If want_record if false, the event receives each value in fields as a separate argument.
+		## If it is set to true (default), the event receives all fields in a single record value.
 		want_record: bool &default=T;
 
-		## The event that is rised each time a new line is received from the reader.
+		## The event that is raised each time a new line is received from the reader.
 		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
 		ev: any;
 

From 2a9993619f6637ac6afcb8a6e4fd3afcba34a676 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 27 Jul 2012 13:49:49 -0400
Subject: [PATCH 108/238] Script-level rotation postprocessor fix.

- This fixes a problem with writers that don't have a
  postprocessor.  Jon is still looking into the rotation
  problem in the core.
---
 scripts/base/frameworks/logging/main.bro | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro
index 79c9884f9d..db79324d0d 100644
--- a/scripts/base/frameworks/logging/main.bro
+++ b/scripts/base/frameworks/logging/main.bro
@@ -341,8 +341,9 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
 	{
 	if ( info$writer in default_rotation_postprocessors )
 		return default_rotation_postprocessors[info$writer](info);
-
-	return F;
+	else
+		# Return T by default so that postprocessor-less writers don't shutdown.
+		return T;
 	}
 
 function default_path_func(id: ID, path: string, rec: any) : string

From 76520645bb6e134e28adab59d9af93129150db3f Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 27 Jul 2012 13:51:03 -0400
Subject: [PATCH 109/238] Small (potential performance) improvement for logging
 framework.

---
 scripts/base/frameworks/logging/main.bro | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro
index db79324d0d..c29215fd86 100644
--- a/scripts/base/frameworks/logging/main.bro
+++ b/scripts/base/frameworks/logging/main.bro
@@ -348,16 +348,16 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
 
 function default_path_func(id: ID, path: string, rec: any) : string
 	{
+	# The suggested path value is a previous result of this function
+	# or a filter path explicitly set by the user, so continue using it.
+	if ( path != "" )
+		return path;
+
 	local id_str = fmt("%s", id);
 
 	local parts = split1(id_str, /::/);
 	if ( |parts| == 2 )
 		{
-		# The suggested path value is a previous result of this function
-		# or a filter path explicitly set by the user, so continue using it.
-		if ( path != "" )
-			return path;
-
 		# Example: Notice::LOG -> "notice"
 		if ( parts[2] == "LOG" )
 			{

From 1fd0d7a607ddfc2b06a82aa085abcd082841463b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 27 Jul 2012 12:15:21 -0700
Subject: [PATCH 110/238] Changing the start/end markers in logs to open/close
 now reflecting wall clock.

Triggers lots of (simple) baseline updates.
---
 NEWS                                          |  6 +--
 src/logging/writers/Ascii.cc                  | 19 +++++----
 src/logging/writers/Ascii.h                   |  2 +-
 testing/btest/Baseline/core.checksums/bad.out | 40 +++++++++----------
 .../btest/Baseline/core.checksums/good.out    | 28 ++++++-------
 .../core.disable-mobile-ipv6/weird.log        |  4 +-
 .../Baseline/core.expr-exception/reporter.log |  4 +-
 testing/btest/Baseline/core.ipv6-frag/dns.log |  4 +-
 .../Baseline/core.print-bpf-filters/conn.log  |  4 +-
 .../Baseline/core.print-bpf-filters/output    | 24 +++++------
 testing/btest/Baseline/core.truncation/output | 16 ++++----
 .../Baseline/core.tunnels.ayiya/conn.log      |  4 +-
 .../Baseline/core.tunnels.ayiya/http.log      |  4 +-
 .../Baseline/core.tunnels.ayiya/tunnel.log    |  4 +-
 .../core.tunnels.false-teredo/dpd.log         |  4 +-
 .../core.tunnels.false-teredo/weird.log       |  4 +-
 .../Baseline/core.tunnels.teredo/conn.log     |  4 +-
 .../Baseline/core.tunnels.teredo/http.log     |  4 +-
 .../Baseline/core.tunnels.teredo/tunnel.log   |  4 +-
 .../conn.log                                  |  4 +-
 .../http.log                                  |  4 +-
 .../tunnel.log                                |  4 +-
 .../weird.log                                 |  4 +-
 .../btest/Baseline/core.vlan-mpls/conn.log    |  4 +-
 .../canonified_loaded_scripts.log             |  4 +-
 .../canonified_loaded_scripts.log             |  4 +-
 .../istate.events-ssl/receiver.http.log       |  4 +-
 .../istate.events-ssl/sender.http.log         |  4 +-
 .../Baseline/istate.events/receiver.http.log  |  4 +-
 .../Baseline/istate.events/sender.http.log    |  4 +-
 .../send.log                                  |  4 +-
 .../ssh-new-default.log                       |  4 +-
 .../ssh.log                                   |  4 +-
 .../test.log                                  |  4 +-
 .../http.log                                  |  4 +-
 .../test.log                                  |  4 +-
 .../ssh.log                                   | 12 +++---
 .../test.log                                  |  4 +-
 .../test.log                                  |  4 +-
 .../ssh.log                                   |  4 +-
 .../ssh.log                                   |  4 +-
 .../ssh.log                                   |  4 +-
 .../ssh.log                                   |  4 +-
 .../ssh.log                                   |  4 +-
 .../ssh.log                                   |  4 +-
 .../local.log                                 |  4 +-
 .../remote.log                                |  4 +-
 .../output                                    | 28 ++++++-------
 .../test.failure.log                          |  4 +-
 .../test.success.log                          |  4 +-
 .../receiver.test.log                         |  4 +-
 .../sender.test.failure.log                   |  4 +-
 .../sender.test.log                           |  4 +-
 .../sender.test.success.log                   |  4 +-
 .../ssh.failure.log                           |  4 +-
 .../ssh.log                                   |  4 +-
 .../out                                       | 22 +---------
 .../out                                       | 40 +++++++++----------
 .../output                                    |  4 +-
 .../ssh.log                                   |  4 +-
 .../ssh.log                                   |  4 +-
 .../testing.log                               |  4 +-
 .../ssh.log                                   |  4 +-
 .../http-2-2.log                              |  4 +-
 .../http-2.log                                |  4 +-
 .../http-3.log                                |  4 +-
 .../http.log                                  |  4 +-
 .../reporter.log                              |  4 +-
 .../manager-1.metrics.log                     |  4 +-
 .../metrics.log                               |  4 +-
 .../manager-1.notice.log                      |  4 +-
 .../notice.log                                |  4 +-
 .../manager-1.notice.log                      |  4 +-
 .../manager-1.notice.log                      |  4 +-
 .../notice.log                                |  4 +-
 .../conn.log                                  |  4 +-
 .../ftp.log                                   |  4 +-
 .../conn.log                                  |  4 +-
 .../ftp.log                                   |  4 +-
 .../http.log                                  |  4 +-
 .../http.log                                  |  4 +-
 .../http.log                                  |  4 +-
 .../http.log                                  |  4 +-
 .../scripts.base.protocols.irc.basic/irc.log  |  4 +-
 .../irc.log                                   |  4 +-
 .../smtp.log                                  |  4 +-
 .../smtp_entities.log                         |  4 +-
 .../smtp_entities.log                         |  4 +-
 .../socks.log                                 |  4 +-
 .../tunnel.log                                |  4 +-
 .../socks.log                                 |  4 +-
 .../tunnel.log                                |  4 +-
 .../tunnel.log                                |  4 +-
 .../scripts.base.protocols.ssl.basic/ssl.log  |  4 +-
 .../knownhosts-all.log                        |  4 +-
 .../knownhosts-local.log                      |  4 +-
 .../knownhosts-remote.log                     |  4 +-
 .../knownservices-all.log                     |  4 +-
 .../knownservices-local.log                   |  4 +-
 .../knownservices-remote.log                  |  4 +-
 .../dns.log                                   |  4 +-
 testing/scripts/diff-remove-timestamps        |  2 +-
 102 files changed, 294 insertions(+), 305 deletions(-)
 mode change 100755 => 100644 testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log

diff --git a/NEWS b/NEWS
index 00aeb62132..7b60a05ccd 100644
--- a/NEWS
+++ b/NEWS
@@ -152,9 +152,9 @@ the full set.
   understands.
 
 - ASCII logs now record the time when they were opened/closed at the
-  beginning and end of the file, respectively. The options
-  LogAscii::header_prefix and LogAscii::include_header have been
-  renamed to LogAscii::meta_prefix and LogAscii::include_meta,
+  beginning and end of the file, respectively (wall clock). The
+  options LogAscii::header_prefix and LogAscii::include_header have
+  been renamed to LogAscii::meta_prefix and LogAscii::include_meta,
   respectively.
 
 - The ASCII writers "header_*" options have been renamed to "meta_*"
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index c471b3db0c..c4c6b06563 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -81,10 +81,7 @@ void Ascii::CloseFile(double t)
 		return;
 
 	if ( include_meta )
-		{
-		string ts = t ? Timestamp(t) : string("<abnormal termination>");
-		WriteHeaderField("end", ts);
-		}
+		WriteHeaderField("close", Timestamp(0));
 
 	safe_close(fd);
 	fd = 0;
@@ -124,8 +121,6 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 		if ( ! safe_write(fd, str.c_str(), str.length()) )
 			goto write_error;
 
-		string ts = Timestamp(info.network_time);
-
 		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
 		            string(set_separator, set_separator_len), false)) &&
 		        WriteHeaderField("empty_field", get_escaped_string(
@@ -133,7 +128,7 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 		        WriteHeaderField("unset_field", get_escaped_string(
 		            string(unset_field, unset_field_len), false)) &&
 		        WriteHeaderField("path", get_escaped_string(path, false)) &&
-		        WriteHeaderField("start", ts)) )
+		        WriteHeaderField("open", Timestamp(0))) )
 			goto write_error;
 
 		for ( int i = 0; i < num_fields; ++i )
@@ -419,6 +414,16 @@ string Ascii::Timestamp(double t)
 	{
 	time_t teatime = time_t(t);
 
+	if ( ! teatime )
+		{
+		// Use wall clock.
+		struct timeval tv;
+		if ( gettimeofday(&tv, 0) < 0 )
+			Error("gettimeofday failed");
+		else
+			teatime = tv.tv_sec;
+		}
+
 	struct tm tmbuf;
 	struct tm* tm = localtime_r(&teatime, &tmbuf);
 
diff --git a/src/logging/writers/Ascii.h b/src/logging/writers/Ascii.h
index cb82860cb7..cf0190aa80 100644
--- a/src/logging/writers/Ascii.h
+++ b/src/logging/writers/Ascii.h
@@ -35,7 +35,7 @@ private:
 	bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
 	bool WriteHeaderField(const string& key, const string& value);
 	void CloseFile(double t);
-	string Timestamp(double t);
+	string Timestamp(double t); // Uses current time if t is zero.
 
 	int fd;
 	string fname;
diff --git a/testing/btest/Baseline/core.checksums/bad.out b/testing/btest/Baseline/core.checksums/bad.out
index de4538e32b..94b141c9e1 100644
--- a/testing/btest/Baseline/core.checksums/bad.out
+++ b/testing/btest/Baseline/core.checksums/bad.out
@@ -3,101 +3,101 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-18-03-01
+#open	2012-03-26-18-03-01
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784981.078396	-	-	-	-	-	bad_IP_checksum	-	F	bro
-#end	2012-03-26-18-03-01
+#close	2012-03-26-18-03-01
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-18-01-25
+#open	2012-03-26-18-01-25
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784885.686428	UWkUyAuUGXf	127.0.0.1	30000	127.0.0.1	80	bad_TCP_checksum	-	F	bro
-#end	2012-03-26-18-01-25
+#close	2012-03-26-18-01-25
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-18-02-13
+#open	2012-03-26-18-02-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784933.501023	UWkUyAuUGXf	127.0.0.1	30000	127.0.0.1	13000	bad_UDP_checksum	-	F	bro
-#end	2012-03-26-18-02-13
+#close	2012-03-26-18-02-13
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-29-23
+#open	2012-04-10-16-29-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075363.536871	UWkUyAuUGXf	192.168.1.100	8	192.168.1.101	0	bad_ICMP_checksum	-	F	bro
-#end	2012-04-10-16-29-23
+#close	2012-04-10-16-29-23
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-18-06-50
+#open	2012-03-26-18-06-50
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785210.013051	-	-	-	-	-	routing0_hdr	-	F	bro
 1332785210.013051	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	80	bad_TCP_checksum	-	F	bro
-#end	2012-03-26-18-06-50
+#close	2012-03-26-18-06-50
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-17-23-00
+#open	2012-03-26-17-23-00
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332782580.798420	-	-	-	-	-	routing0_hdr	-	F	bro
 1332782580.798420	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	13000	bad_UDP_checksum	-	F	bro
-#end	2012-03-26-17-23-00
+#close	2012-03-26-17-23-00
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-25-11
+#open	2012-04-10-16-25-11
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075111.800086	-	-	-	-	-	routing0_hdr	-	F	bro
 1334075111.800086	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:78:1:32::1	129	bad_ICMP_checksum	-	F	bro
-#end	2012-04-10-16-25-11
+#close	2012-04-10-16-25-11
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-18-07-30
+#open	2012-03-26-18-07-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785250.469132	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	80	bad_TCP_checksum	-	F	bro
-#end	2012-03-26-18-07-30
+#close	2012-03-26-18-07-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-17-02-22
+#open	2012-03-26-17-02-22
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332781342.923813	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	13000	bad_UDP_checksum	-	F	bro
-#end	2012-03-26-17-02-22
+#close	2012-03-26-17-02-22
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-22-19
+#open	2012-04-10-16-22-19
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
-#end	2012-04-10-16-22-19
+#close	2012-04-10-16-22-19
diff --git a/testing/btest/Baseline/core.checksums/good.out b/testing/btest/Baseline/core.checksums/good.out
index ed6c071ffc..a47931a15c 100644
--- a/testing/btest/Baseline/core.checksums/good.out
+++ b/testing/btest/Baseline/core.checksums/good.out
@@ -3,68 +3,68 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-22-19
+#open	2012-04-10-16-22-19
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
-#end	2012-04-10-16-22-19
+#close	2012-04-10-16-22-19
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-18-05-25
+#open	2012-03-26-18-05-25
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785125.596793	-	-	-	-	-	routing0_hdr	-	F	bro
-#end	2012-03-26-18-05-25
+#close	2012-03-26-18-05-25
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-03-26-17-21-48
+#open	2012-03-26-17-21-48
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332782508.592037	-	-	-	-	-	routing0_hdr	-	F	bro
-#end	2012-03-26-17-21-48
+#close	2012-03-26-17-21-48
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-23-47
+#open	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
-#end	2012-04-10-16-23-47
+#close	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-23-47
+#open	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
-#end	2012-04-10-16-23-47
+#close	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-23-47
+#open	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
-#end	2012-04-10-16-23-47
+#close	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-16-23-47
+#open	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
-#end	2012-04-10-16-23-47
+#close	2012-04-10-16-23-47
diff --git a/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log b/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
index d29456f75f..9da1a8d3ba 100644
--- a/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
+++ b/testing/btest/Baseline/core.disable-mobile-ipv6/weird.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-05-21-56-51
+#open	2012-04-05-21-56-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1333663011.602839	-	-	-	-	-	unknown_protocol_135	-	F	bro
-#end	2012-04-05-21-56-51
+#close	2012-04-05-21-56-51
diff --git a/testing/btest/Baseline/core.expr-exception/reporter.log b/testing/btest/Baseline/core.expr-exception/reporter.log
index f9e33d9718..d6e07b42b3 100644
--- a/testing/btest/Baseline/core.expr-exception/reporter.log
+++ b/testing/btest/Baseline/core.expr-exception/reporter.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	ts	level	message	location
 #types	time	enum	string	string
 1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
@@ -15,4 +15,4 @@
 1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
 1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
 1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/core.ipv6-frag/dns.log b/testing/btest/Baseline/core.ipv6-frag/dns.log
index 2003d1f253..d763fc4fee 100644
--- a/testing/btest/Baseline/core.ipv6-frag/dns.log
+++ b/testing/btest/Baseline/core.ipv6-frag/dns.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
-#start	2012-03-07-01-37-58
+#open	2012-03-07-01-37-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs
 #types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]
 1331084278.438444	UWkUyAuUGXf	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51850	2607:f740:b::f93	53	udp	3903	txtpadding_323.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
 1331084293.592245	arKYeMETxOg	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51851	2607:f740:b::f93	53	udp	40849	txtpadding_3230.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
-#end	2012-03-07-01-38-18
+#close	2012-03-07-01-38-18
diff --git a/testing/btest/Baseline/core.print-bpf-filters/conn.log b/testing/btest/Baseline/core.print-bpf-filters/conn.log
index 4033b64e2a..0fd86b8dc4 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/conn.log
+++ b/testing/btest/Baseline/core.print-bpf-filters/conn.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2005-10-07-23-23-57
+#open	2005-10-07-23-23-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
-#end	2005-10-07-23-23-57
+#close	2005-10-07-23-23-57
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output
index e4bc04192a..c55952ffed 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@@ -3,38 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#start	1970-01-01-00-00-00
+#open	2012-07-27-19-14-29
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1342748953.570646	-	ip or not ip	T	T
-#end	<abnormal termination>
+1343416469.508262	-	ip or not ip	T	T
+#close	2012-07-27-19-14-29
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#start	1970-01-01-00-00-00
+#open	2012-07-27-19-14-29
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1342748953.898675	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
-#end	<abnormal termination>
+1343416469.888870	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+#close	2012-07-27-19-14-29
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#start	1970-01-01-00-00-00
+#open	2012-07-27-19-14-30
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1342748954.278211	-	port 42	T	T
-#end	<abnormal termination>
+1343416470.252918	-	port 42	T	T
+#close	2012-07-27-19-14-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#start	1970-01-01-00-00-00
+#open	2012-07-27-19-14-30
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1342748954.883780	-	port 56730	T	T
-#end	2005-10-07-23-23-57
+1343416470.614962	-	port 56730	T	T
+#close	2012-07-27-19-14-30
diff --git a/testing/btest/Baseline/core.truncation/output b/testing/btest/Baseline/core.truncation/output
index 836f9170d4..9243c2f873 100644
--- a/testing/btest/Baseline/core.truncation/output
+++ b/testing/btest/Baseline/core.truncation/output
@@ -3,38 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-11-16-01-35
+#open	2012-04-11-16-01-35
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334160095.895421	-	-	-	-	-	truncated_IP	-	F	bro
-#end	2012-04-11-16-01-35
+#close	2012-04-11-16-01-35
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-11-14-57-21
+#open	2012-04-11-14-57-21
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334156241.519125	-	-	-	-	-	truncated_IP	-	F	bro
-#end	2012-04-11-14-57-21
+#close	2012-04-11-14-57-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-04-10-21-50-48
+#open	2012-04-10-21-50-48
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334094648.590126	-	-	-	-	-	truncated_IP	-	F	bro
-#end	2012-04-10-21-50-48
+#close	2012-04-10-21-50-48
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-05-29-22-02-34
+#open	2012-05-29-22-02-34
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1338328954.078361	-	-	-	-	-	internally_truncated_header	-	F	bro
-#end	2012-05-29-22-02-34
+#close	2012-05-29-22-02-34
diff --git a/testing/btest/Baseline/core.tunnels.ayiya/conn.log b/testing/btest/Baseline/core.tunnels.ayiya/conn.log
index 82a3828f0d..7646fa574a 100644
--- a/testing/btest/Baseline/core.tunnels.ayiya/conn.log
+++ b/testing/btest/Baseline/core.tunnels.ayiya/conn.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2009-11-08-04-41-57
+#open	2009-11-08-04-41-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1257655301.595604	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	tcp	http	2.101052	2981	4665	S1	-	0	ShADad	10	3605	11	5329	k6kgXLOoSKl
@@ -14,4 +14,4 @@
 1257655296.585188	TEfuqmmG4bh	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff00:2	130	icmp	-	0.919988	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
 1257655296.585151	j4u32Pc5bif	fe80::216:cbff:fe9a:4cb9	131	ff02::2:f901:d225	130	icmp	-	0.719947	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
 1257655296.585034	nQcgTWjvg4c	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff9a:4cb9	130	icmp	-	4.922880	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
-#end	2009-11-08-04-41-57
+#close	2009-11-08-04-41-57
diff --git a/testing/btest/Baseline/core.tunnels.ayiya/http.log b/testing/btest/Baseline/core.tunnels.ayiya/http.log
index 4fbcd508f4..2a97fd9b69 100644
--- a/testing/btest/Baseline/core.tunnels.ayiya/http.log
+++ b/testing/btest/Baseline/core.tunnels.ayiya/http.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2009-11-08-04-41-41
+#open	2009-11-08-04-41-41
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1257655301.652206	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	10102	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1257655302.514424	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	2	GET	ipv6.google.com	/csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
 1257655303.603569	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	3	GET	ipv6.google.com	/gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
-#end	2009-11-08-04-41-57
+#close	2009-11-08-04-41-57
diff --git a/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log b/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
index 123ea8a792..60e0a4a108 100644
--- a/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.ayiya/tunnel.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
-#start	2009-11-08-04-41-33
+#open	2009-11-08-04-41-33
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1257655293.629048	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::DISCOVER
 1257655296.585034	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::DISCOVER
 1257655317.464035	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::CLOSE
 1257655317.464035	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::CLOSE
-#end	2009-11-08-04-41-57
+#close	2009-11-08-04-41-57
diff --git a/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log b/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
index 63a0437445..3300a3ef95 100644
--- a/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
+++ b/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dpd
-#start	2009-11-18-17-59-51
+#open	2009-11-18-17-59-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	analyzer	failure_reason
 #types	time	string	addr	port	addr	port	enum	string	string
 1258567191.486869	UWkUyAuUGXf	192.168.1.105	57696	192.168.1.1	53	udp	TEREDO	Teredo payload length [c\x1d\x81\x80\x00\x01\x00\x02\x00\x02\x00\x00\x04amch\x0equestionmarket\x03com\x00\x00\x01\x00...]
@@ -12,4 +12,4 @@
 1258581768.898165	TEfuqmmG4bh	192.168.1.104	50798	192.168.1.1	53	udp	TEREDO	Teredo payload length [o\xe3\x81\x80\x00\x01\x00\x02\x00\x04\x00\x04\x03www\x0fnashuatelegraph\x03com\x00\x00\x01\x00...]
 1258584478.989528	FrJExwHcSal	192.168.1.104	64963	192.168.1.1	53	udp	TEREDO	Teredo payload length [e\xbd\x81\x80\x00\x01\x00\x08\x00\x06\x00\x06\x08wellness\x05blogs\x04time\x03com\x00\x00\x01\x00...]
 1258600683.934672	5OKnoww6xl4	192.168.1.103	59838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xf0\x81\x80\x00\x01\x00\x01\x00\x02\x00\x00\x06update\x0csanasecurity\x03com\x00\x00\x01\x00...]
-#end	2009-11-19-03-18-03
+#close	2009-11-19-03-18-03
diff --git a/testing/btest/Baseline/core.tunnels.false-teredo/weird.log b/testing/btest/Baseline/core.tunnels.false-teredo/weird.log
index eb4319c7eb..a84d469660 100644
--- a/testing/btest/Baseline/core.tunnels.false-teredo/weird.log
+++ b/testing/btest/Baseline/core.tunnels.false-teredo/weird.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2009-11-18-17-59-51
+#open	2009-11-18-17-59-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1258567191.405770	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
@@ -12,4 +12,4 @@
 1258581768.568451	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
 1258584478.859853	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
 1258600683.934458	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
-#end	2009-11-19-03-18-03
+#close	2009-11-19-03-18-03
diff --git a/testing/btest/Baseline/core.tunnels.teredo/conn.log b/testing/btest/Baseline/core.tunnels.teredo/conn.log
index 2342953339..657e86b8b3 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/conn.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2008-05-16-15-50-57
+#open	2008-05-16-15-50-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1210953047.736921	arKYeMETxOg	192.168.2.16	1576	75.126.130.163	80	tcp	-	0.000357	0	0	SHR	-	0	fA	1	40	1	40	(empty)
@@ -27,4 +27,4 @@
 1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
 1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c
 1210953052.202579	j4u32Pc5bif	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	nQcgTWjvg4c
-#end	2008-05-16-15-51-16
+#close	2008-05-16-15-51-16
diff --git a/testing/btest/Baseline/core.tunnels.teredo/http.log b/testing/btest/Baseline/core.tunnels.teredo/http.log
index c0db5fc146..c77297c58d 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/http.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/http.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2008-05-16-15-50-58
+#open	2008-05-16-15-50-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1210953057.917183	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	1	POST	download913.avast.com	/cgi-bin/iavs4stats.cgi	-	Syncer/4.80 (av_pro-1169;f)	589	0	204	<empty>	-	-	-	(empty)	-	-	-	text/plain	-	-
 1210953061.585996	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1210953073.381474	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1210953074.674817	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	1	GET	www.wireshark.org	/	http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	11845	200	OK	-	-	-	(empty)	-	-	-	text/xml	-	-
-#end	2008-05-16-15-51-16
+#close	2008-05-16-15-51-16
diff --git a/testing/btest/Baseline/core.tunnels.teredo/tunnel.log b/testing/btest/Baseline/core.tunnels.teredo/tunnel.log
index ab14bf68bc..120089caa0 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/tunnel.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
-#start	2008-05-16-15-50-52
+#open	2008-05-16-15-50-52
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::DISCOVER
@@ -12,4 +12,4 @@
 1210953076.058333	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::CLOSE
 1210953076.058333	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	Tunnel::TEREDO	Tunnel::CLOSE
 1210953076.058333	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	Tunnel::TEREDO	Tunnel::CLOSE
-#end	2008-05-16-15-51-16
+#close	2008-05-16-15-51-16
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
index 7b9ff58624..757eaf62ca 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2012-06-19-17-39-37
+#open	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1340127577.354166	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	tcp	http	0.052829	1675	10467	S1	-	0	ShADad	10	2279	12	11191	j4u32Pc5bif
@@ -13,4 +13,4 @@
 1340127577.339015	nQcgTWjvg4c	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	k6kgXLOoSKl
 1340127577.343969	TEfuqmmG4bh	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.007778	4	4	OTH	-	0	-	1	52	1	52	UWkUyAuUGXf,j4u32Pc5bif
 1340127577.336558	arKYeMETxOg	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	UWkUyAuUGXf
-#end	2012-06-19-17-39-37
+#close	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
index 12f0d7be7a..e0b223d114 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2012-06-19-17-39-37
+#open	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1340127577.361683	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1340127577.379360	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-#end	2012-06-19-17-39-37
+#close	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
index 1a14b3edb7..86c2c94c04 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
-#start	2012-06-19-17-39-37
+#open	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340127577.336558	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::DISCOVER
@@ -12,4 +12,4 @@
 1340127577.406995	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::CLOSE
 1340127577.406995	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Tunnel::TEREDO	Tunnel::CLOSE
 1340127577.406995	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	Tunnel::TEREDO	Tunnel::CLOSE
-#end	2012-06-19-17-39-37
+#close	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
index 8b252a5819..4ead29302f 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#start	2012-06-19-17-39-37
+#open	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1340127577.346849	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Teredo_bubble_with_payload	-	F	bro
 1340127577.349292	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
-#end	2012-06-19-17-39-37
+#close	2012-06-19-17-39-37
diff --git a/testing/btest/Baseline/core.vlan-mpls/conn.log b/testing/btest/Baseline/core.vlan-mpls/conn.log
index 72e13ee9b4..d4cc8370a5 100644
--- a/testing/btest/Baseline/core.vlan-mpls/conn.log
+++ b/testing/btest/Baseline/core.vlan-mpls/conn.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2005-10-07-23-23-55
+#open	2005-10-07-23-23-55
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.102560	26	0	SH	-	0	SADF	11	470	0	0	(empty)
 1128727435.450898	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
 1278600802.069419	k6kgXLOoSKl	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
-#end	2010-07-08-14-53-22
+#close	2010-07-08-14-53-22
diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
index ca8749956f..41209a4084 100644
--- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#start	2012-07-20-14-34-11
+#open	2012-07-20-14-34-11
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -30,4 +30,4 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/./readers/raw.bro
     scripts/base/frameworks/input/./readers/benchmark.bro
 scripts/policy/misc/loaded-scripts.bro
-#end	2012-07-20-14-34-11
+#close	2012-07-20-14-34-11
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index b464c916f2..b2afadc0fe 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#start	2012-07-20-14-34-40
+#open	2012-07-20-14-34-40
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -110,4 +110,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/./consts.bro
     scripts/base/protocols/syslog/./main.bro
 scripts/policy/misc/loaded-scripts.bro
-#end	2012-07-20-14-34-40
+#close	2012-07-20-14-34-40
diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log
index c9a996ef5b..3fc7f1b66f 100644
--- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log
+++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2012-07-20-01-53-03
+#open	2012-07-20-01-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1342749182.906082	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-#end	2012-07-20-01-53-04
+#close	2012-07-20-01-53-04
diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log
index c9a996ef5b..3fc7f1b66f 100644
--- a/testing/btest/Baseline/istate.events-ssl/sender.http.log
+++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2012-07-20-01-53-03
+#open	2012-07-20-01-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1342749182.906082	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-#end	2012-07-20-01-53-04
+#close	2012-07-20-01-53-04
diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log
index 566457b996..6862c08b98 100644
--- a/testing/btest/Baseline/istate.events/receiver.http.log
+++ b/testing/btest/Baseline/istate.events/receiver.http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2012-07-20-01-53-12
+#open	2012-07-20-01-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1342749191.765740	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-#end	2012-07-20-01-53-13
+#close	2012-07-20-01-53-13
diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log
index 566457b996..6862c08b98 100644
--- a/testing/btest/Baseline/istate.events/sender.http.log
+++ b/testing/btest/Baseline/istate.events/sender.http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2012-07-20-01-53-12
+#open	2012-07-20-01-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1342749191.765740	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
-#end	2012-07-20-01-53-13
+#close	2012-07-20-01-53-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
index 7e21ff86b7..c6a19029b6 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	communication
-#start	2012-07-20-01-49-40
+#open	2012-07-20-01-49-40
 #fields	ts	peer	src_name	connected_peer_desc	connected_peer_addr	connected_peer_port	level	message
 #types	time	string	string	string	addr	port	string	string
 1342748980.737451	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
@@ -21,4 +21,4 @@
 1342748980.793108	bro	parent	-	-	-	info	terminating...
 1342748980.796454	bro	child	-	-	-	info	terminating
 1342748980.797536	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
-#end	2012-07-20-01-49-40
+#close	2012-07-20-01-49-40
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
index a0359c2d70..655d9a5fbd 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh-new-default
-#start	2012-07-20-01-49-19
+#open	2012-07-20-01-49-19
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748959.430282	1.2.3.4	1234	2.3.4.5	80	success	unknown
 1342748959.430282	1.2.3.4	1234	2.3.4.5	80	failure	US
-#end	2012-07-20-01-49-19
+#close	2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
index 0c826f9694..b2528467a1 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log
@@ -3,10 +3,10 @@
 #empty_field|(empty)
 #unset_field|-
 #path|ssh
-#start|2012-07-20-01-49-19
+#open|2012-07-20-01-49-19
 #fields|data|data2
 #types|string|string
 abc\x0a\xffdef|DATA2
 abc\x7c\xffdef|DATA2
 abc\xff\x7cdef|DATA2
-#end|2012-07-20-01-49-19
+#close|2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
index b1a4ba52e2..b77541d35e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2012-07-20-01-49-19
+#open	2012-07-20-01-49-19
 #fields	x	y	z
 #types	string	string	string
 \x2d	-	(empty)
-#end	2012-07-20-01-49-19
+#close	2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
index 683f149317..f1ff4db3b8 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2011-09-12-03-57-36
+#open	2011-09-12-03-57-36
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1315799856.264750	UWkUyAuUGXf	10.0.1.104	64216	193.40.5.162	80	1	GET	lepo.it.da.ut.ee	/~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf	-	Wget/1.12 (darwin10.8.0)	0	346	404	Not Found	-	-	-	(empty)	-	-	-	text/html	-	-
-#end	2011-09-12-03-57-37
+#close	2011-09-12-03-57-37
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
index a03c6f954b..25e9319eec 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2012-07-20-01-49-19
+#open	2012-07-20-01-49-19
 #fields	ss
 #types	table[string]
 CC,AA,\x2c,\x2c\x2c
-#end	2012-07-20-01-49-19
+#close	2012-07-20-01-49-19
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
index 0c6a266de0..7a448ce6c1 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
@@ -3,10 +3,12 @@
 #empty_field||(empty)
 #unset_field||-
 #path||ssh
+#open||2012-07-27-19-14-35
 #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country
 #types||time||addr||port||addr||port||string||string
-1342759749.586006||1.2.3.4||1234||2.3.4.5||80||success||unknown
-1342759749.586006||1.2.3.4||1234||2.3.4.5||80||failure||US
-1342759749.586006||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
-1342759749.586006||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
-1342759749.586006||1.2.3.4||1234||2.3.4.5||80||failure||MX
+1343416475.837726||1.2.3.4||1234||2.3.4.5||80||success||unknown
+1343416475.837726||1.2.3.4||1234||2.3.4.5||80||failure||US
+1343416475.837726||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
+1343416475.837726||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
+1343416475.837726||1.2.3.4||1234||2.3.4.5||80||failure||MX
+#close||2012-07-27-19-14-35
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
index 21b81abf95..0f825462ab 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-line-like-comment/test.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2012-07-20-01-49-22
+#open	2012-07-20-01-49-22
 #fields	data	c
 #types	string	count
 Test1	42
 \x23Kaputt	42
 Test2	42
-#end	2012-07-20-01-49-22
+#close	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
index 5fba268afa..c644dab007 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	data
 #types	time
 1234567890.000000
@@ -14,4 +14,4 @@
 1234567890.000010
 1234567890.000001
 1234567890.000000
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
index 7d3bbc0774..9eb2f0e663 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	status	country	a1	b1	b2
 #types	string	string	count	count	count
 success	unknown	1	3	4
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
index c3163dba6f..bcedd1174e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	status	country
 #types	string	string
 success	unknown
@@ -11,4 +11,4 @@ failure	US
 failure	UK
 success	BR
 failure	MX
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
index 42f945bf0c..b255ac3489 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748960.468458	1.2.3.4	1234	2.3.4.5	80	success	unknown
@@ -11,4 +11,4 @@
 1342748960.468458	1.2.3.4	1234	2.3.4.5	80	failure	UK
 1342748960.468458	1.2.3.4	1234	2.3.4.5	80	success	BR
 1342748960.468458	1.2.3.4	1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
index 3fe01ff913..f795159a16 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	port	addr	port	string	string
 1234	2.3.4.5	80	success	unknown
@@ -11,4 +11,4 @@
 1234	2.3.4.5	80	failure	UK
 1234	2.3.4.5	80	success	BR
 1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
index 205f37243f..34d5f28b82 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	t	f
 #types	time	file
 1342748960.757056	Foo.log
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
index cafacf9c4e..8935046687 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-20
+#open	2012-07-20-01-49-20
 #fields	t	id.orig_h
 #types	time	addr
 1342748960.796093	1.2.3.4
@@ -11,4 +11,4 @@
 1342748960.796093	1.2.3.4
 1342748960.796093	1.2.3.4
 1342748960.796093	1.2.3.4
-#end	2012-07-20-01-49-20
+#close	2012-07-20-01-49-20
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
index 3240e9f824..819b7b9bc2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	local
-#start	2011-03-18-19-06-13
+#open	2011-03-18-19-06-13
 #fields	ts	id.orig_h
 #types	time	addr
 1300475168.859163	141.142.220.118
@@ -36,4 +36,4 @@
 1300475168.902195	141.142.220.118
 1300475168.894787	141.142.220.118
 1300475168.901749	141.142.220.118
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
index 84980836c4..41f575ef63 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	remote
-#start	2011-03-18-19-06-13
+#open	2011-03-18-19-06-13
 #fields	ts	id.orig_h
 #types	time	addr
 1300475169.780331	173.192.163.128
 1300475167.097012	fe80::217:f2ff:fed7:cf65
 1300475171.675372	fe80::3074:17d5:2052:c324
 1300475173.116749	fe80::3074:17d5:2052:c324
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
index 1c67ff52b6..c67a12e1d9 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output
@@ -10,68 +10,68 @@ static-prefix-2-UK.log
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-0-BR
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	success	BR
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-0-MX3
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	MX3
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-0-unknown
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	success	unknown
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-1-MX
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-1-US
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	US
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-2-MX2
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	MX2
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	static-prefix-2-UK
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.180156	1.2.3.4	1234	2.3.4.5	80	failure	UK
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
index 96dede8965..a362135318 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.488370	1.2.3.4	1234	2.3.4.5	80	failure	US
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
index 85b5ca9f45..dd9c300429 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.488370	1.2.3.4	1234	2.3.4.5	80	success	unknown
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
index aa18822daf..13364f8e77 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log
@@ -3,8 +3,8 @@
 #empty_field	EMPTY
 #unset_field	-
 #path	test
-#start	1970-01-01-00-00-00
+#open	1970-01-01-00-00-00
 #fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve
 #types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table[count]	table[string]	table[string]	vector[count]	vector[string]
 T	-42	Test::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1342749004.579242	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY
-#end	2012-07-20-01-50-05
+#close	2012-07-20-01-50-05
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
index 36b88e496d..71e1d18c73 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
-#start	2012-07-20-01-50-18
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-50-18
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
index 22d354fce4..bc3dac5a1a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2012-07-20-01-50-18
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
@@ -11,4 +11,4 @@
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-50-18
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
index 888dc424b5..f0b26454b4 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
-#start	2012-07-20-01-50-18
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
 1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
-#end	2012-07-20-01-50-18
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
index 5a23ad2066..de324c337f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh.failure
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	US
 1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	UK
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
index cea1069748..ed0a118cac 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	US
 1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	UK
 1342748961.521536	1.2.3.4	1234	2.3.4.5	80	failure	BR
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
index 19354f8df2..3acce6f1ce 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out
@@ -28,32 +28,14 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, pat
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
+#close	2012-07-27-19-14-39
 #empty_field	(empty)
-#end	2011-03-07-03-59-55
-#end	2011-03-07-04-00-05
-#end	2011-03-07-04-59-55
-#end	2011-03-07-05-00-05
-#end	2011-03-07-05-59-55
-#end	2011-03-07-06-00-05
-#end	2011-03-07-06-59-55
-#end	2011-03-07-07-00-05
-#end	2011-03-07-07-59-55
-#end	2011-03-07-08-00-05
-#end	2011-03-07-08-59-55
-#end	2011-03-07-09-00-05
-#end	2011-03-07-09-59-55
-#end	2011-03-07-10-00-05
-#end	2011-03-07-10-59-55
-#end	2011-03-07-11-00-05
-#end	2011-03-07-11-59-55
-#end	2011-03-07-12-00-05
-#end	2011-03-07-12-59-55
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+#open	2012-07-27-19-14-39
 #path	test
 #path	test2
 #separator \x09
 #set_separator	,
-#start	2011-03-07-03-00-05
 #types	time	addr	port	addr	port
 #unset_field	-
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
index 4764ff23d0..b26d2fcd1b 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out
@@ -14,117 +14,117 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1 ascii
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
 1299470395.000000	10.0.0.2	20	10.0.0.3	0
-#end	2011-03-07-04-00-05
+#close	2011-03-07-04-00-05
 > test.2011-03-07-04-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299470405.000000	10.0.0.1	20	10.0.0.2	1025
 1299473995.000000	10.0.0.2	20	10.0.0.3	1
-#end	2011-03-07-05-00-05
+#close	2011-03-07-05-00-05
 > test.2011-03-07-05-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299474005.000000	10.0.0.1	20	10.0.0.2	1026
 1299477595.000000	10.0.0.2	20	10.0.0.3	2
-#end	2011-03-07-06-00-05
+#close	2011-03-07-06-00-05
 > test.2011-03-07-06-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299477605.000000	10.0.0.1	20	10.0.0.2	1027
 1299481195.000000	10.0.0.2	20	10.0.0.3	3
-#end	2011-03-07-07-00-05
+#close	2011-03-07-07-00-05
 > test.2011-03-07-07-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299481205.000000	10.0.0.1	20	10.0.0.2	1028
 1299484795.000000	10.0.0.2	20	10.0.0.3	4
-#end	2011-03-07-08-00-05
+#close	2011-03-07-08-00-05
 > test.2011-03-07-08-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299484805.000000	10.0.0.1	20	10.0.0.2	1029
 1299488395.000000	10.0.0.2	20	10.0.0.3	5
-#end	2011-03-07-09-00-05
+#close	2011-03-07-09-00-05
 > test.2011-03-07-09-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299488405.000000	10.0.0.1	20	10.0.0.2	1030
 1299491995.000000	10.0.0.2	20	10.0.0.3	6
-#end	2011-03-07-10-00-05
+#close	2011-03-07-10-00-05
 > test.2011-03-07-10-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299492005.000000	10.0.0.1	20	10.0.0.2	1031
 1299495595.000000	10.0.0.2	20	10.0.0.3	7
-#end	2011-03-07-11-00-05
+#close	2011-03-07-11-00-05
 > test.2011-03-07-11-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299495605.000000	10.0.0.1	20	10.0.0.2	1032
 1299499195.000000	10.0.0.2	20	10.0.0.3	8
-#end	2011-03-07-12-00-05
+#close	2011-03-07-12-00-05
 > test.2011-03-07-12-00-05.log
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	test
-#start	2011-03-07-03-00-05
+#open	2011-03-07-03-00-05
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
 1299499205.000000	10.0.0.1	20	10.0.0.2	1033
 1299502795.000000	10.0.0.2	20	10.0.0.3	9
-#end	2011-03-07-12-59-55
+#close	2011-03-07-12-59-55
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
index 110cef054a..6ff5237afa 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	/dev/stdout
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.732599	1.2.3.4	1234	2.3.4.5	80	success	unknown
@@ -11,4 +11,4 @@
 1342748961.732599	1.2.3.4	1234	2.3.4.5	80	failure	UK
 1342748961.732599	1.2.3.4	1234	2.3.4.5	80	success	BR
 1342748961.732599	1.2.3.4	1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
index c9191b666e..d2d484e02f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-21
+#open	2012-07-20-01-49-21
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
 1342748961.748481	1.2.3.4	1234	2.3.4.5	80	success	unknown
@@ -11,4 +11,4 @@
 1342748961.748481	1.2.3.4	1234	2.3.4.5	80	failure	UK
 1342748961.748481	1.2.3.4	1234	2.3.4.5	80	success	BR
 1342748961.748481	1.2.3.4	1234	2.3.4.5	80	failure	MX
-#end	2012-07-20-01-49-21
+#close	2012-07-20-01-49-21
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
index 1fc29dbb4e..6b75d056cf 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log
@@ -3,8 +3,8 @@
 #empty_field	EMPTY
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-22
+#open	2012-07-20-01-49-22
 #fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
 #types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table[count]	table[string]	table[string]	vector[count]	vector[string]	func
 T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1342748962.114672	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
-#end	2012-07-20-01-49-22
+#close	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
index b4089aeee8..0ebe8838ad 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	testing
-#start	2012-07-20-01-49-22
+#open	2012-07-20-01-49-22
 #fields	a.val1	a.val2	b
 #types	count	count	count
 -	-	6
 1	2	3
-#end	2012-07-20-01-49-22
+#close	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
index ae5d6d246e..3e8e1e737e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh
-#start	2012-07-20-01-49-22
+#open	2012-07-20-01-49-22
 #fields	vec
 #types	vector[string]
 -,2,-,-,5
-#end	2012-07-20-01-49-22
+#close	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log
index 1e41aca795..cbc90d9926 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2-2.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http-2-2
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	status_code
 #types	count
 304
@@ -20,4 +20,4 @@
 304
 304
 304
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log
index 4d3622c7a0..8f66184146 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-2.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http-2
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	host
 #types	string
 bits.wikimedia.org
@@ -20,4 +20,4 @@ upload.wikimedia.org
 upload.wikimedia.org
 upload.wikimedia.org
 upload.wikimedia.org
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log
index 727a6c02fa..d64b9aa128 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http-3.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http-3
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	uri
 #types	string
 /skins-1.5/monobook/main.css
@@ -20,4 +20,4 @@
 /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png
 /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png
 /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log
index 9ac9b6304c..97273995bc 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/http.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1300475168.784020	j4u32Pc5bif	141.142.220.118	48649	208.80.152.118	80	1	GET	bits.wikimedia.org	/skins-1.5/monobook/main.css	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
@@ -20,4 +20,4 @@
 1300475169.014619	Tw8jXtpTGu6	141.142.220.118	50000	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
 1300475169.014593	P654jzLoe3a	141.142.220.118	49999	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
 1300475169.014927	0Q4FH8sESw5	141.142.220.118	50001	208.80.152.3	80	2	GET	upload.wikimedia.org	/wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png	http://www.wikipedia.org/	Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15	0	0	304	Not Modified	-	-	-	(empty)	-	-	-	-	-	-
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
old mode 100755
new mode 100644
index 3514ca5134..35e9134583
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.writer-path-conflict/reporter.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	ts	level	message	location
 #types	time	enum	string	string
 1300475168.843894	Reporter::WARNING	Write using filter 'host-only' on path 'http' changed to use new path 'http-2' to avoid conflict with filter 'default'	(empty)
 1300475168.843894	Reporter::WARNING	Write using filter 'uri-only' on path 'http' changed to use new path 'http-3' to avoid conflict with filter 'default'	(empty)
 1300475168.843894	Reporter::WARNING	Write using filter 'status-only' on path 'http-2' changed to use new path 'http-2-2' to avoid conflict with filter 'host-only'	(empty)
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
index a3f476c1fb..cb1bd5af01 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#start	2012-07-20-01-50-41
+#open	2012-07-20-01-50-41
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
 1342749041.601712	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
 1342749041.601712	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
 1342749041.601712	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
-#end	2012-07-20-01-50-49
+#close	2012-07-20-01-50-49
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
index b497da5194..fb6476ee88 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
-#start	2012-07-20-01-49-22
+#open	2012-07-20-01-49-22
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
 1342748962.841548	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
 1342748962.841548	TEST_METRIC	foo-bar	7.2.1.5	-	-	1
 1342748962.841548	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
-#end	2012-07-20-01-49-22
+#close	2012-07-20-01-49-22
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
index 8f3a9dc70c..217b3ed49b 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#start	2012-07-20-01-50-59
+#open	2012-07-20-01-50-59
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
 1342749059.978651	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
-#end	2012-07-20-01-51-08
+#close	2012-07-20-01-51-08
diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
index 5a214b26cc..ba6c680e27 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#start	2012-07-20-01-49-23
+#open	2012-07-20-01-49-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
 1342748963.085888	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	1.2.3.4	-	-
 1342748963.085888	-	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	6.5.4.3	-	-
-#end	2012-07-20-01-49-23
+#close	2012-07-20-01-49-23
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
index 4903ec0c01..6c93cb875e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#start	2012-07-20-01-51-18
+#open	2012-07-20-01-51-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
 1342749078.270791	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
-#end	2012-07-20-01-51-27
+#close	2012-07-20-01-51-27
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
index bd77a90c86..88f25b066f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#start	2012-07-20-01-51-36
+#open	2012-07-20-01-51-36
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
 1342749096.545663	-	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
-#end	2012-07-20-01-51-45
+#close	2012-07-20-01-51-45
diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
index 5a3cdfa69f..7c7254f87e 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#start	2012-07-20-01-49-23
+#open	2012-07-20-01-49-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double
 1342748963.685754	-	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-
-#end	2012-07-20-01-49-23
+#close	2012-07-20-01-49-23
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log
index 316056fa8c..3520980833 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/conn.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2012-02-21-16-53-13
+#open	2012-02-21-16-53-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562	(empty)
@@ -11,4 +11,4 @@
 1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164	(empty)
 1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164	(empty)
 1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458	(empty)
-#end	2012-02-21-16-53-20
+#close	2012-02-21-16-53-20
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log
index cee57182ed..0d0a8f57f1 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv4/ftp.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ftp
-#start	2012-02-21-16-53-13
+#open	2012-02-21-16-53-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	user	password	command	arg	mime_type	mime_desc	file_size	reply_code	reply_msg	tags	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	count	string	table[string]	file
 1329843179.926563	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	ASCII text	77	226	Transfer complete.	-	-
 1329843197.727769	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	anonymous	test	RETR	ftp://199.233.217.249/./robots.txt	text/plain	ASCII text, with CRLF line terminators	77	226	Transfer complete.	-	-
-#end	2012-02-21-16-53-20
+#close	2012-02-21-16-53-20
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log
index 299bdbc4ba..3d81f45670 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/conn.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
-#start	2012-02-15-17-43-15
+#open	2012-02-15-17-43-15
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1329327783.316897	arKYeMETxOg	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49186	2001:470:4867:99::21	57086	tcp	ftp-data	0.219721	0	342	SF	-	0	ShAdfFa	5	372	4	642	(empty)
@@ -12,4 +12,4 @@
 1329327795.571921	j4u32Pc5bif	2001:470:4867:99::21	55785	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49189	tcp	ftp-data	0.109813	77	0	SF	-	0	ShADFaf	5	449	4	300	(empty)
 1329327777.822004	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	tcp	ftp	26.658219	310	3448	SF	-	0	ShAdDfFa	57	4426	34	5908	(empty)
 1329327800.017649	TEfuqmmG4bh	2001:470:4867:99::21	55647	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49190	tcp	ftp-data	0.109181	342	0	SF	-	0	ShADFaf	5	714	4	300	(empty)
-#end	2012-02-15-17-43-24
+#close	2012-02-15-17-43-24
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log
index 096b91df65..62ea4df18d 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.ftp-ipv6/ftp.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ftp
-#start	2012-02-15-17-43-07
+#open	2012-02-15-17-43-07
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	user	password	command	arg	mime_type	mime_desc	file_size	reply_code	reply_msg	tags	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	count	string	table[string]	file
 1329327787.396984	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	anonymous	test	RETR	ftp://[2001:470:4867:99::21]/robots.txt	-	-	77	226	Transfer complete.	-	-
 1329327795.463946	UWkUyAuUGXf	2001:470:1f11:81f:c999:d94:aa7c:2e3e	49185	2001:470:4867:99::21	21	anonymous	test	RETR	ftp://[2001:470:4867:99::21]/robots.txt	-	-	77	226	Transfer complete.	-	-
-#end	2012-02-15-17-43-24
+#close	2012-02-15-17-43-24
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
index c457f9b64b..13c8b12502 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2009-03-19-05-21-36
+#open	2009-03-19-05-21-36
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1237440095.634312	UWkUyAuUGXf	192.168.3.103	54102	128.146.216.51	80	1	POST	www.osu.edu	/	-	curl/7.17.1 (i386-apple-darwin8.11.1) libcurl/7.17.1 zlib/1.2.3	2001	60731	200	OK	100	Continue	-	(empty)	-	-	-	text/html	-	-
-#end	2009-03-19-05-21-36
+#close	2009-03-19-05-21-36
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
index 46ae431fc2..0d61a6c8b3 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2005-10-07-23-23-56
+#open	2005-10-07-23-23-56
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1128727435.634189	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	http-item_141.42.64.125:56730-125.190.109.199:80_resp_1.dat
-#end	2005-10-07-23-23-57
+#close	2005-10-07-23-23-57
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
index 69e6613a3c..409d8fc812 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2009-11-18-20-58-04
+#open	2009-11-18-20-58-04
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	FAKE_MIME	-	-
@@ -11,4 +11,4 @@
 1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	FAKE_MIME	-	-
 1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	image/png	e0029eea80812e9a8e57b8d05d52938a	-
 1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	image/png	30aa926344f58019d047e85ba049ca1e	-
-#end	2009-11-18-20-58-32
+#close	2009-11-18-20-58-32
diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
index 6e7eb96454..6b5e395902 100644
--- a/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
+++ b/testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
-#start	2009-11-18-20-58-04
+#open	2009-11-18-20-58-04
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	file
 1258577884.844956	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	1	GET	www.mozilla.org	/style/enhanced.css	http://www.mozilla.org/projects/calendar/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2675	200	OK	-	-	-	(empty)	-	-	-	-	-
@@ -11,4 +11,4 @@
 1258577885.317160	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	3	GET	www.mozilla.org	/images/template/screen/bullet_utility.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	94	200	OK	-	-	-	(empty)	-	-	-	-	-
 1258577885.349639	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	4	GET	www.mozilla.org	/images/template/screen/key-point-top.png	http://www.mozilla.org/style/screen.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	2349	200	OK	-	-	-	(empty)	-	-	-	-	-
 1258577885.394612	UWkUyAuUGXf	192.168.1.104	1673	63.245.209.11	80	5	GET	www.mozilla.org	/projects/calendar/images/header-sunbird.png	http://www.mozilla.org/projects/calendar/calendar.css	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.1.5) Gecko/20091102 Firefox/3.5.5	0	27579	200	OK	-	-	-	(empty)	-	-	-	-	-
-#end	2009-11-18-20-58-32
+#close	2009-11-18-20-58-32
diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
index fe18751420..46adaa4c3e 100644
--- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
+++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
-#start	2011-07-20-19-12-44
+#open	2011-07-20-19-12-44
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	file
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-
 1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	-
-#end	2011-07-20-19-15-42
+#close	2011-07-20-19-15-42
diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
index 8bd6bd8394..e204a627b1 100644
--- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
+++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
-#start	2011-07-20-19-12-44
+#open	2011-07-20-19-12-44
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	dcc_mime_type	extraction_file
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	string	file
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-	-
 1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-	-
 1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-	-
 1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
-#end	2011-07-20-19-15-42
+#close	2011-07-20-19-15-42
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
index eca41f7d09..ba16578dfb 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	smtp
-#start	2009-10-05-06-06-12
+#open	2009-10-05-06-06-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
 #types	time	string	addr	port	addr	port	count	string	string	table[string]	string	string	table[string]	string	string	string	string	addr	string	string	string	vector[addr]	string
 1254722768.219663	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	GP	<gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	Mon, 5 Oct 2009 11:36:07 +0530	"Gurpartap Singh" <gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	-	<000301ca4581$ef9e57f0$cedb07d0$@in>	-	SMTP	-	-	-	250 OK id=1Mugho-0003Dg-Un	74.53.140.153,10.10.1.4	Microsoft Office Outlook 12.0
-#end	2009-10-05-06-06-16
+#close	2009-10-05-06-06-16
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
index 9bae222897..396a2e058d 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	smtp_entities
-#start	2009-10-05-06-06-10
+#open	2009-10-05-06-06-10
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	file	string
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	79	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat	(empty)
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	1918	FAKE_MIME	-	-	(empty)
 1254722770.692804	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	NEWS.txt	10823	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat	(empty)
-#end	2009-10-05-06-06-16
+#close	2009-10-05-06-06-16
diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
index 5cb4bb15ef..1abe35e90f 100644
--- a/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
+++ b/testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	smtp_entities
-#start	2009-10-05-06-06-10
+#open	2009-10-05-06-06-10
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	file	string
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	79	FAKE_MIME	92bca2e6cdcde73647125da7dccbdd07	-	(empty)
 1254722770.692743	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	-	1918	FAKE_MIME	-	-	(empty)
 1254722770.692804	arKYeMETxOg	10.10.1.4	1470	74.53.140.153	25	1	NEWS.txt	10823	FAKE_MIME	a968bb0f9f9d95835b2e74c845877e87	-	(empty)
-#end	2009-10-05-06-06-16
+#close	2009-10-05-06-06-16
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log
index 960ea71720..b2a8ef7d4c 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	socks
-#start	2012-06-20-17-23-38
+#open	2012-06-20-17-23-38
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	user	status	request.host	request.name	request_p	bound.host	bound.name	bound_p
 #types	time	string	addr	port	addr	port	count	string	string	addr	string	port	addr	string	port
 1340213015.276495	UWkUyAuUGXf	10.0.0.55	53994	60.190.189.214	8124	5	-	succeeded	-	www.osnews.com	80	192.168.0.31	-	2688
-#end	2012-06-20-17-28-10
+#close	2012-06-20-17-28-10
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log
index d914b3074e..d5aa58652e 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/tunnel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
-#start	2012-06-20-17-23-35
+#open	2012-06-20-17-23-35
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340213015.276495	-	10.0.0.55	0	60.190.189.214	8124	Tunnel::SOCKS	Tunnel::DISCOVER
-#end	2012-06-20-17-28-10
+#close	2012-06-20-17-28-10
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log
index ef07cc31a5..4053bd7359 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/socks.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	socks
-#start	2012-06-19-13-41-02
+#open	2012-06-19-13-41-02
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	user	status	request.host	request.name	request_p	bound.host	bound.name	bound_p
 #types	time	string	addr	port	addr	port	count	string	string	addr	string	port	addr	string	port
 1340113261.914619	UWkUyAuUGXf	10.0.0.50	59580	85.194.84.197	1080	5	-	succeeded	-	www.google.com	443	0.0.0.0	-	443
-#end	2012-06-19-13-41-05
+#close	2012-06-19-13-41-05
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log
index 10f079b888..82df9b76df 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace2/tunnel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
-#start	2012-06-19-13-41-01
+#open	2012-06-19-13-41-01
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340113261.914619	-	10.0.0.50	0	85.194.84.197	1080	Tunnel::SOCKS	Tunnel::DISCOVER
-#end	2012-06-19-13-41-05
+#close	2012-06-19-13-41-05
diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log
index 4299e302ce..867f3ed157 100644
--- a/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log
+++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace3/tunnel.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
-#start	2008-04-15-22-43-49
+#open	2008-04-15-22-43-49
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1208299429.265774	-	127.0.0.1	0	127.0.0.1	1080	Tunnel::SOCKS	Tunnel::DISCOVER
-#end	2008-04-15-22-43-49
+#close	2008-04-15-22-43-49
diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
index b77925e498..5bf3feddc5 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#start	2012-04-27-14-53-12
+#open	2012-04-27-14-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert
 #types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string
 1335538392.319381	UWkUyAuUGXf	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-
-#end	2012-04-27-14-53-16
+#close	2012-04-27-14-53-16
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
index 6951e4d51f..d5f665e4bc 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log
@@ -3,11 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	ts	host
 #types	time	addr
 1300475168.783842	141.142.220.118
 1300475168.783842	208.80.152.118
 1300475168.915940	208.80.152.3
 1300475168.962628	208.80.152.2
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
index b70a701448..a625691aa4 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	ts	host
 #types	time	addr
 1300475168.783842	141.142.220.118
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
index 8e9d8c6c79..d05ccf6081 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_hosts
-#start	2011-03-18-19-06-08
+#open	2011-03-18-19-06-08
 #fields	ts	host
 #types	time	addr
 1300475168.783842	208.80.152.118
 1300475168.915940	208.80.152.3
 1300475168.962628	208.80.152.2
-#end	2011-03-18-19-06-13
+#close	2011-03-18-19-06-13
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
index 25198e92d5..af097e5db3 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log
@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
-#start	2011-06-24-15-51-31
+#open	2011-06-24-15-51-31
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table[string]
 1308930691.049431	172.16.238.131	22	tcp	SSH
@@ -11,4 +11,4 @@
 1308930716.462556	74.125.225.81	80	tcp	HTTP
 1308930718.361665	172.16.238.131	21	tcp	FTP
 1308930726.872485	141.142.192.39	22	tcp	SSH
-#end	2011-06-24-15-52-08
+#close	2011-06-24-15-52-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
index 598f49fa65..7c27e63a24 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log
@@ -3,10 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
-#start	2011-06-24-15-51-31
+#open	2011-06-24-15-51-31
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table[string]
 1308930691.049431	172.16.238.131	22	tcp	SSH
 1308930694.550308	172.16.238.131	80	tcp	HTTP
 1308930718.361665	172.16.238.131	21	tcp	FTP
-#end	2011-06-24-15-52-08
+#close	2011-06-24-15-52-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
index c248b18146..77fbe1ef70 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	known_services
-#start	2011-06-24-15-51-56
+#open	2011-06-24-15-51-56
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table[string]
 1308930716.462556	74.125.225.81	80	tcp	HTTP
 1308930726.872485	141.142.192.39	22	tcp	SSH
-#end	2011-06-24-15-52-08
+#close	2011-06-24-15-52-08
diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
index fb024db6d2..f4b77edde7 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
-#start	1999-06-28-23-40-27
+#open	1999-06-28-23-40-27
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	auth	addl
 #types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	table[string]	table[string]
 930613226.529070	UWkUyAuUGXf	212.180.42.100	25000	131.243.64.3	53	tcp	34798	-	-	-	-	-	0	NOERROR	F	F	F	T	0	4.3.2.1	31337.000000	-	-
-#end	1999-06-28-23-40-27
+#close	1999-06-28-23-40-27
diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps
index 84bd21aa60..138b901743 100755
--- a/testing/scripts/diff-remove-timestamps
+++ b/testing/scripts/diff-remove-timestamps
@@ -11,4 +11,4 @@ fi
 
 # The first sed uses a "basic" regexp, the 2nd a "modern:.
 sed 's/[0-9]\{10\}\.[0-9]\{2,8\}/XXXXXXXXXX.XXXXXX/g' | \
-$sed 's/^#(start|end).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'
+$sed 's/^#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g'

From 596f07e50569d0ecb4d65ef58bdf6c8ba65fe50e Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 27 Jul 2012 15:31:10 -0400
Subject: [PATCH 111/238] Reworked how the logs-to-elasticsearch scripts works
 to stop abusing the logging framework.

- New variable in logging framework Log::active_streams to indicate
  Log:ID enums which are currently active.
---
 scripts/base/frameworks/logging/main.bro      |  9 ++++++
 .../policy/tuning/logs-to-elasticsearch.bro   | 28 ++++++-------------
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro
index c29215fd86..aa44547567 100644
--- a/scripts/base/frameworks/logging/main.bro
+++ b/scripts/base/frameworks/logging/main.bro
@@ -327,6 +327,11 @@ export {
 	##    Log::default_rotation_postprocessor_cmd
 	##    Log::default_rotation_postprocessors
 	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
+
+	## The streams which are currently active and not disabled.
+	## This set is not meant to be modified by users!  Only use it for
+	## examining which streams are active.
+	global active_streams: set[ID] = set();
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
@@ -412,11 +417,15 @@ function create_stream(id: ID, stream: Stream) : bool
 	if ( ! __create_stream(id, stream) )
 		return F;
 
+	add active_streams[id];
+
 	return add_default_filter(id);
 	}
 
 function disable_stream(id: ID) : bool
 	{
+	delete active_streams[id];
+	
 	return __disable_stream(id);
 	}
 
diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro
index b4d16a19a1..44fc3800b8 100644
--- a/scripts/policy/tuning/logs-to-elasticsearch.bro
+++ b/scripts/policy/tuning/logs-to-elasticsearch.bro
@@ -4,7 +4,7 @@ module LogElasticSearch;
 
 export {
 	## An elasticsearch specific rotation interval.
-	const rotation_interval = 24hr &redef;
+	const rotation_interval = 3hr &redef;
 
 	## Optionally ignore any :bro:type:`Log::ID` from being sent to
 	## ElasticSearch with this script.
@@ -17,29 +17,17 @@ export {
 	const send_logs: set[string] = set() &redef;
 }
 
-module Log;
-
 event bro_init() &priority=-5
 	{
-	local my_filters: table[ID, string] of Filter = table();
-
-	for ( [id, name] in filters )
+	for ( stream_id in Log::active_streams )
 		{
-		local filter = filters[id, name];
-		if ( fmt("%s", id) in LogElasticSearch::excluded_log_ids ||
-		     (|LogElasticSearch::send_logs| > 0 && fmt("%s", id) !in LogElasticSearch::send_logs) )
+		if ( fmt("%s", stream_id) in excluded_log_ids ||
+		     (|send_logs| > 0 && fmt("%s", stream_id) !in send_logs) )
 			next;
 
-		filter$name = cat(name, "-es");
-		filter$writer = Log::WRITER_ELASTICSEARCH;
-		filter$interv = LogElasticSearch::rotation_interval;
-		my_filters[id, name] = filter;
-		}
-
-	# This had to be done separately to avoid an ever growing filters list
-	# where the for loop would never end.
-	for ( [id, name] in my_filters )
-		{
-		Log::add_filter(id, filter);
+		local filter: Log::Filter = [$name = "default-es",
+		                             $writer = Log::WRITER_ELASTICSEARCH,
+		                             $interv = LogElasticSearch::rotation_interval];
+		Log::add_filter(stream_id, filter);
 		}
 	}

From 767a7921482599eae41707a931fee065b6038c06 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 27 Jul 2012 12:30:40 -0700
Subject: [PATCH 112/238] Tests updates for recent open/close log change.

---
 .../ssh.log                                          | 12 +++++-------
 .../btest/scripts/base/frameworks/input/binary.bro   |  2 +-
 .../scripts/base/frameworks/logging/ascii-escape.bro |  2 +-
 .../scripts/base/frameworks/logging/remote-types.bro |  4 ++--
 4 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
index 7a448ce6c1..d61eae873a 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log
@@ -3,12 +3,10 @@
 #empty_field||(empty)
 #unset_field||-
 #path||ssh
-#open||2012-07-27-19-14-35
 #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country
 #types||time||addr||port||addr||port||string||string
-1343416475.837726||1.2.3.4||1234||2.3.4.5||80||success||unknown
-1343416475.837726||1.2.3.4||1234||2.3.4.5||80||failure||US
-1343416475.837726||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
-1343416475.837726||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
-1343416475.837726||1.2.3.4||1234||2.3.4.5||80||failure||MX
-#close||2012-07-27-19-14-35
+1343417536.767956||1.2.3.4||1234||2.3.4.5||80||success||unknown
+1343417536.767956||1.2.3.4||1234||2.3.4.5||80||failure||US
+1343417536.767956||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
+1343417536.767956||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
+1343417536.767956||1.2.3.4||1234||2.3.4.5||80||failure||MX
diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro
index ce7f66a01d..8d75abc5a9 100644
--- a/testing/btest/scripts/base/frameworks/input/binary.bro
+++ b/testing/btest/scripts/base/frameworks/input/binary.bro
@@ -16,7 +16,7 @@ redef InputAscii::unset_field = "-";
 #empty_field|(empty)
 #unset_field|-
 #path|ssh
-#start|2012-07-20-01-49-19
+#open|2012-07-20-01-49-19
 #fields|data|data2
 #types|string|string
 abc\x0a\xffdef|DATA2
diff --git a/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro b/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
index 1d0742216d..d73464777a 100644
--- a/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
+++ b/testing/btest/scripts/base/frameworks/logging/ascii-escape.bro
@@ -1,6 +1,6 @@
 #
 # @TEST-EXEC: bro -b %INPUT 
-# @TEST-EXEC: cat ssh.log | egrep -v '#start|#end' >ssh.log.tmp && mv ssh.log.tmp ssh.log
+# @TEST-EXEC: cat ssh.log | egrep -v '#open|#close' >ssh.log.tmp && mv ssh.log.tmp ssh.log
 # @TEST-EXEC: btest-diff ssh.log
 
 redef LogAscii::separator = "||";
diff --git a/testing/btest/scripts/base/frameworks/logging/remote-types.bro b/testing/btest/scripts/base/frameworks/logging/remote-types.bro
index 3f102e6319..b8425428d3 100644
--- a/testing/btest/scripts/base/frameworks/logging/remote-types.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote-types.bro
@@ -4,8 +4,8 @@
 # @TEST-EXEC: btest-bg-run receiver bro -B threading,logging --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 10
 # @TEST-EXEC: btest-diff receiver/test.log
-# @TEST-EXEC: cat receiver/test.log | egrep -v '#start|#end' >r.log
-# @TEST-EXEC: cat sender/test.log   | egrep -v '#start|#end' >s.log
+# @TEST-EXEC: cat receiver/test.log | egrep -v '#open|#close' >r.log
+# @TEST-EXEC: cat sender/test.log   | egrep -v '#open|#close' >s.log
 # @TEST-EXEC: cmp r.log s.log
 
 # Remote version testing all types.

From 9f2abd0697568377c901b3fa8cd38f79f5ccf953 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 27 Jul 2012 12:39:20 -0700
Subject: [PATCH 113/238] Fix input test for recent default change on fastpath.

---
 testing/btest/scripts/base/frameworks/input/missing-file.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro
index 269e287acc..aa5acf619e 100644
--- a/testing/btest/scripts/base/frameworks/input/missing-file.bro
+++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro
@@ -25,6 +25,6 @@ event bro_init()
 	{
 	try = 0;
 	outfile = open("../out");
-	Input::add_event([$source="does-not-exist.dat", $name="input", $fields=Val, $ev=line]);
+	Input::add_event([$source="does-not-exist.dat", $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::remove("input");
 	}

From 4bdac985cbbe53b2767fb56412e6bdc1a577da0b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sat, 28 Jul 2012 11:21:20 -0700
Subject: [PATCH 114/238] Tweaking logs-to-elasticsearch.bro so that it doesn't
 do anything if ES server is unset.

---
 scripts/policy/tuning/logs-to-elasticsearch.bro | 3 +++
 testing/external/scripts/testing-setup.bro      | 8 +++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro
index 44fc3800b8..207a9acc04 100644
--- a/scripts/policy/tuning/logs-to-elasticsearch.bro
+++ b/scripts/policy/tuning/logs-to-elasticsearch.bro
@@ -19,6 +19,9 @@ export {
 
 event bro_init() &priority=-5
 	{
+	if ( server_host == "" )
+		return;
+	
 	for ( stream_id in Log::active_streams )
 		{
 		if ( fmt("%s", stream_id) in excluded_log_ids ||
diff --git a/testing/external/scripts/testing-setup.bro b/testing/external/scripts/testing-setup.bro
index fa5664a877..4b4d110864 100644
--- a/testing/external/scripts/testing-setup.bro
+++ b/testing/external/scripts/testing-setup.bro
@@ -1,6 +1,12 @@
 # Sets some testing specific options.
 
 @ifdef ( SMTP::never_calc_md5 )
-        # MDD5s can depend on libmagic output.
+	# MDD5s can depend on libmagic output.
 	redef SMTP::never_calc_md5 = T;
 @endif
+
+@ifdef ( LogElasticSearch::server_host )
+	# Set to empty so that logs-to-elasticsearch.bro doesn't try to setup
+	#log forwarding to ES.
+	redef LogElasticSearch::server_host = "";
+@endif

From 4359bf6b42fb6438bf5d2285f07275625d9b542b Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 27 Jul 2012 13:31:17 -0500
Subject: [PATCH 115/238] Fix log manager hanging on waiting for pending file
 rotations.

This changes writer implementations to always respond to rotation
messages in their DoRotate() method, even for failure/no-op cases
with a new RotationFailedMessage.  This informs the manager to
decrement its count of pending rotations.

Addresses #860.
---
 src/logging/Manager.cc               | 14 +++++++++++-
 src/logging/Manager.h                |  7 ++++++
 src/logging/WriterBackend.cc         | 33 ++++++++++++++++++++++++++++
 src/logging/WriterBackend.h          | 19 ++++++++++++++++
 src/logging/writers/Ascii.cc         |  4 ++++
 src/logging/writers/DataSeries.cc    |  1 +
 src/logging/writers/ElasticSearch.cc |  1 +
 src/logging/writers/None.cc          |  1 +
 8 files changed, 79 insertions(+), 1 deletion(-)

diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 269ba32bfa..bcbea6e266 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -1215,12 +1215,16 @@ bool Manager::Flush(EnumVal* id)
 void Manager::Terminate()
 	{
 	// Make sure we process all the pending rotations.
-	while ( rotations_pending )
+
+	while ( rotations_pending > 0 )
 		{
 		thread_mgr->ForceProcessing(); // A blatant layering violation ...
 		usleep(1000);
 		}
 
+	if ( rotations_pending < 0 )
+		reporter->InternalError("Negative pending log rotations: %d", rotations_pending);
+
 	for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
 		{
 		if ( ! *s )
@@ -1384,3 +1388,11 @@ bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, con
 	return result;
 	}
 
+bool Manager::FailedRotation(WriterFrontend* writer, const char* filename,
+                             double open, double close, bool terminating)
+	{
+	--rotations_pending;
+	DBG_LOG(DBG_LOGGING, "Failed rotating writer '%s', file '%s' at %.6f,",
+	        writer->Name(), filename, network_time);
+	return true;
+	}
diff --git a/src/logging/Manager.h b/src/logging/Manager.h
index d2041592c1..7de99035c4 100644
--- a/src/logging/Manager.h
+++ b/src/logging/Manager.h
@@ -153,6 +153,7 @@ public:
 protected:
 	friend class WriterFrontend;
 	friend class RotationFinishedMessage;
+	friend class RotationFailedMessage;
 	friend class ::RemoteSerializer;
 	friend class ::RotationTimer;
 
@@ -178,6 +179,12 @@ protected:
 	bool FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
 			      double open, double close, bool terminating);
 
+	// Signals that a file couldn't be rotated, either because the writer
+	// implementation decided there was nothing to do or because a real error
+	// occurred.  In the error case, a separate message for the reason is sent.
+	bool FailedRotation(WriterFrontend* writer, const char* filename,
+	                    double open, double close, bool terminating);
+
 	// Deletes the values as passed into Write().
 	void DeleteVals(int num_fields, threading::Value** vals);
 
diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index afdc4b99c5..8b4d49d6e9 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -43,6 +43,32 @@ private:
         bool terminating;
 };
 
+class RotationFailedMessage : public threading::OutputMessage<WriterFrontend>
+{
+public:
+	RotationFailedMessage(WriterFrontend* writer, const char* filename,
+				double open, double close, bool terminating)
+		: threading::OutputMessage<WriterFrontend>("RotationFailed", writer),
+		filename(copy_string(filename)), open(open),
+		close(close), terminating(terminating)	{ }
+
+	virtual ~RotationFailedMessage()
+		{
+		delete [] filename;
+		}
+
+	virtual bool Process()
+		{
+		return log_mgr->FailedRotation(Object(), filename, open, close, terminating);
+		}
+
+private:
+        const char* filename;
+        double open;
+        double close;
+        bool terminating;
+};
+
 class FlushWriteBufferMessage : public threading::OutputMessage<WriterFrontend>
 {
 public:
@@ -164,6 +190,13 @@ bool WriterBackend::FinishedRotation(const char* new_name, const char* old_name,
 	return true;
 	}
 
+bool WriterBackend::FailedRotation(const char* filename, double open,
+                                   double close, bool terminating)
+	{
+	SendOut(new RotationFailedMessage(frontend, filename, open, close, terminating));
+	return true;
+	}
+
 void WriterBackend::DisableFrontend()
 	{
 	SendOut(new DisableMessage(frontend));
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index 77dbe71f45..64eb13ddec 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -229,6 +229,25 @@ public:
 	bool FinishedRotation(const char* new_name, const char* old_name,
 			      double open, double close, bool terminating);
 
+	/**
+	 * Signals that a file couldn't be rotated. This must be called by a
+	 * writer's implementation of DoRotate() in all cases where
+	 * FinishedRotation() was not called or failed.
+	 *
+	 * Most of the parameters should be passed through from DoRotate().
+	 *
+	 * @param filename The name of the file that was attempted to be rotated.
+	 *
+	 * @param open: The timestamp when the original file was opened.
+	 *
+	 * @param close: The timestamp when the origina file was closed.
+	 *
+	 * @param terminating: True if the original rotation request occured
+	 * due to the main Bro process shutting down.
+	 */
+	bool FailedRotation(const char* filename, double open, double close,
+	                    bool terminating);
+
 	/** Helper method to render an IP address as a string.
 	  *
 	  * @param addr The address.
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index c4c6b06563..805ccaa4cc 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -373,7 +373,10 @@ bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool t
 	{
 	// Don't rotate special files or if there's not one currently open.
 	if ( ! fd || IsSpecial(Info().path) )
+		{
+		FailedRotation(rotated_path, open, close, terminating);
 		return true;
+		}
 
 	CloseFile(close);
 
@@ -382,6 +385,7 @@ bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool t
 
 	if ( ! FinishedRotation(nname.c_str(), fname.c_str(), open, close, terminating) )
 		{
+		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
 		return false;
 		}
diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc
index 7d3053e341..29e1705bf5 100644
--- a/src/logging/writers/DataSeries.cc
+++ b/src/logging/writers/DataSeries.cc
@@ -407,6 +407,7 @@ bool DataSeries::DoRotate(const char* rotated_path, double open, double close, b
 
 	if ( ! FinishedRotation(nname.c_str(), dsname.c_str(), open, close, terminating) )
 		{
+		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
 		return false;
 		}
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index cc6f8b1c4f..d663e375c5 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -323,6 +323,7 @@ bool ElasticSearch::DoRotate(const char* rotated_path, double open, double close
 
 	if ( ! FinishedRotation(current_index.c_str(), prev_index.c_str(), open, close, terminating) )
 		{
+		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
 		}
 
diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc
index 9b91b82199..0d659ed34e 100644
--- a/src/logging/writers/None.cc
+++ b/src/logging/writers/None.cc
@@ -46,6 +46,7 @@ bool None::DoRotate(const char* rotated_path, double open, double close, bool te
 	{
 	if ( ! FinishedRotation("/dev/null", Info().path, open, close, terminating))
 		{
+		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s", Info().path));
 		return false;
 		}

From 4ba038070f0047a81e422ada9a347395f5ba911d Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sat, 28 Jul 2012 11:55:31 -0700
Subject: [PATCH 116/238] Tweaking writer API for failed rotations.

There are now two FinishedRotation() methods, one that triggers
post-processing and one that doesn't. There's also insurance built in
against a writer not calling either (or both), in which case we abort
with an internal error.
---
 CHANGES                              |  9 +++++
 VERSION                              |  2 +-
 src/logging/Manager.cc               | 20 +++++------
 src/logging/Manager.h                |  8 +----
 src/logging/WriterBackend.cc         | 52 ++++++++++------------------
 src/logging/WriterBackend.h          | 32 +++++++++++------
 src/logging/WriterFrontend.cc        |  5 ++-
 src/logging/writers/Ascii.cc         |  3 +-
 src/logging/writers/DataSeries.cc    |  1 -
 src/logging/writers/ElasticSearch.cc |  3 --
 src/logging/writers/None.cc          |  1 -
 src/util.cc                          |  3 ++
 12 files changed, 65 insertions(+), 74 deletions(-)

diff --git a/CHANGES b/CHANGES
index aaa2c53569..b3fe4ad620 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,13 @@
 
+2.0-905 | 2012-07-28 16:24:34 -0700
+
+  * Fix log manager hanging on waiting for pending file rotations,
+    plus writer API tweak for failed rotations. Addresses #860. (Jon
+    Siwek and Robin Sommer)
+
+  * Tweaking logs-to-elasticsearch.bro so that it doesn't do anything
+    if ES server is unset. (Robin Sommer)
+
 2.0-902 | 2012-07-27 12:42:13 -0700
 
   * New variable in logging framework Log::active_streams to indicate
diff --git a/VERSION b/VERSION
index f320985bf6..57c0d2a8a9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0-902
+2.0-905
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index bcbea6e266..7a182a78b7 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -1338,13 +1338,18 @@ void Manager::Rotate(WriterInfo* winfo)
 	}
 
 bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
-		      double open, double close, bool terminating)
+		      double open, double close, bool success, bool terminating)
 	{
+	assert(writer);
+
 	--rotations_pending;
 
-	if ( ! writer )
-		// Writer didn't produce local output.
+	if ( ! success )
+		{
+		DBG_LOG(DBG_LOGGING, "Non-successful rotating writer '%s', file '%s' at %.6f,",
+			writer->Name(), filename, network_time);
 		return true;
+		}
 
 	DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
 		writer->Name(), network_time, new_name);
@@ -1387,12 +1392,3 @@ bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, con
 
 	return result;
 	}
-
-bool Manager::FailedRotation(WriterFrontend* writer, const char* filename,
-                             double open, double close, bool terminating)
-	{
-	--rotations_pending;
-	DBG_LOG(DBG_LOGGING, "Failed rotating writer '%s', file '%s' at %.6f,",
-	        writer->Name(), filename, network_time);
-	return true;
-	}
diff --git a/src/logging/Manager.h b/src/logging/Manager.h
index 7de99035c4..864a23ca88 100644
--- a/src/logging/Manager.h
+++ b/src/logging/Manager.h
@@ -177,13 +177,7 @@ protected:
 
 	// Signals that a file has been rotated.
 	bool FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
-			      double open, double close, bool terminating);
-
-	// Signals that a file couldn't be rotated, either because the writer
-	// implementation decided there was nothing to do or because a real error
-	// occurred.  In the error case, a separate message for the reason is sent.
-	bool FailedRotation(WriterFrontend* writer, const char* filename,
-	                    double open, double close, bool terminating);
+			      double open, double close, bool success, bool terminating);
 
 	// Deletes the values as passed into Write().
 	void DeleteVals(int num_fields, threading::Value** vals);
diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc
index 8b4d49d6e9..47fdec27ef 100644
--- a/src/logging/WriterBackend.cc
+++ b/src/logging/WriterBackend.cc
@@ -19,10 +19,10 @@ class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
 {
 public:
 	RotationFinishedMessage(WriterFrontend* writer, const char* new_name, const char* old_name,
-				double open, double close, bool terminating)
+				double open, double close, bool success, bool terminating)
 		: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
 		new_name(copy_string(new_name)), old_name(copy_string(old_name)), open(open),
-		close(close), terminating(terminating)	{ }
+		close(close), success(success), terminating(terminating)	{ }
 
 	virtual ~RotationFinishedMessage()
 		{
@@ -32,7 +32,7 @@ public:
 
 	virtual bool Process()
 		{
-		return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating);
+		return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, success, terminating);
 		}
 
 private:
@@ -40,32 +40,7 @@ private:
         const char* old_name;
         double open;
         double close;
-        bool terminating;
-};
-
-class RotationFailedMessage : public threading::OutputMessage<WriterFrontend>
-{
-public:
-	RotationFailedMessage(WriterFrontend* writer, const char* filename,
-				double open, double close, bool terminating)
-		: threading::OutputMessage<WriterFrontend>("RotationFailed", writer),
-		filename(copy_string(filename)), open(open),
-		close(close), terminating(terminating)	{ }
-
-	virtual ~RotationFailedMessage()
-		{
-		delete [] filename;
-		}
-
-	virtual bool Process()
-		{
-		return log_mgr->FailedRotation(Object(), filename, open, close, terminating);
-		}
-
-private:
-        const char* filename;
-        double open;
-        double close;
+	bool success;
         bool terminating;
 };
 
@@ -152,6 +127,7 @@ WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
 	buffering = true;
 	frontend = arg_frontend;
 	info = new WriterInfo(frontend->Info());
+	rotation_counter = 0;
 
 	SetName(frontend->Name());
 	}
@@ -186,14 +162,15 @@ void WriterBackend::DeleteVals(int num_writes, Value*** vals)
 bool WriterBackend::FinishedRotation(const char* new_name, const char* old_name,
 				     double open, double close, bool terminating)
 	{
-	SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating));
+	--rotation_counter;
+	SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, true, terminating));
 	return true;
 	}
 
-bool WriterBackend::FailedRotation(const char* filename, double open,
-                                   double close, bool terminating)
+bool WriterBackend::FinishedRotation()
 	{
-	SendOut(new RotationFailedMessage(frontend, filename, open, close, terminating));
+	--rotation_counter;
+	SendOut(new RotationFinishedMessage(frontend, 0, 0, 0, 0, false, false));
 	return true;
 	}
 
@@ -303,12 +280,21 @@ bool WriterBackend::Rotate(const char* rotated_path, double open,
 	if ( Failed() )
 		return true;
 
+	rotation_counter = 1;
+
 	if ( ! DoRotate(rotated_path, open, close, terminating) )
 		{
 		DisableFrontend();
 		return false;
 		}
 
+	// Insurance against broken writers.
+	if ( rotation_counter > 0 )
+		InternalError(Fmt("writer %s did not call FinishedRotation() in DoRotation()", Name()));
+
+	if ( rotation_counter < 0 )
+		InternalError(Fmt("writer %s called FinishedRotation() more than once in DoRotation()", Name()));
+
 	return true;
 	}
 
diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h
index 64eb13ddec..89185619c4 100644
--- a/src/logging/WriterBackend.h
+++ b/src/logging/WriterBackend.h
@@ -210,11 +210,15 @@ public:
 	bool IsBuf()	{ return buffering; }
 
 	/**
-	 * Signals that a file has been rotated. This must be called by a
-	 * writer's implementation of DoRotate() once rotation has finished.
+	 * Signals that a file has been successfully rotated and any
+	 * potential post-processor can now run.
 	 *
 	 * Most of the parameters should be passed through from DoRotate().
 	 *
+	 * Note: Exactly one of the two FinishedRotation() methods must be
+	 * called by a writer's implementation of DoRotate() once rotation
+	 * has finished.
+	 *
 	 * @param new_name The filename of the rotated file.
 	 *
 	 * @param old_name The filename of the original file.
@@ -230,13 +234,18 @@ public:
 			      double open, double close, bool terminating);
 
 	/**
-	 * Signals that a file couldn't be rotated. This must be called by a
-	 * writer's implementation of DoRotate() in all cases where
-	 * FinishedRotation() was not called or failed.
+	 * Signals that a file rotation request has been processed, but no
+	 * further post-processing needs to be performed (either because
+	 * there was an error, or there was nothing to rotate to begin with
+	 * with this writer).
 	 *
-	 * Most of the parameters should be passed through from DoRotate().
+	 * Note: Exactly one of the two FinishedRotation() methods must be
+	 * called by a writer's implementation of DoRotate() once rotation
+	 * has finished.
 	 *
-	 * @param filename The name of the file that was attempted to be rotated.
+	 * @param new_name The filename of the rotated file.
+	 *
+	 * @param old_name The filename of the original file.
 	 *
 	 * @param open: The timestamp when the original file was opened.
 	 *
@@ -245,8 +254,7 @@ public:
 	 * @param terminating: True if the original rotation request occured
 	 * due to the main Bro process shutting down.
 	 */
-	bool FailedRotation(const char* filename, double open, double close,
-	                    bool terminating);
+	bool FinishedRotation();
 
 	/** Helper method to render an IP address as a string.
 	  *
@@ -344,8 +352,8 @@ protected:
 	 * Writer-specific method implementing log rotation.  Most directly
 	 * this only applies to writers writing into files, which should then
 	 * close the current file and open a new one.  However, a writer may
-	 * also trigger other apppropiate actions if semantics are similar. *
-	 * Once rotation has finished, the implementation must call
+	 * also trigger other apppropiate actions if semantics are similar.
+	 * Once rotation has finished, the implementation *must* call
 	 * FinishedRotation() to signal the log manager that potential
 	 * postprocessors can now run.
 	 *
@@ -407,6 +415,8 @@ private:
 	int num_fields;	// Number of log fields.
 	const threading::Field* const*  fields;	// Log fields.
 	bool buffering;	// True if buffering is enabled.
+
+	int rotation_counter; // Tracks FinishedRotation() calls.
 };
 
 
diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc
index 7c8f6861cf..a97f48c1ed 100644
--- a/src/logging/WriterFrontend.cc
+++ b/src/logging/WriterFrontend.cc
@@ -248,9 +248,8 @@ void WriterFrontend::Rotate(const char* rotated_path, double open, double close,
 	if ( backend )
 		backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating));
 	else
-		// Still signal log manager that we're done, but signal that
-		// nothing happened by setting the writer to zeri.
-		log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating);
+		// Still signal log manager that we're done.
+		log_mgr->FinishedRotation(this, 0, 0, 0, 0, false, terminating);
 	}
 
 void WriterFrontend::DeleteVals(Value** vals)
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index 805ccaa4cc..f6df3b9336 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -374,7 +374,7 @@ bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool t
 	// Don't rotate special files or if there's not one currently open.
 	if ( ! fd || IsSpecial(Info().path) )
 		{
-		FailedRotation(rotated_path, open, close, terminating);
+		FinishedRotation();
 		return true;
 		}
 
@@ -385,7 +385,6 @@ bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool t
 
 	if ( ! FinishedRotation(nname.c_str(), fname.c_str(), open, close, terminating) )
 		{
-		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
 		return false;
 		}
diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc
index 29e1705bf5..7d3053e341 100644
--- a/src/logging/writers/DataSeries.cc
+++ b/src/logging/writers/DataSeries.cc
@@ -407,7 +407,6 @@ bool DataSeries::DoRotate(const char* rotated_path, double open, double close, b
 
 	if ( ! FinishedRotation(nname.c_str(), dsname.c_str(), open, close, terminating) )
 		{
-		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
 		return false;
 		}
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index d663e375c5..7a80866bf7 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -322,10 +322,7 @@ bool ElasticSearch::DoRotate(const char* rotated_path, double open, double close
 		}
 
 	if ( ! FinishedRotation(current_index.c_str(), prev_index.c_str(), open, close, terminating) )
-		{
-		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
-		}
 
 	return true;
 	}
diff --git a/src/logging/writers/None.cc b/src/logging/writers/None.cc
index 0d659ed34e..9b91b82199 100644
--- a/src/logging/writers/None.cc
+++ b/src/logging/writers/None.cc
@@ -46,7 +46,6 @@ bool None::DoRotate(const char* rotated_path, double open, double close, bool te
 	{
 	if ( ! FinishedRotation("/dev/null", Info().path, open, close, terminating))
 		{
-		FailedRotation(rotated_path, open, close, terminating);
 		Error(Fmt("error rotating %s", Info().path));
 		return false;
 		}
diff --git a/src/util.cc b/src/util.cc
index 228e40dddb..2d981e952e 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -113,6 +113,9 @@ std::string get_escaped_string(const std::string& str, bool escape_all)
 
 char* copy_string(const char* s)
 	{
+	if ( ! s )
+		return 0;
+
 	char* c = new char[strlen(s)+1];
 	strcpy(c, s);
 	return c;

From 00d41bb549732d0a66a0fa683264a063705821d9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 30 Jul 2012 11:07:43 -0500
Subject: [PATCH 117/238] Add missing breaks to switch cases in
 ElasticSearch::HTTPReceive().

Observed as reason for segfault in
testing/btest/scripts/check-test-all-policy.bro unit test when compiled
with optimizations.
---
 src/logging/writers/ElasticSearch.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 7a80866bf7..9e5e3fb207 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -385,12 +385,14 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 			if ( ! failing )
 				Error(Fmt("ElasticSearch server may not be accessible."));
 			}
+			break;
 
 		case CURLE_OPERATION_TIMEDOUT:
 			{
 			if ( ! failing )
 				Warning(Fmt("HTTP operation with elasticsearch server timed out at %" PRIu64 " msecs.", transfer_timeout));
 			}
+			break;
 
 		case CURLE_OK:
 			{
@@ -402,10 +404,12 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 			else if ( ! failing )
 				Error(Fmt("Received a non-successful status code back from ElasticSearch server, check the elasticsearch server log."));
 			}
+			break;
 
 		default:
 			{
 			}
+			break;
 		}
 		// The "successful" return happens above
 		return false;

From 7b2c3db4881dd8acb3836c91c6d9da0895578405 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 30 Jul 2012 13:09:13 -0500
Subject: [PATCH 118/238] Improve log filter compatibility with remote logging.

If a log filter attempts to write to a path for which a writer is
already instantiated due to remote logging, it will re-use the writer
as long as the fields of the filter and writer are compatible, else
the filter path will be auto-adjusted to not conflict with existing
writer's.  Conflicts between two local filters are still always
auto-adjusted even if field types agree (since they could still
be semantically different).

Addresses #842.
---
 src/RemoteSerializer.cc |  3 ++-
 src/logging/Manager.cc  | 38 +++++++++++++++++++++++++++++++-------
 src/logging/Manager.h   |  4 +++-
 3 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc
index 4e9ccb7dd2..cfd20eba39 100644
--- a/src/RemoteSerializer.cc
+++ b/src/RemoteSerializer.cc
@@ -2716,7 +2716,8 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 	id_val = new EnumVal(id, BifType::Enum::Log::ID);
 	writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
 
-	if ( ! log_mgr->CreateWriter(id_val, writer_val, info, num_fields, fields, true, false) )
+	if ( ! log_mgr->CreateWriter(id_val, writer_val, info, num_fields, fields,
+	                             true, false, true) )
 		goto error;
 
 	Unref(id_val);
diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 7a182a78b7..4c6d2e92fd 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -86,6 +86,7 @@ struct Manager::WriterInfo {
 	Func* postprocessor;
 	WriterFrontend* writer;
 	WriterBackend::WriterInfo* info;
+	bool from_remote;
 	string instantiating_filter;
 	};
 
@@ -240,6 +241,29 @@ Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer)
 	return 0;
 	}
 
+bool Manager::CompareFields(const Filter* filter, const WriterFrontend* writer)
+	{
+	if ( filter->num_fields != writer->NumFields() )
+		return false;
+
+	for ( int i = 0; i < filter->num_fields; ++ i)
+		if ( filter->fields[i]->type != writer->Fields()[i]->type )
+			return false;
+
+	return true;
+	}
+
+bool Manager::CheckFilterWriterConflict(const WriterInfo* winfo, const Filter* filter)
+	{
+	if ( winfo->from_remote )
+		// If the writer was instantiated as a result of remote logging, then
+		// a filter and writer are only compatible if field types match
+		return ! CompareFields(filter, winfo->writer);
+	else
+		// If the writer was instantiated locally, it is bound to one filter
+		return winfo->instantiating_filter != filter->name;
+	}
+
 void Manager::RemoveDisabledWriters(Stream* stream)
 	{
 	list<Stream::WriterPathPair> disabled;
@@ -756,10 +780,9 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 		Stream::WriterMap::iterator w = stream->writers.find(wpp);
 
 		if ( w != stream->writers.end() &&
-		     w->second->instantiating_filter != filter->name )
+		     CheckFilterWriterConflict(w->second, filter) )
 			{
-			// Auto-correct path due to conflict with another filter over the
-			// same writer/path pair
+			// Auto-correct path due to conflict over the writer/path pairs.
 			string instantiator = w->second->instantiating_filter;
 			string new_path;
 			unsigned int i = 2;
@@ -771,7 +794,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 				wpp.second = new_path;
 				w = stream->writers.find(wpp);
 			} while ( w != stream->writers.end() &&
-				  w->second->instantiating_filter != filter->name );
+			          CheckFilterWriterConflict(w->second, filter) );
 
 			Unref(filter->path_val);
 			filter->path_val = new StringVal(new_path.c_str());
@@ -824,8 +847,8 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 			// CreateWriter() will set the other fields in info.
 
 			writer = CreateWriter(stream->id, filter->writer,
-					      info, filter->num_fields,
-					      arg_fields, filter->local, filter->remote, filter->name);
+					      info, filter->num_fields, arg_fields, filter->local,
+					      filter->remote, false, filter->name);
 
 			if ( ! writer )
 				{
@@ -1024,7 +1047,7 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 	}
 
 WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
-				int num_fields, const threading::Field* const*  fields, bool local, bool remote,
+				int num_fields, const threading::Field* const*  fields, bool local, bool remote, bool from_remote,
 				const string& instantiating_filter)
 	{
 	Stream* stream = FindStream(id);
@@ -1049,6 +1072,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken
 	winfo->interval = 0;
 	winfo->postprocessor = 0;
 	winfo->info = info;
+	winfo->from_remote = from_remote;
 	winfo->instantiating_filter = instantiating_filter;
 
 	// Search for a corresponding filter for the writer/path pair and use its
diff --git a/src/logging/Manager.h b/src/logging/Manager.h
index 864a23ca88..90ad944bc6 100644
--- a/src/logging/Manager.h
+++ b/src/logging/Manager.h
@@ -166,7 +166,7 @@ protected:
 	// Takes ownership of fields and info.
 	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
 				int num_fields, const threading::Field* const* fields,
-				bool local, bool remote, const string& instantiating_filter="");
+				bool local, bool remote, bool from_remote, const string& instantiating_filter="");
 
 	// Takes ownership of values..
 	bool Write(EnumVal* id, EnumVal* writer, string path,
@@ -200,6 +200,8 @@ private:
 	void Rotate(WriterInfo* info);
 	Filter* FindFilter(EnumVal* id, StringVal* filter);
 	WriterInfo* FindWriter(WriterFrontend* writer);
+	bool CompareFields(const Filter* filter, const WriterFrontend* writer);
+	bool CheckFilterWriterConflict(const WriterInfo* winfo, const Filter* filter);
 
 	vector<Stream *> streams;	// Indexed by stream enum.
 	int rotations_pending;	// Number of rotations not yet finished.

From e3acf3af58979b1d0a42c5eb6ae45edc8f208188 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 30 Jul 2012 11:59:53 -0700
Subject: [PATCH 119/238] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 4f01ea4081..99e7a27431 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 4f01ea40817ad232a96535c64fce7dc16d4e2fff
+Subproject commit 99e7a274319619a94a421eb62537c7a5c184f71b
diff --git a/aux/broccoli b/aux/broccoli
index 8234b8903c..b3692a02ba 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 8234b8903cbc775f341bdb6a1c0159981d88d27b
+Subproject commit b3692a02bae9a47d701d2d547e327dd429a86e76
diff --git a/aux/broctl b/aux/broctl
index 231358f166..5c9ed0d77b 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 231358f166f61cc32201a8ac3671ea0c0f5c324e
+Subproject commit 5c9ed0d77bcb3e03d7e68334fe0d088fa2f92c71

From 01d91602ca60a0e2fc868b350c5170a9dd8452ce Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 30 Jul 2012 12:00:14 -0700
Subject: [PATCH 120/238] Updating CHANGES and VERSION.

---
 CHANGES | 2 +-
 NEWS    | 3 ++-
 VERSION | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 8b0303d520..5267fa9f37 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.0-909 | 2012-07-30 11:46:45 -0700
+2.1-beta | 2012-07-30 11:59:53 -0700
 
   * Improve log filter compatibility with remote logging. Addresses
     #842. (Jon Siwek)
diff --git a/NEWS b/NEWS
index 7b60a05ccd..949b51d832 100644
--- a/NEWS
+++ b/NEWS
@@ -82,7 +82,8 @@ New Functionality
 
     * ElasticSearch: a distributed RESTful, storage engine and search
       engine built on top of Apache Lucene. It scales very well, both
-      for distributed indexing and distributed searching.
+      for distributed indexing and distributed searching. See
+      doc/logging-elasticsearch.rst for more information.
 
   Note that at this point, we consider Bro's support for these two
   formats as prototypes for collecting experience with alternative
diff --git a/VERSION b/VERSION
index 08cd7ce835..0fb956a360 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.0-909
+2.1-beta

From 3bb6d4e54e6883cd9d64812d11aa9d2be9ed4fb4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 1 Aug 2012 13:58:18 -0500
Subject: [PATCH 121/238] Fix configure script to exit with non-zero status on
 error

---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 configure    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 99e7a27431..22120825f8 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 99e7a274319619a94a421eb62537c7a5c184f71b
+Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
diff --git a/aux/bro-aux b/aux/bro-aux
index c691c01e9c..941ee753f7 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
+Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
diff --git a/aux/broccoli b/aux/broccoli
index b3692a02ba..5ff3e6a8e8 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit b3692a02bae9a47d701d2d547e327dd429a86e76
+Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
diff --git a/aux/broctl b/aux/broctl
index 5c9ed0d77b..903108f6b4 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 5c9ed0d77bcb3e03d7e68334fe0d088fa2f92c71
+Subproject commit 903108f6b43ad228309713da880026d50add41f4
diff --git a/configure b/configure
index bfe54123f0..b4ca606103 100755
--- a/configure
+++ b/configure
@@ -1,7 +1,7 @@
 #!/bin/sh
 # Convenience wrapper for easily viewing/setting options that
 # the project's CMake scripts will recognize
-
+set -e
 command="$0 $*"
 
 # check for `cmake` command

From 9829cf9a296b4f1c6614658b80225cd80f2e24ec Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 3 Aug 2012 10:44:46 -0700
Subject: [PATCH 122/238] Fixing little typo with big impact.

---
 CHANGES                      | 5 +++++
 VERSION                      | 2 +-
 src/logging/writers/Ascii.cc | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 5267fa9f37..644a56d458 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,9 @@
 
+2.0-912 | 2012-08-03 10:44:46 -0700
+
+  * Fixing little typo with big impact. (Robin Sommer)
+
+
 2.1-beta | 2012-07-30 11:59:53 -0700
 
   * Improve log filter compatibility with remote logging. Addresses
diff --git a/VERSION b/VERSION
index 0fb956a360..f1cb181c5c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta
+2.0-912
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index f6df3b9336..11b322f5a3 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -359,7 +359,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	if ( ! safe_write(fd, bytes, len) )
 		goto write_error;
 
-        if ( IsBuf() )
+        if ( ! IsBuf() )
 		fsync(fd);
 
 	return true;

From 10b671a6389ab0720a18ec6fb32be6e03ba6fa0b Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 3 Aug 2012 17:24:04 -0500
Subject: [PATCH 123/238] Add tests for untested BIFs

---
 testing/btest/Baseline/bifs.analyzer_name/out |  1 +
 testing/btest/Baseline/bifs.entropy_test/out  |  2 ++
 testing/btest/Baseline/bifs.global_sizes/out  |  1 +
 testing/btest/Baseline/bifs.identify_data/out |  4 ++++
 testing/btest/Baseline/bifs.strftime/out      |  4 ++++
 testing/btest/bifs/analyzer_name.bro          |  9 +++++++
 testing/btest/bifs/bro_version.bro            |  9 +++++++
 testing/btest/bifs/checkpoint_state.bro       | 10 ++++++++
 testing/btest/bifs/current_analyzer.bro       | 11 +++++++++
 testing/btest/bifs/current_time.bro           |  9 +++++++
 testing/btest/bifs/entropy_test.bro           | 24 +++++++++++++++++++
 testing/btest/bifs/gethostname.bro            |  9 +++++++
 testing/btest/bifs/getpid.bro                 |  9 +++++++
 testing/btest/bifs/global_sizes.bro           | 16 +++++++++++++
 testing/btest/bifs/identify_data.bro          | 16 +++++++++++++
 testing/btest/bifs/resource_usage.bro         |  9 +++++++
 testing/btest/bifs/strftime.bro               | 17 +++++++++++++
 17 files changed, 160 insertions(+)
 create mode 100644 testing/btest/Baseline/bifs.analyzer_name/out
 create mode 100644 testing/btest/Baseline/bifs.entropy_test/out
 create mode 100644 testing/btest/Baseline/bifs.global_sizes/out
 create mode 100644 testing/btest/Baseline/bifs.identify_data/out
 create mode 100644 testing/btest/Baseline/bifs.strftime/out
 create mode 100644 testing/btest/bifs/analyzer_name.bro
 create mode 100644 testing/btest/bifs/bro_version.bro
 create mode 100644 testing/btest/bifs/checkpoint_state.bro
 create mode 100644 testing/btest/bifs/current_analyzer.bro
 create mode 100644 testing/btest/bifs/current_time.bro
 create mode 100644 testing/btest/bifs/entropy_test.bro
 create mode 100644 testing/btest/bifs/gethostname.bro
 create mode 100644 testing/btest/bifs/getpid.bro
 create mode 100644 testing/btest/bifs/global_sizes.bro
 create mode 100644 testing/btest/bifs/identify_data.bro
 create mode 100644 testing/btest/bifs/resource_usage.bro
 create mode 100644 testing/btest/bifs/strftime.bro

diff --git a/testing/btest/Baseline/bifs.analyzer_name/out b/testing/btest/Baseline/bifs.analyzer_name/out
new file mode 100644
index 0000000000..84613e9dd1
--- /dev/null
+++ b/testing/btest/Baseline/bifs.analyzer_name/out
@@ -0,0 +1 @@
+PIA_TCP
diff --git a/testing/btest/Baseline/bifs.entropy_test/out b/testing/btest/Baseline/bifs.entropy_test/out
new file mode 100644
index 0000000000..08a09de4e4
--- /dev/null
+++ b/testing/btest/Baseline/bifs.entropy_test/out
@@ -0,0 +1,2 @@
+[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
+[entropy=2.083189, chi_square=3906.018182, mean=69.054545, monte_carlo_pi=4.0, serial_correlation=0.849402]
diff --git a/testing/btest/Baseline/bifs.global_sizes/out b/testing/btest/Baseline/bifs.global_sizes/out
new file mode 100644
index 0000000000..76c40b297a
--- /dev/null
+++ b/testing/btest/Baseline/bifs.global_sizes/out
@@ -0,0 +1 @@
+found bro_init
diff --git a/testing/btest/Baseline/bifs.identify_data/out b/testing/btest/Baseline/bifs.identify_data/out
new file mode 100644
index 0000000000..a2872877f9
--- /dev/null
+++ b/testing/btest/Baseline/bifs.identify_data/out
@@ -0,0 +1,4 @@
+ASCII text, with no line terminators
+text/plain; charset=us-ascii
+PNG image data
+image/png; charset=binary
diff --git a/testing/btest/Baseline/bifs.strftime/out b/testing/btest/Baseline/bifs.strftime/out
new file mode 100644
index 0000000000..b32393b332
--- /dev/null
+++ b/testing/btest/Baseline/bifs.strftime/out
@@ -0,0 +1,4 @@
+1970-01-01 00:00:00
+000000 19700101
+1973-11-29 21:33:09
+213309 19731129
diff --git a/testing/btest/bifs/analyzer_name.bro b/testing/btest/bifs/analyzer_name.bro
new file mode 100644
index 0000000000..034344f5c4
--- /dev/null
+++ b/testing/btest/bifs/analyzer_name.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = 1;
+	print analyzer_name(a);
+	}
diff --git a/testing/btest/bifs/bro_version.bro b/testing/btest/bifs/bro_version.bro
new file mode 100644
index 0000000000..7465cbc0f5
--- /dev/null
+++ b/testing/btest/bifs/bro_version.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = bro_version();
+	if ( |a| == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/checkpoint_state.bro b/testing/btest/bifs/checkpoint_state.bro
new file mode 100644
index 0000000000..2a66bd1729
--- /dev/null
+++ b/testing/btest/bifs/checkpoint_state.bro
@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: test -f .state/state.bst
+
+event bro_init()
+	{
+	local a = checkpoint_state();
+	if ( a != T )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/current_analyzer.bro b/testing/btest/bifs/current_analyzer.bro
new file mode 100644
index 0000000000..45b495c046
--- /dev/null
+++ b/testing/btest/bifs/current_analyzer.bro
@@ -0,0 +1,11 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = current_analyzer();
+	if ( a != 0 )
+		exit(1);
+
+	# TODO: add a test for non-zero return value
+	}
diff --git a/testing/btest/bifs/current_time.bro b/testing/btest/bifs/current_time.bro
new file mode 100644
index 0000000000..5d16df396d
--- /dev/null
+++ b/testing/btest/bifs/current_time.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = current_time();
+	if ( a <= double_to_time(0) )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/entropy_test.bro b/testing/btest/bifs/entropy_test.bro
new file mode 100644
index 0000000000..ca01c79ed7
--- /dev/null
+++ b/testing/btest/bifs/entropy_test.bro
@@ -0,0 +1,24 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f";
+	if ( entropy_test_init(1) != T )
+		exit(1);
+
+	if ( entropy_test_add(1, a) != T )
+		exit(1);
+
+	print entropy_test_finish(1);
+
+	local b = "0011000aaabbbbcccc000011111000000000aaaabbbbcccc0000000";
+	if ( entropy_test_init(2) != T )
+		exit(1);
+
+	if ( entropy_test_add(2, b) != T )
+		exit(1);
+
+	print entropy_test_finish(2);
+	}
diff --git a/testing/btest/bifs/gethostname.bro b/testing/btest/bifs/gethostname.bro
new file mode 100644
index 0000000000..97af719745
--- /dev/null
+++ b/testing/btest/bifs/gethostname.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = gethostname();
+	if ( |a| == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/getpid.bro b/testing/btest/bifs/getpid.bro
new file mode 100644
index 0000000000..98edc19a44
--- /dev/null
+++ b/testing/btest/bifs/getpid.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = getpid();
+	if ( a == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/global_sizes.bro b/testing/btest/bifs/global_sizes.bro
new file mode 100644
index 0000000000..4862db318b
--- /dev/null
+++ b/testing/btest/bifs/global_sizes.bro
@@ -0,0 +1,16 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = global_sizes();
+	for ( i in a )
+		{
+		# the table is quite large, so just look for one item we expect
+		if ( i == "bro_init" )
+			print "found bro_init";
+
+		}
+
+	}
diff --git a/testing/btest/bifs/identify_data.bro b/testing/btest/bifs/identify_data.bro
new file mode 100644
index 0000000000..11824b5e85
--- /dev/null
+++ b/testing/btest/bifs/identify_data.bro
@@ -0,0 +1,16 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	# plain text
+	local a = "This is a test";
+	print identify_data(a, F);
+	print identify_data(a, T);
+
+	# PNG image
+	local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a";
+	print identify_data(b, F);
+	print identify_data(b, T);
+	}
diff --git a/testing/btest/bifs/resource_usage.bro b/testing/btest/bifs/resource_usage.bro
new file mode 100644
index 0000000000..35f5b020d6
--- /dev/null
+++ b/testing/btest/bifs/resource_usage.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = resource_usage();
+	if ( a$version != bro_version() )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/strftime.bro b/testing/btest/bifs/strftime.bro
new file mode 100644
index 0000000000..31f9538632
--- /dev/null
+++ b/testing/btest/bifs/strftime.bro
@@ -0,0 +1,17 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local f1 = "%Y-%m-%d %H:%M:%S";
+	local f2 = "%H%M%S %Y%m%d";
+
+	local a = double_to_time(0);
+	print strftime(f1, a);
+	print strftime(f2, a);
+
+	a = double_to_time(123456789);
+	print strftime(f1, a);
+	print strftime(f2, a);
+	}

From 18550ab009852059ecacc98b8035fc370a5e8fee Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sat, 4 Aug 2012 22:24:44 -0700
Subject: [PATCH 124/238] small bug in test script. Still worked, because the
 internal type checking let this through...

---
 testing/btest/scripts/base/frameworks/input/predicate.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro
index 2cda6f5fb9..fcd986c9a6 100644
--- a/testing/btest/scripts/base/frameworks/input/predicate.bro
+++ b/testing/btest/scripts/base/frameworks/input/predicate.bro
@@ -35,7 +35,7 @@ type Val: record {
 	b: bool;
 };
 
-global servers: table[int] of Val = table();
+global servers: table[int] of bool = table();
 
 event bro_init()
 	{

From a2b5028b58dee3dfd2759235a65a7c829ca40555 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sat, 4 Aug 2012 22:38:26 -0700
Subject: [PATCH 125/238] fix little sneaky bug in input framework with an edge
 case.

An assertion would trigger in the case when a predicate refuses
a new entry and another entry with the same index elements was
already in the table. (I thought that code block was unreachable
... did not think of this case).
---
 src/input/Manager.cc                          |  4 +-
 .../out                                       |  3 +
 .../input/predicaterefusesecondsamerecord.bro | 56 +++++++++++++++++++
 3 files changed, 60 insertions(+), 3 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 64e54f9333..3c29f14928 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1044,9 +1044,7 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals)
 
 			if ( ! updated )
 				{
-				// throw away. Hence - we quit. And remove the entry from the current dictionary...
-				// (but why should it be in there? assert this).
-				assert ( stream->currDict->RemoveEntry(idxhash) == 0 );
+				// just quit and delete everything we created.
 				delete idxhash;
 				delete h;
 				return stream->num_val_fields + stream->num_idx_fields;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out
new file mode 100644
index 0000000000..f752ff451a
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out
@@ -0,0 +1,3 @@
+{
+[1.228.83.33] = [asn=9318 HANARO-AS Hanaro Telecom Inc., severity=medium, confidence=95, detecttime=1342569600.0]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
new file mode 100644
index 0000000000..d572b30090
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
@@ -0,0 +1,56 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+# Ok, this one tests a fun case.
+# Input file contains two lines mapping to the same index, but with different values,
+# where the predicate accepts the first one and refuses the second one.
+# Desired result -> first entry stays.
+
+@TEST-START-FILE input.log
+#fields	restriction	guid	severity	confidence	detecttime	address	protocol	portlist	asn	prefix	rir	cc	impact	description	alternativeid_restriction	alternativeid
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.0.17.227	-	-	2519 VECTANT VECTANT Ltd.	1.0.16.0/23	apnic	JP	spam infrastructure	spamming	public	http://reputation.alienvault.com/reputation.generic
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	95	1342569600	1.228.83.33	6	25	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	direct ube sources, spam operations & spam services	public	http://www.spamhaus.org/query/bl?ip=1.228.83.33
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.228.83.33	-	-	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	spamming;malware domain	public	http://reputation.alienvault.com/reputation.generic
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	address: addr;
+};
+
+type Val: record {
+	asn: string;
+	severity: string;
+	confidence: count;
+	detecttime: time;
+};
+
+global servers: table[addr] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers,
+				$pred(typ: Input::Event, left: Idx, right: Val) = { if ( right$confidence > 90 ) { return T; } return F; }
+				]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From bda8631f32d366128622fb474567573d54184d8f Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 7 Aug 2012 14:10:55 -0500
Subject: [PATCH 126/238] Add more BIF tests

---
 testing/btest/Baseline/bifs.capture_state_updates/out |  1 +
 testing/btest/Baseline/bifs.is_local_interface/out    |  4 ++++
 testing/btest/Baseline/bifs.reading_traces/out1       |  1 +
 testing/btest/Baseline/bifs.reading_traces/out2       |  1 +
 testing/btest/bifs/capture_state_updates.bro          |  9 +++++++++
 testing/btest/bifs/get_matcher_stats.bro              |  9 +++++++++
 testing/btest/bifs/is_local_interface.bro             | 11 +++++++++++
 testing/btest/bifs/reading_traces.bro                 | 10 ++++++++++
 8 files changed, 46 insertions(+)
 create mode 100644 testing/btest/Baseline/bifs.capture_state_updates/out
 create mode 100644 testing/btest/Baseline/bifs.is_local_interface/out
 create mode 100644 testing/btest/Baseline/bifs.reading_traces/out1
 create mode 100644 testing/btest/Baseline/bifs.reading_traces/out2
 create mode 100644 testing/btest/bifs/capture_state_updates.bro
 create mode 100644 testing/btest/bifs/get_matcher_stats.bro
 create mode 100644 testing/btest/bifs/is_local_interface.bro
 create mode 100644 testing/btest/bifs/reading_traces.bro

diff --git a/testing/btest/Baseline/bifs.capture_state_updates/out b/testing/btest/Baseline/bifs.capture_state_updates/out
new file mode 100644
index 0000000000..62a6e3c9df
--- /dev/null
+++ b/testing/btest/Baseline/bifs.capture_state_updates/out
@@ -0,0 +1 @@
+T
diff --git a/testing/btest/Baseline/bifs.is_local_interface/out b/testing/btest/Baseline/bifs.is_local_interface/out
new file mode 100644
index 0000000000..328bff6687
--- /dev/null
+++ b/testing/btest/Baseline/bifs.is_local_interface/out
@@ -0,0 +1,4 @@
+T
+F
+F
+T
diff --git a/testing/btest/Baseline/bifs.reading_traces/out1 b/testing/btest/Baseline/bifs.reading_traces/out1
new file mode 100644
index 0000000000..cf84443e49
--- /dev/null
+++ b/testing/btest/Baseline/bifs.reading_traces/out1
@@ -0,0 +1 @@
+F
diff --git a/testing/btest/Baseline/bifs.reading_traces/out2 b/testing/btest/Baseline/bifs.reading_traces/out2
new file mode 100644
index 0000000000..62a6e3c9df
--- /dev/null
+++ b/testing/btest/Baseline/bifs.reading_traces/out2
@@ -0,0 +1 @@
+T
diff --git a/testing/btest/bifs/capture_state_updates.bro b/testing/btest/bifs/capture_state_updates.bro
new file mode 100644
index 0000000000..3abfdffdc1
--- /dev/null
+++ b/testing/btest/bifs/capture_state_updates.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: test -f testfile
+
+event bro_init()
+	{
+	print capture_state_updates("testfile");
+	}
diff --git a/testing/btest/bifs/get_matcher_stats.bro b/testing/btest/bifs/get_matcher_stats.bro
new file mode 100644
index 0000000000..baee49fe1e
--- /dev/null
+++ b/testing/btest/bifs/get_matcher_stats.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = get_matcher_stats();
+	if ( a$matchers == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/is_local_interface.bro b/testing/btest/bifs/is_local_interface.bro
new file mode 100644
index 0000000000..8befdca385
--- /dev/null
+++ b/testing/btest/bifs/is_local_interface.bro
@@ -0,0 +1,11 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	print is_local_interface(127.0.0.1);
+	print is_local_interface(1.2.3.4);
+	print is_local_interface([2607::a:b:c:d]);
+	print is_local_interface([::1]);
+	}
diff --git a/testing/btest/bifs/reading_traces.bro b/testing/btest/bifs/reading_traces.bro
new file mode 100644
index 0000000000..fc83c50ccb
--- /dev/null
+++ b/testing/btest/bifs/reading_traces.bro
@@ -0,0 +1,10 @@
+
+# @TEST-EXEC: bro %INPUT >out1
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: bro -r $TRACES/web.trace %INPUT >out2
+# @TEST-EXEC: btest-diff out2
+
+event bro_init()
+	{
+	print reading_traces();
+	}

From 7c6b891b633c0c26298803d19b882b02f4a6f526 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 9 Aug 2012 13:46:58 -0400
Subject: [PATCH 127/238] Small improvements for printing reporter messages to
 STDERR.

---
 scripts/base/frameworks/reporter/main.bro | 27 +++++++++++++++++------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro
index 8b45819442..0248b82d10 100644
--- a/scripts/base/frameworks/reporter/main.bro
+++ b/scripts/base/frameworks/reporter/main.bro
@@ -37,15 +37,15 @@ export {
 		location: string &log &optional;
 	};
 	
-	## Send reporter error messages to STDERR by default.  The option to 
+	## Tunable for sending reporter warning messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
+
+	## Tunable for sending reporter error messages to STDERR.  The option to 
 	## turn it off is presented here in case Bro is being run by some 
 	## external harness and shouldn't output anything to the console.
 	const errors_to_stderr = T &redef;
-	
-	## Send reporter warning messages to STDERR by default.  The option to 
-	## turn it off is presented here in case Bro is being run by some 
-	## external harness and shouldn't output anything to the console.
-	const warnings_to_stderr = T &redef;
 }
 
 global stderr: file;
@@ -65,13 +65,26 @@ event reporter_info(t: time, msg: string, location: string) &priority=-5
 	
 event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( warnings_to_stderr )
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("WARNING: %s (%s)", msg, location);
+		}
+	
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
 event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
 	if ( errors_to_stderr )
-		print stderr, fmt("ERROR: %s", msg);
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("ERROR: %s (%s)", msg, location);
+		}
 	
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}

From cfe1402281eeb5fc935485f5e8c8082395820c29 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 9 Aug 2012 14:48:46 -0400
Subject: [PATCH 128/238] A couple of tests for printing reporter messages to
 STDERR.

---
 .../.stderr                                         |  0
 .../reporter.log                                    |  8 ++++++++
 .../scripts.base.frameworks.reporter.stderr/.stderr |  1 +
 .../reporter.log                                    |  8 ++++++++
 .../base/frameworks/reporter/disable-stderr.bro     | 13 +++++++++++++
 .../scripts/base/frameworks/reporter/stderr.bro     | 10 ++++++++++
 6 files changed, 40 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
 create mode 100644 testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
 create mode 100644 testing/btest/scripts/base/frameworks/reporter/stderr.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
new file mode 100644
index 0000000000..5c6e795074
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
@@ -0,0 +1,8 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr
new file mode 100644
index 0000000000..78af1e7a73
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr
@@ -0,0 +1 @@
+ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
new file mode 100644
index 0000000000..4a00bb95ad
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
@@ -0,0 +1,8 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
diff --git a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
new file mode 100644
index 0000000000..438e24d80b
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
@@ -0,0 +1,13 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+
+redef Reporter::warnings_to_stderr = F;
+redef Reporter::errors_to_stderr = F;
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}
\ No newline at end of file
diff --git a/testing/btest/scripts/base/frameworks/reporter/stderr.bro b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
new file mode 100644
index 0000000000..7ea748d94f
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
@@ -0,0 +1,10 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}
\ No newline at end of file

From 38912c182c0d6d051b3040fb1a206f102b65966e Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 10 Aug 2012 12:33:45 -0700
Subject: [PATCH 129/238] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 22120825f8..99e7a27431 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
+Subproject commit 99e7a274319619a94a421eb62537c7a5c184f71b
diff --git a/aux/bro-aux b/aux/bro-aux
index 941ee753f7..c691c01e9c 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
+Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
diff --git a/aux/broccoli b/aux/broccoli
index 5ff3e6a8e8..b3692a02ba 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
+Subproject commit b3692a02bae9a47d701d2d547e327dd429a86e76
diff --git a/aux/broctl b/aux/broctl
index 903108f6b4..84428286a1 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 903108f6b43ad228309713da880026d50add41f4
+Subproject commit 84428286a1980e21cafc4e066d95bf58f82a92b8

From d1c78d030045569fa1205e73a52aafe8483e9409 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 10 Aug 2012 13:10:24 -0700
Subject: [PATCH 130/238] Updating baselines.

---
 .../btest/Baseline/core.reporter-error-in-handler/output   | 3 ++-
 testing/btest/Baseline/core.reporter-runtime-error/output  | 2 +-
 testing/btest/Baseline/core.reporter/output                | 7 ++++---
 .../reporter.log                                           | 4 +++-
 .../scripts.base.frameworks.reporter.stderr/reporter.log   | 4 +++-
 .../scripts/base/frameworks/reporter/disable-stderr.bro    | 4 ++--
 testing/btest/scripts/base/frameworks/reporter/stderr.bro  | 4 ++--
 7 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index 83b310ab61..190631f4d1 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,2 +1,3 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+ERROR: no such index (a[1]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
+
 1st error printed on script level
diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output
index 59bcc3ac9b..94f7860cb4 100644
--- a/testing/btest/Baseline/core.reporter-runtime-error/output
+++ b/testing/btest/Baseline/core.reporter-runtime-error/output
@@ -1 +1 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+ERROR: no such index (a[2]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output
index 2735adc931..b4f89bad2f 100644
--- a/testing/btest/Baseline/core.reporter/output
+++ b/testing/btest/Baseline/core.reporter/output
@@ -1,3 +1,4 @@
-/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+WARNING: init test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
+ERROR: init test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
+WARNING: done test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
+ERROR: done test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
index 5c6e795074..144c094b2f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
+#open	2012-08-10-20-09-16
 #fields	ts	level	message	location
 #types	time	enum	string	string
-0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
+0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
+#close	2012-08-10-20-09-16
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
index 4a00bb95ad..b314bc45c3 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
+#open	2012-08-10-20-09-23
 #fields	ts	level	message	location
 #types	time	enum	string	string
-0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
+0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
+#close	2012-08-10-20-09-23
diff --git a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
index 438e24d80b..b1afb99b5c 100644
--- a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
+++ b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
@@ -1,6 +1,6 @@
 # @TEST-EXEC: bro %INPUT
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
-# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
 
 redef Reporter::warnings_to_stderr = F;
 redef Reporter::errors_to_stderr = F;
@@ -10,4 +10,4 @@ global test: table[count] of string = {};
 event bro_init()
 	{
 	print test[3];
-	}
\ No newline at end of file
+	}
diff --git a/testing/btest/scripts/base/frameworks/reporter/stderr.bro b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
index 7ea748d94f..ef01c9fdf9 100644
--- a/testing/btest/scripts/base/frameworks/reporter/stderr.bro
+++ b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
@@ -1,10 +1,10 @@
 # @TEST-EXEC: bro %INPUT
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
-# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
 
 global test: table[count] of string = {};
 
 event bro_init()
 	{
 	print test[3];
-	}
\ No newline at end of file
+	}

From eee4fbf7ad2b8855f5d6a488b5a7b83bd75dfe9b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 10 Aug 2012 13:33:57 -0700
Subject: [PATCH 131/238] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 99e7a27431..22120825f8 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 99e7a274319619a94a421eb62537c7a5c184f71b
+Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
diff --git a/aux/bro-aux b/aux/bro-aux
index c691c01e9c..941ee753f7 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
+Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
diff --git a/aux/broccoli b/aux/broccoli
index b3692a02ba..5ff3e6a8e8 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit b3692a02bae9a47d701d2d547e327dd429a86e76
+Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
diff --git a/aux/broctl b/aux/broctl
index 84428286a1..6d0eb6083a 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 84428286a1980e21cafc4e066d95bf58f82a92b8
+Subproject commit 6d0eb6083acdc77e0a912bec0fb23df79b98da63

From 205ad78369701a5e67260b421411a52b28c45440 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 14 Aug 2012 15:09:38 -0400
Subject: [PATCH 132/238] Fix some problems in logs-to-elasticsearch.bro

---
 scripts/policy/tuning/logs-to-elasticsearch.bro | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro
index 207a9acc04..2a4b70362a 100644
--- a/scripts/policy/tuning/logs-to-elasticsearch.bro
+++ b/scripts/policy/tuning/logs-to-elasticsearch.bro
@@ -8,13 +8,13 @@ export {
 
 	## Optionally ignore any :bro:type:`Log::ID` from being sent to
 	## ElasticSearch with this script.
-	const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
+	const excluded_log_ids: set[Log::ID] &redef;
 
 	## If you want to explicitly only send certain :bro:type:`Log::ID` 
 	## streams, add them to this set.  If the set remains empty, all will 
 	## be sent.  The :bro:id:`LogElasticSearch::excluded_log_ids` option will remain in 
 	## effect as well.
-	const send_logs: set[string] = set() &redef;
+	const send_logs: set[Log::ID] &redef;
 }
 
 event bro_init() &priority=-5
@@ -24,8 +24,8 @@ event bro_init() &priority=-5
 	
 	for ( stream_id in Log::active_streams )
 		{
-		if ( fmt("%s", stream_id) in excluded_log_ids ||
-		     (|send_logs| > 0 && fmt("%s", stream_id) !in send_logs) )
+		if ( stream_id in excluded_log_ids ||
+		     (|send_logs| > 0 && stream_id !in send_logs) )
 			next;
 
 		local filter: Log::Filter = [$name = "default-es",

From b13196cbf194419836f9b7627aab5cab25c47397 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 16 Aug 2012 09:24:25 -0400
Subject: [PATCH 133/238] Fixed more potential problems with deadlocked ES
 threads and signals from libcurl.

---
 src/logging/writers/ElasticSearch.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index e688686b35..cb3248a044 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -371,7 +371,11 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 	// The best (only?) way to disable that is to just use HTTP 1.0
 	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 
-	//curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
+	// Some timeout options.  These will need more attention later.
+	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout);
+	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2);
+	curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60);
 
 	CURLcode return_code = curl_easy_perform(handle);
 

From 4da209d3b1fe6fa9c5118e055752843b2fb73a45 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 16 Aug 2012 11:48:56 -0700
Subject: [PATCH 134/238] Installing a handler for running out of memory in
 "new".

Bro will now print an error message in that case rather than abort
with an uncaught exception.
---
 CHANGES     | 6 ++++++
 VERSION     | 2 +-
 src/main.cc | 4 ++++
 src/util.cc | 8 +++++++-
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 08998ab9f4..f0c73ce8d9 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,10 @@
 
+2.1-beta-21 | 2012-08-16 11:48:56 -0700
+
+  * Installing a handler for running out of memory in "new". Bro will
+    now print an error message in that case rather than abort with an
+    uncaught exception. (Robin Sommer)
+
 2.1-beta-20 | 2012-08-16 11:43:31 -0700
 
   * Fixed potential problems with ElasticSearch output plugin. (Seth
diff --git a/VERSION b/VERSION
index c42c76c8ba..5d7a2a2cce 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta-20
+2.1-beta-21
diff --git a/src/main.cc b/src/main.cc
index 407f67c9af..5999186240 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -337,6 +337,8 @@ void terminate_bro()
 	delete log_mgr;
 	delete thread_mgr;
 	delete reporter;
+
+	reporter = 0;
 	}
 
 void termination_signal()
@@ -380,6 +382,8 @@ static void bro_new_handler()
 
 int main(int argc, char** argv)
 	{
+	std::set_new_handler(bro_new_handler);
+
 	brofiler.ReadStats();
 
 	bro_argc = argc;
diff --git a/src/util.cc b/src/util.cc
index 2d981e952e..3b6fcac76f 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1383,7 +1383,13 @@ void safe_close(int fd)
 
 void out_of_memory(const char* where)
 	{
-	reporter->FatalError("out of memory in %s.\n", where);
+	fprintf(stderr, "out of memory in %s.\n", where);
+
+	if ( reporter )
+		// Guess that might fail here if memory is really tight ...
+		reporter->FatalError("out of memory in %s.\n", where);
+
+	abort();
 	}
 
 void get_memory_usage(unsigned int* total, unsigned int* malloced)

From a6f7fd9c874ffdab31c3c79c9956857617b723d5 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 16 Aug 2012 15:59:26 -0500
Subject: [PATCH 135/238] Fix memory leak of serialized IDs when compiled with
 --enable-debug.

When using --enable-debug, values keep track of the last identifier
to which they were bound by storing a ref'd ID pointer.  This could
lead to some circular dependencies in which an ID is never reclaimed
because the Val is bound to the ID and the ID is bound to the Val, with
both holding references to each other.

There might be more cases where this feature of --enable-debug caused
a leak, but it showed up in particular when running the
core.leaks.remote unit test due to the internal
SendID("peer_description") call during the handshake between remote
processes.  Other tests showed the send_id() BIF leaked more generally.

Tracking the ID last bound to a Val through just the identifier string
instead of a ref'd ID pointer fixes the leak.
---
 src/RemoteSerializer.cc |  5 -----
 src/Val.cc              |  2 +-
 src/Val.h               | 16 +++++++++-------
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc
index cfd20eba39..564ad2be68 100644
--- a/src/RemoteSerializer.cc
+++ b/src/RemoteSerializer.cc
@@ -2897,11 +2897,6 @@ void RemoteSerializer::GotID(ID* id, Val* val)
 					(desc && *desc) ? desc : "not set"),
 			current_peer);
 
-#ifdef USE_PERFTOOLS_DEBUG
-		// May still be cached, but we don't care.
-		heap_checker->IgnoreObject(id);
-#endif
-
 		Unref(id);
 		return;
 		}
diff --git a/src/Val.cc b/src/Val.cc
index 8a8c2b18c0..79fa8a0c69 100644
--- a/src/Val.cc
+++ b/src/Val.cc
@@ -64,7 +64,7 @@ Val::~Val()
 
 	Unref(type);
 #ifdef DEBUG
-	Unref(bound_id);
+	delete [] bound_id;
 #endif
 	}
 
diff --git a/src/Val.h b/src/Val.h
index 2ca18e6131..c3ec5b04fb 100644
--- a/src/Val.h
+++ b/src/Val.h
@@ -347,13 +347,15 @@ public:
 #ifdef DEBUG
 	// For debugging, we keep a reference to the global ID to which a
 	// value has been bound *last*.
-	ID* GetID() const	{ return bound_id; }
+	ID* GetID() const
+		{
+		return bound_id ? global_scope()->Lookup(bound_id) : 0;
+		}
+
 	void SetID(ID* id)
 		{
-		if ( bound_id )
-			::Unref(bound_id);
-		bound_id = id;
-		::Ref(bound_id);
+		delete [] bound_id;
+		bound_id = id ? copy_string(id->Name()) : 0;
 		}
 #endif
 
@@ -401,8 +403,8 @@ protected:
 	RecordVal* attribs;
 
 #ifdef DEBUG
-	// For debugging, we keep the ID to which a Val is bound.
-	ID* bound_id;
+	// For debugging, we keep the name of the ID to which a Val is bound.
+	const char* bound_id;
 #endif
 
 };

From 508ac1c7ba1b9fbddc128a109b51bd6376ba4bd9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 16 Aug 2012 16:33:46 -0500
Subject: [PATCH 136/238] Unit test tweaks/fixes.

- Some baselines for tests in "leaks" group were outdated.

- Changed a few of the cluster/communication tests to terminate
  more explicitly instead of relying on btest-bg-wait to kill
  processes.  This makes the tests finish faster in the success case
  and makes the reason for failing clearer in the that case.
---
 .../manager-1.metrics.log                     |  8 +++--
 .../core.leaks.remote/sender.test.failure.log |  8 +++--
 .../core.leaks.remote/sender.test.log         | 12 ++++---
 .../core.leaks.remote/sender.test.success.log |  6 ++--
 testing/btest/core/leaks/basic-cluster.bro    | 23 +++++++++++++-
 testing/btest/core/leaks/remote.bro           | 31 ++++++++++++++-----
 .../base/frameworks/logging/remote.bro        | 23 +++++++++++---
 .../base/frameworks/metrics/basic-cluster.bro | 23 +++++++++++++-
 .../metrics/cluster-intermediate-update.bro   | 17 +++++++++-
 9 files changed, 122 insertions(+), 29 deletions(-)

diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log
index 42fcd6a526..cb1bd5af01 100644
--- a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
+#open	2012-07-20-01-50-41
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1331256494.591966	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1331256494.591966	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
-1331256494.591966	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+1342749041.601712	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1342749041.601712	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1342749041.601712	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+#close	2012-07-20-01-50-49
diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log
index 5a26f322f4..71e1d18c73 100644
--- a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log
+++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log
index 9d2ba26f48..bc3dac5a1a 100644
--- a/testing/btest/Baseline/core.leaks.remote/sender.test.log
+++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log
index 1b2ed452a0..f0b26454b4 100644
--- a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log
+++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index f5b40c1104..d9d2f97b1e 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -9,7 +9,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
-# @TEST-EXEC: btest-bg-wait -k 30
+# @TEST-EXEC: btest-bg-wait 40
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -40,3 +40,24 @@ event bro_init() &priority=5
 		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
 		}
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
+		{
+		terminate_communication();
+		terminate();
+		}
+	}
+
+@endif
diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro
index f888d8f6ee..8c8dc73364 100644
--- a/testing/btest/core/leaks/remote.bro
+++ b/testing/btest/core/leaks/remote.bro
@@ -4,17 +4,19 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
-# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
-# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+# @TEST-EXEC: ( cd sender && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: ( cd receiver && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: cmp receiver/c.test.log sender/c.test.log
+# @TEST-EXEC: cmp receiver/c.test.failure.log sender/c.test.failure.log
+# @TEST-EXEC: cmp receiver/c.test.success.log sender/c.test.success.log
 
 # This is the common part loaded by both sender and receiver.
 module Test;
@@ -43,10 +45,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -68,14 +70,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
diff --git a/testing/btest/scripts/base/frameworks/logging/remote.bro b/testing/btest/scripts/base/frameworks/logging/remote.bro
index 48683148f5..ba577cc92b 100644
--- a/testing/btest/scripts/base/frameworks/logging/remote.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote.bro
@@ -1,10 +1,10 @@
 # @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender bro -b --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro -b --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
@@ -41,10 +41,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -66,14 +66,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index 09479b7a2f..4aa1afa96f 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -5,7 +5,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -36,3 +36,24 @@ event bro_init() &priority=5
 		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
 		}
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
+		{
+		terminate_communication();
+		terminate();
+		}
+	}
+
+@endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
index 654e42976a..db2c7e9f5d 100644
--- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
@@ -5,7 +5,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -37,6 +37,21 @@ event bro_init() &priority=5
 		 $log=T]);
 	}
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	terminate();
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::WORKER )
 
 event do_metrics(i: count)

From 907c92e1ccd692023ea305fa9e1acba5f4819aa9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 17 Aug 2012 15:22:51 -0500
Subject: [PATCH 137/238] Fix mime type diff canonifier to also skip mime_desc
 columns

In particular, the ftp.log baseline in the new ipv6 test in bro-testing
was failign on various platforms because of this.
---
 testing/scripts/diff-remove-mime-types | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/testing/scripts/diff-remove-mime-types b/testing/scripts/diff-remove-mime-types
index fb447a9989..b8cc3d1e6d 100755
--- a/testing/scripts/diff-remove-mime-types
+++ b/testing/scripts/diff-remove-mime-types
@@ -3,20 +3,27 @@
 # A diff canonifier that removes all MIME types because libmagic output
 # can differ between installations.
 
-BEGIN { FS="\t"; OFS="\t"; column = -1; }
+BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 }
 
 /^#fields/ {
     for ( i = 2; i < NF; ++i )
+        {
         if ( $i == "mime_type" )
-            column = i-1;
+            type_col = i-1;
+        if ( $i == "mime_desc" )
+            desc_col = i-1;
+        }
 }
 
-column >= 0 {
-    if ( $column != "-" )
+function remove_mime (n) {
+    if ( n >= 0 && $n != "-" )
         # Mark that it's set, but ignore content.
-        $column = "+";
+        $n = "+"
 }
 
+remove_mime(type_col)
+remove_mime(desc_col)
+
 {
     print;
 }

From f201a9f1a7f52329f1c8db35ab46dbfa50f0bda4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 17 Aug 2012 17:27:02 -0500
Subject: [PATCH 138/238] Fix portability of printing to files returned by
 open("/dev/stderr").

The BroFile ctor now wraps /dev/std{in,out,err} string arguments
into the actual FILE* provided by stdio.h because use of the former
directly isn't POSIX compliant and led to subtle differences that
broke unit tests on certain platforms (e.g. OS X redirection of stderr
behavior started differing from Linux).  The BroFile (un)serialization
methods already did this kind of logic, so adding it in the ctor also
should make things more consistent.

Some of the reporter-related unit tests looked like they were missing
output because of this, and the coverage test for bare-mode errors
needed tweaking to branch on whether or not libcurl was available
(since the error output differs when elasticsearch isn't there).
---
 src/File.cc                                        | 14 ++++++++++++--
 .../Baseline/core.reporter-error-in-handler/output |  4 ++--
 .../Baseline/core.reporter-runtime-error/output    |  3 ++-
 .../btest/Baseline/core.reporter/logger-test.log   | 12 ++++++------
 testing/btest/Baseline/core.reporter/output        | 11 +++++++----
 .../unique_errors_no_elasticsearch                 |  1 +
 testing/btest/coverage/bare-mode-errors.test       |  3 ++-
 7 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch

diff --git a/src/File.cc b/src/File.cc
index 20e845c09f..20ab2e1013 100644
--- a/src/File.cc
+++ b/src/File.cc
@@ -138,11 +138,21 @@ BroFile::BroFile(FILE* arg_f, const char* arg_name, const char* arg_access)
 BroFile::BroFile(const char* arg_name, const char* arg_access, BroType* arg_t)
 	{
 	Init();
-
+	f = 0;
 	name = copy_string(arg_name);
 	access = copy_string(arg_access);
 	t = arg_t ? arg_t : base_type(TYPE_STRING);
-	if ( ! Open() )
+
+	if ( streq(name, "/dev/stdin") )
+		f = stdin;
+	else if ( streq(name, "/dev/stdout") )
+		f = stdout;
+	else if ( streq(name, "/dev/stderr") )
+		f = stderr;
+
+	if ( f )
+		is_open = 1;
+	else if ( ! Open() )
 		{
 		reporter->Error("cannot open %s: %s", name, strerror(errno));
 		is_open = 0;
diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index 190631f4d1..b20b1b2292 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,3 +1,3 @@
-ERROR: no such index (a[1]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
-
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
 1st error printed on script level
diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output
index 94f7860cb4..5a03f5feb2 100644
--- a/testing/btest/Baseline/core.reporter-runtime-error/output
+++ b/testing/btest/Baseline/core.reporter-runtime-error/output
@@ -1 +1,2 @@
-ERROR: no such index (a[2]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log
index 6f7ba1d8c7..5afd904b63 100644
--- a/testing/btest/Baseline/core.reporter/logger-test.log
+++ b/testing/btest/Baseline/core.reporter/logger-test.log
@@ -1,6 +1,6 @@
-reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output
index b4f89bad2f..f2c59259c2 100644
--- a/testing/btest/Baseline/core.reporter/output
+++ b/testing/btest/Baseline/core.reporter/output
@@ -1,4 +1,7 @@
-WARNING: init test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
-ERROR: init test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
-WARNING: done test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
-ERROR: done test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
+/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
+ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
+WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
+ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch b/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch
new file mode 100644
index 0000000000..e95f88e74b
--- /dev/null
+++ b/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch
@@ -0,0 +1 @@
+error: unknown writer type requested
diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test
index 21e7d4f4a9..7084d74e83 100644
--- a/testing/btest/coverage/bare-mode-errors.test
+++ b/testing/btest/coverage/bare-mode-errors.test
@@ -10,4 +10,5 @@
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
-# @TEST-EXEC: btest-diff unique_errors
+# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
+# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

From 0dbf2f18fa679a1231f957e474a1bb1bb59e5042 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 20 Aug 2012 13:26:17 -0400
Subject: [PATCH 139/238] Add the Stream record to Log:active_streams to make
 more dynamic logging possible.

---
 scripts/base/frameworks/logging/main.bro | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro
index ccc65ddf67..bed76a1ae5 100644
--- a/scripts/base/frameworks/logging/main.bro
+++ b/scripts/base/frameworks/logging/main.bro
@@ -329,9 +329,9 @@ export {
 	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
 
 	## The streams which are currently active and not disabled.
-	## This set is not meant to be modified by users!  Only use it for
+	## This table is not meant to be modified by users!  Only use it for
 	## examining which streams are active.
-	global active_streams: set[ID] = set();
+	global active_streams: table[ID] of Stream = table();
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
@@ -417,7 +417,7 @@ function create_stream(id: ID, stream: Stream) : bool
 	if ( ! __create_stream(id, stream) )
 		return F;
 
-	add active_streams[id];
+	active_streams[id] = stream;
 
 	return add_default_filter(id);
 	}

From 434d6a84d8bb73cef7704799fcd391375bba5862 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 21 Aug 2012 08:32:42 -0700
Subject: [PATCH 140/238] Linking ES docs into logging document.

---
 CHANGES         | 4 ++++
 VERSION         | 2 +-
 doc/logging.rst | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 7b381b5c5d..b6225097db 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.1-beta-28 | 2012-08-21 08:32:42 -0700
+
+  * Linking ES docs into logging document. (Robin Sommer)
+
 2.1-beta-27 | 2012-08-20 20:06:20 -0700
 
   * Add the Stream record to Log:active_streams to make more dynamic
diff --git a/VERSION b/VERSION
index e82f524ce7..c403b714f8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta-27
+2.1-beta-28
diff --git a/doc/logging.rst b/doc/logging.rst
index cc6cb1e54d..7fb4205b9a 100644
--- a/doc/logging.rst
+++ b/doc/logging.rst
@@ -383,3 +383,4 @@ Bro supports the following output formats other than ASCII:
    :maxdepth: 1
 
    logging-dataseries
+   logging-elasticsearch

From 06b7379bc3f112faab220d59663844d449add3a8 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 21 Aug 2012 14:54:57 -0500
Subject: [PATCH 141/238] Ignore small mem leak every rotation interval for
 dataseries logs.

Not sure if more can be done to work around it, but reported to
dataseries devs here: https://github.com/dataseries/DataSeries/issues/1

The core/leaks/dataseries-rotate.bro unit test fails without this.
---
 src/logging/writers/DataSeries.cc | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc
index 7d3053e341..bc5a82ec54 100644
--- a/src/logging/writers/DataSeries.cc
+++ b/src/logging/writers/DataSeries.cc
@@ -243,8 +243,25 @@ bool DataSeries::OpenLog(string path)
 	log_file->writeExtentLibrary(log_types);
 
 	for( size_t i = 0; i < schema_list.size(); ++i )
-		extents.insert(std::make_pair(schema_list[i].field_name,
-					      GeneralField::create(log_series, schema_list[i].field_name)));
+		{
+		string fn = schema_list[i].field_name;
+		GeneralField* gf = 0;
+#ifdef USE_PERFTOOLS_DEBUG
+		{
+		// GeneralField isn't cleaning up some results of xml parsing, reported
+		// here: https://github.com/dataseries/DataSeries/issues/1
+		// Ignore for now to make leak tests pass.  There's confidence that
+		// we do clean up the GeneralField* since the ExtentSeries dtor for
+		// member log_series would trigger an assert if dynamically allocated
+		// fields aren't deleted beforehand.
+		HeapLeakChecker::Disabler disabler;
+#endif
+		gf = GeneralField::create(log_series, fn);
+#ifdef USE_PERFTOOLS_DEBUG
+		}
+#endif
+		extents.insert(std::make_pair(fn, gf));
+		}
 
 	if ( ds_extent_size < ROW_MIN )
 		{

From bb4b68946f9530b119a0144191e8e72a27896b9d Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 21 Aug 2012 15:22:54 -0500
Subject: [PATCH 142/238] Tweak to rotate-custom.bro unit test.

This one would fail intermittently in the cases where log files were
opened or closed on a different second of the time of day from each
other since the "out" baseline contains only a single "#open" and
"#close" tag (indicating all logs opened/closed on same second of time
of day).  Piping aggregated log output through the timestamp canonifier
before `uniq` makes it so "#open" and "#close" tags for different
seconds of the time of day are reduced to a single one.
---
 testing/btest/scripts/base/frameworks/logging/rotate-custom.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
index 07fc8cef7c..c0f0ef8643 100644
--- a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
+++ b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
@@ -1,7 +1,7 @@
 #
 # @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out.tmp
 # @TEST-EXEC: cat out.tmp pp.log | sort >out
-# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
+# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | $SCRIPTS/diff-remove-timestamps | uniq >>out
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
 

From cd67603f49b3e287d5244d702b62373a265ede10 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 21 Aug 2012 21:48:49 -0700
Subject: [PATCH 143/238] add testcase for input of set. Sets can be imported
 by not specifying $val in the add_table call.

This actually was already implemented, I just completely forgot about it.
---
 .../scripts.base.frameworks.input.set/out     |  7 +++
 .../scripts/base/frameworks/input/set.bro     | 46 +++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.set/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/set.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.set/out b/testing/btest/Baseline/scripts.base.frameworks.input.set/out
new file mode 100644
index 0000000000..998244cf3f
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.set/out
@@ -0,0 +1,7 @@
+{
+192.168.17.7,
+192.168.17.42,
+192.168.17.14,
+192.168.17.1,
+192.168.17.2
+}
diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro
new file mode 100644
index 0000000000..5215523ee3
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/set.bro
@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	ip
+#types	addr
+192.168.17.1
+192.168.17.2
+192.168.17.7
+192.168.17.14
+192.168.17.42
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	ip: addr;
+};
+
+global servers: set[addr] = set();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From ec224ada0679d8dcc1c7925969ba44b459145957 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 21 Aug 2012 22:17:28 -0700
Subject: [PATCH 144/238] single-line documentation addition to main input
 framework script.

---
 scripts/base/frameworks/input/main.bro | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index 55da6ae7ec..758bc94732 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -8,6 +8,7 @@ export {
 	## The default input reader used. Defaults to `READER_ASCII`.
 	const default_reader = READER_ASCII &redef;
 
+	## The default reader mode used. Defaults to `MANUAL`.
 	const default_mode = MANUAL &redef;
 
 	## TableFilter description type used for the `table` method.

From b53be217502d6bf143e61e2f5d09bd7cdd23c525 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 21 Aug 2012 23:00:04 -0700
Subject: [PATCH 145/238] add an option to the input framework that allows the
 user to chose to not die upon encountering files/functions.

I am not entirely sure if I like the approach I took for
this, it is a bit... hacky.
---
 scripts/base/frameworks/input/main.bro        |  7 ++
 src/input.bif                                 |  4 ++
 src/input/Manager.cc                          | 40 ++++++++++--
 src/input/Manager.h                           |  2 +-
 .../out                                       | 14 ++++
 .../frameworks/input/unsupported_types.bro    | 64 +++++++++++++++++++
 6 files changed, 125 insertions(+), 6 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/unsupported_types.bro

diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index 55da6ae7ec..e8aa67b23b 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -10,6 +10,13 @@ export {
 
 	const default_mode = MANUAL &redef;
 
+	## Flag that controls if the input framework accepts records
+	## that contain types that are not supported (at the moment
+	## file and function). If true, the input framework will
+	## warn in these cases, but continue. If false, it will
+	## abort. Defaults to false (abort)
+	const accept_unsupported_types = F &redef;
+
 	## TableFilter description type used for the `table` method.
 	type TableDescription: record {
 		## Common definitions for tables and events
diff --git a/src/input.bif b/src/input.bif
index f494ef3b2f..199b665fa6 100644
--- a/src/input.bif
+++ b/src/input.bif
@@ -34,6 +34,10 @@ function Input::__force_update%(id: string%) : bool
 	return new Val(res, TYPE_BOOL);
 	%}
 
+# Options for the input framework
+
+const accept_unsupported_types: bool;
+
 # Options for Ascii Reader
 
 module InputAscii;
diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 3c29f14928..4422a9814f 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -388,6 +388,8 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	FuncType* etype = event->FType()->AsFuncType();
 
+	bool allow_file_func = false;
+
 	if ( ! etype->IsEvent() )
 		{
 		reporter->Error("stream event is a function, not an event");
@@ -453,6 +455,8 @@ bool Manager::CreateEventStream(RecordVal* fval)
 			return false;
 			}
 
+		allow_file_func = BifConst::Input::accept_unsupported_types;
+
 		}
 
 	else
@@ -461,7 +465,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	vector<Field*> fieldsV; // vector, because UnrollRecordType needs it
 
-	bool status = !UnrollRecordType(&fieldsV, fields, "");
+	bool status = !UnrollRecordType(&fieldsV, fields, "", allow_file_func);
 
 	if ( status )
 		{
@@ -609,12 +613,12 @@ bool Manager::CreateTableStream(RecordVal* fval)
 
 	vector<Field*> fieldsV; // vector, because we don't know the length beforehands
 
-	bool status = !UnrollRecordType(&fieldsV, idx, "");
+	bool status = !UnrollRecordType(&fieldsV, idx, "", false);
 
 	int idxfields = fieldsV.size();
 
 	if ( val ) // if we are not a set
-		status = status || !UnrollRecordType(&fieldsV, val, "");
+		status = status || !UnrollRecordType(&fieldsV, val, "", BifConst::Input::accept_unsupported_types);
 
 	int valfields = fieldsV.size() - idxfields;
 
@@ -773,7 +777,7 @@ bool Manager::RemoveStreamContinuation(ReaderFrontend* reader)
 	}
 
 bool Manager::UnrollRecordType(vector<Field*> *fields,
-		const RecordType *rec, const string& nameprepend)
+		const RecordType *rec, const string& nameprepend, bool allow_file_func)
 	{
 
 	for ( int i = 0; i < rec->NumFields(); i++ )
@@ -781,6 +785,23 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 
 		if ( ! IsCompatibleType(rec->FieldType(i)) )
 	       		{
+
+			// if the field is a file or a function type
+			// and it is optional, we accept it nevertheless.
+			// This allows importing logfiles containing this
+			// stuff that we actually cannot read :)
+			if ( allow_file_func ) 
+				{
+				if ( ( rec->FieldType(i)->Tag() == TYPE_FILE ||
+				       rec->FieldType(i)->Tag() == TYPE_FUNC ) &&
+				     rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)
+				   ) 
+					{
+					reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag()));
+					continue;
+					}
+				}
+
 			reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag()));
 			return false;
 			}
@@ -789,7 +810,7 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 			{
 			string prep = nameprepend + rec->FieldName(i) + ".";
 
-			if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) )
+			if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) )
 				{
 				return false;
 				}
@@ -1675,6 +1696,15 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals,
 		Val* fieldVal = 0;
 		if ( request_type->FieldType(i)->Tag() == TYPE_RECORD )
 			fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position);
+		else if ( request_type->FieldType(i)->Tag() == TYPE_FILE || 
+			  request_type->FieldType(i)->Tag() == TYPE_FUNC )
+			{
+			// If those two unsupported types are encountered here, they have
+			// been let through by the type checking.
+			// That means that they are optional & the user agreed to ignore
+			// them and has been warned by reporter.
+			// Hence -> assign null to the field, done.
+			}
 		else
 			{
 			fieldVal = ValueToVal(vals[*position], request_type->FieldType(i));
diff --git a/src/input/Manager.h b/src/input/Manager.h
index 1590042183..cc81df38b7 100644
--- a/src/input/Manager.h
+++ b/src/input/Manager.h
@@ -158,7 +158,7 @@ private:
 	// Check if a record is made up of compatible types and return a list
 	// of all fields that are in the record in order. Recursively unrolls
 	// records
-	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend);
+	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend, bool allow_file_func);
 
 	// Send events
 	void SendEvent(EventHandlerPtr ev, const int numvals, ...);
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out
new file mode 100644
index 0000000000..7ef82cf368
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out
@@ -0,0 +1,14 @@
+{
+[-42] = [fi=<uninitialized>, b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
new file mode 100644
index 0000000000..7affa4065d
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
@@ -0,0 +1,64 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	fi	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	file	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+whatever	T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+redef Input::accept_unsupported_types = T;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	fi: file &optional;
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From b6bd849018aa41c910a9675bf56d08a7e11b4e29 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 22 Aug 2012 12:12:16 -0400
Subject: [PATCH 146/238] Fixed ack tracking which could overflow quickly in
 some situations.

- Problem presented itself through incorrect results in capture-loss.bro
  under odd traffic circumstances (exact circumstances unknown).

- Changed variables involved in ack tracking to all be uint64 values.
---
 src/Stats.cc           |  8 ++++----
 src/Stats.h            |  8 ++++----
 src/TCP_Reassembler.cc | 16 ++++++++--------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/Stats.cc b/src/Stats.cc
index c3035231e9..8d48c47a25 100644
--- a/src/Stats.cc
+++ b/src/Stats.cc
@@ -12,10 +12,10 @@
 
 int killed_by_inactivity = 0;
 
-uint32 tot_ack_events = 0;
-uint32 tot_ack_bytes = 0;
-uint32 tot_gap_events = 0;
-uint32 tot_gap_bytes = 0;
+uint64 tot_ack_events = 0;
+uint64 tot_ack_bytes = 0;
+uint64 tot_gap_events = 0;
+uint64 tot_gap_bytes = 0;
 
 
 class ProfileTimer : public Timer {
diff --git a/src/Stats.h b/src/Stats.h
index eeebfe2213..a11d66828a 100644
--- a/src/Stats.h
+++ b/src/Stats.h
@@ -116,10 +116,10 @@ extern SampleLogger* sample_logger;
 extern int killed_by_inactivity;
 
 // Content gap statistics.
-extern uint32 tot_ack_events;
-extern uint32 tot_ack_bytes;
-extern uint32 tot_gap_events;
-extern uint32 tot_gap_bytes;
+extern uint64 tot_ack_events;
+extern uint64 tot_ack_bytes;
+extern uint64 tot_gap_events;
+extern uint64 tot_gap_bytes;
 
 
 // A TCPStateStats object tracks the distribution of TCP states for
diff --git a/src/TCP_Reassembler.cc b/src/TCP_Reassembler.cc
index fb67dba7ee..eb2709373c 100644
--- a/src/TCP_Reassembler.cc
+++ b/src/TCP_Reassembler.cc
@@ -20,10 +20,10 @@ const bool DEBUG_tcp_connection_close = false;
 const bool DEBUG_tcp_match_undelivered = false;
 
 static double last_gap_report = 0.0;
-static uint32 last_ack_events = 0;
-static uint32 last_ack_bytes = 0;
-static uint32 last_gap_events = 0;
-static uint32 last_gap_bytes = 0;
+static uint64 last_ack_events = 0;
+static uint64 last_ack_bytes = 0;
+static uint64 last_gap_events = 0;
+static uint64 last_gap_bytes = 0;
 
 TCP_Reassembler::TCP_Reassembler(Analyzer* arg_dst_analyzer,
 				TCP_Analyzer* arg_tcp_analyzer,
@@ -513,10 +513,10 @@ void TCP_Reassembler::AckReceived(int seq)
 		if ( gap_report && gap_report_freq > 0.0 &&
 		     dt >= gap_report_freq )
 			{
-			int devents = tot_ack_events - last_ack_events;
-			int dbytes = tot_ack_bytes - last_ack_bytes;
-			int dgaps = tot_gap_events - last_gap_events;
-			int dgap_bytes = tot_gap_bytes - last_gap_bytes;
+			uint64 devents = tot_ack_events - last_ack_events;
+			uint64 dbytes = tot_ack_bytes - last_ack_bytes;
+			uint64 dgaps = tot_gap_events - last_gap_events;
+			uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes;
 
 			RecordVal* r = new RecordVal(gap_info);
 			r->Assign(0, new Val(devents, TYPE_COUNT));

From e66e9e5d321716ecee47d9ab08155b9fe2ee034a Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 22 Aug 2012 11:12:27 -0500
Subject: [PATCH 147/238] Minor tweak to coverage.bare-mode-errors unit test.

Adding trailing slash to $DIST/scripts makes the `find` work
with a symlinked 'scripts' dir.
---
 testing/btest/coverage/bare-mode-errors.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test
index 7084d74e83..635726841b 100644
--- a/testing/btest/coverage/bare-mode-errors.test
+++ b/testing/btest/coverage/bare-mode-errors.test
@@ -8,7 +8,7 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: test -d $DIST/scripts
-# @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
+# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
 # @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
 # @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

From 201c4aa43aec4371f67851294036556154664808 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Wed, 22 Aug 2012 13:25:22 -0700
Subject: [PATCH 148/238] to be sure - add a small assertion

---
 src/input/Manager.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 4422a9814f..c3176d9c33 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1704,6 +1704,9 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals,
 			// That means that they are optional & the user agreed to ignore
 			// them and has been warned by reporter.
 			// Hence -> assign null to the field, done.
+			
+			// better check that it really is optional. you never know.
+			assert(request_type->FieldDecl(i)->FindAttr(ATTR_OPTIONAL));
 			}
 		else
 			{

From 655a73bc13ff6d9cee18e98f90ad42a90b6a5b29 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 22 Aug 2012 16:46:47 -0500
Subject: [PATCH 149/238] Change to metrics/basic-cluster unit test for
 reliability.

If the metrics break interval happened to occur between first and second
worker starting up and getting connected to the cluster, the test
would fail because the second worker didn't get a chance to connect and
send data.  The test now waits for the cluster setup to complete
before workers send metrics data.
---
 testing/btest/core/leaks/basic-cluster.bro    | 43 +++++++++++++------
 .../base/frameworks/metrics/basic-cluster.bro | 39 ++++++++++++-----
 2 files changed, 60 insertions(+), 22 deletions(-)

diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index d9d2f97b1e..7fb176b8db 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -1,21 +1,21 @@
 # Needs perftools support.
 #
 # @TEST-GROUP: leaks
-
+#
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
-
+#
 # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1   bro -m %INPUT
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
-# @TEST-EXEC: btest-bg-wait 40
+# @TEST-EXEC: btest-bg-wait 60
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -32,13 +32,6 @@ event bro_init() &priority=5
 	Metrics::add_filter(TEST_METRIC, 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
-		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
-		}
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -46,9 +39,25 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 global n = 0;
+global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
@@ -60,4 +69,14 @@ event Metrics::log_metrics(rec: Metrics::Info)
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
 @endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index 4aa1afa96f..89ae5bf79f 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -5,13 +5,13 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -28,13 +28,6 @@ event bro_init() &priority=5
 	Metrics::add_filter(TEST_METRIC, 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
-		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
-		}
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -42,9 +35,25 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 global n = 0;
+global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
@@ -56,4 +65,14 @@ event Metrics::log_metrics(rec: Metrics::Info)
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
 @endif

From 93744c8d9b22888269f466f116559f90f96638d4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 22 Aug 2012 16:54:00 -0500
Subject: [PATCH 150/238] Add test serialization to "leak" unit tests that use
 communication.

---
 testing/btest/core/leaks/basic-cluster.bro | 1 +
 testing/btest/core/leaks/remote.bro        | 1 +
 2 files changed, 2 insertions(+)

diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index 7fb176b8db..319368bc6e 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -1,5 +1,6 @@
 # Needs perftools support.
 #
+# @TEST-SERIALIZE: comm
 # @TEST-GROUP: leaks
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro
index 8c8dc73364..41bbaec076 100644
--- a/testing/btest/core/leaks/remote.bro
+++ b/testing/btest/core/leaks/remote.bro
@@ -1,5 +1,6 @@
 # Needs perftools support.
 #
+# @TEST-SERIALIZE: comm
 # @TEST-GROUP: leaks
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks

From 95d7055373763787628936431e82ea6562f4d7ba Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 22 Aug 2012 16:17:27 -0700
Subject: [PATCH 151/238] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 22120825f8..a93ef13735 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
+Subproject commit a93ef1373512c661ffcd0d0a61bd19b96667e0d5
diff --git a/aux/bro-aux b/aux/bro-aux
index 941ee753f7..4bc1a6f6a8 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
+Subproject commit 4bc1a6f6a8816dfacd8288fcf182ba35520e589b
diff --git a/aux/broccoli b/aux/broccoli
index 5ff3e6a8e8..ebfa4de45a 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
+Subproject commit ebfa4de45a839e58aec200e7e4bad33eaab4f1ed
diff --git a/aux/broctl b/aux/broctl
index 6d0eb6083a..5b3f9e5906 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 6d0eb6083acdc77e0a912bec0fb23df79b98da63
+Subproject commit 5b3f9e5906c90b76c5aa1626e112d4c991cb3fd8

From 25ef0a89e752aec2b1506363ffdeea738d1e3f1b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 22 Aug 2012 18:15:55 -0700
Subject: [PATCH 152/238] Updating NEWS.

---
 NEWS | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 949b51d832..d7018575d3 100644
--- a/NEWS
+++ b/NEWS
@@ -7,8 +7,8 @@ release. For a complete list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
 their own CHANGES.)
 
-Bro 2.1 Beta
-------------
+Bro 2.1
+-------
 
 New Functionality
 ~~~~~~~~~~~~~~~~~
@@ -161,6 +161,7 @@ the full set.
 - The ASCII writers "header_*" options have been renamed to "meta_*"
   (because there's now also a footer).
 
+
 Bro 2.0
 -------
 

From bef0ce1c98bc2dfc0e2dddef821878b7eb91f4b7 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 11:52:39 -0500
Subject: [PATCH 153/238] Add type checking for signature 'eval' condition
 functions.

Otherwise functions could be called with a mismatching argument list
and cause a crash at run-time.  The incorrect function type is now
reported at parse-time.
---
 src/RuleCondition.cc                          | 17 ++++++++++++++
 .../signatures.bad-eval-condition/.stderr     |  2 ++
 .../signatures.eval-condition/conn.log        | 14 ++++++++++++
 .../output                                    |  0
 testing/btest/btest.cfg                       |  2 +-
 .../btest/signatures/bad-eval-condition.bro   | 22 +++++++++++++++++++
 testing/btest/signatures/eval-condition.bro   | 20 +++++++++++++++++
 .../btest/{core => signatures}/load-sigs.bro  |  0
 8 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/signatures.bad-eval-condition/.stderr
 create mode 100644 testing/btest/Baseline/signatures.eval-condition/conn.log
 rename testing/btest/Baseline/{core.load-sigs => signatures.load-sigs}/output (100%)
 create mode 100644 testing/btest/signatures/bad-eval-condition.bro
 create mode 100644 testing/btest/signatures/eval-condition.bro
 rename testing/btest/{core => signatures}/load-sigs.bro (100%)

diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc
index 8852747cc4..3e64f9ffca 100644
--- a/src/RuleCondition.cc
+++ b/src/RuleCondition.cc
@@ -126,6 +126,23 @@ RuleConditionEval::RuleConditionEval(const char* func)
 		rules_error("unknown identifier", func);
 		return;
 		}
+
+	if ( id->Type()->Tag() == TYPE_FUNC )
+		{
+		// validate argument quantity and type
+		FuncType* f = id->Type()->AsFuncType();
+
+		if ( f->YieldType()->Tag() != TYPE_BOOL )
+			rules_error("eval function type must yield a 'bool'", func);
+
+		TypeList tl;
+		tl.Append(internal_type("signature_state")->Ref());
+		tl.Append(base_type(TYPE_STRING));
+
+		if ( ! f->CheckArgs(tl.Types()) )
+			rules_error("eval function parameters must be a 'signature_state' "
+			            "and a 'string' type", func);
+		}
 	}
 
 bool RuleConditionEval::DoMatch(Rule* rule, RuleEndpointState* state,
diff --git a/testing/btest/Baseline/signatures.bad-eval-condition/.stderr b/testing/btest/Baseline/signatures.bad-eval-condition/.stderr
new file mode 100644
index 0000000000..c4de35ffe9
--- /dev/null
+++ b/testing/btest/Baseline/signatures.bad-eval-condition/.stderr
@@ -0,0 +1,2 @@
+error: Error in signature (./blah.sig:6): eval function parameters must be a 'signature_state' and a 'string' type (mark_conn)
+
diff --git a/testing/btest/Baseline/signatures.eval-condition/conn.log b/testing/btest/Baseline/signatures.eval-condition/conn.log
new file mode 100644
index 0000000000..a803f74320
--- /dev/null
+++ b/testing/btest/Baseline/signatures.eval-condition/conn.log
@@ -0,0 +1,14 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2012-08-23-16-41-23
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562	(empty)
+1329843179.871641	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	tcp	ftp-data	0.111218	0	77	SF	-	0	ShAdfFa	4	216	4	297	(empty)
+1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164	(empty)
+1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164	(empty)
+1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp,blah	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458	(empty)
+#close	2012-08-23-16-41-23
diff --git a/testing/btest/Baseline/core.load-sigs/output b/testing/btest/Baseline/signatures.load-sigs/output
similarity index 100%
rename from testing/btest/Baseline/core.load-sigs/output
rename to testing/btest/Baseline/signatures.load-sigs/output
diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg
index 4c4074ee24..d86b45d8a9 100644
--- a/testing/btest/btest.cfg
+++ b/testing/btest/btest.cfg
@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts istate coverage
+TestDirs    = doc bifs language core scripts istate coverage signatures
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
diff --git a/testing/btest/signatures/bad-eval-condition.bro b/testing/btest/signatures/bad-eval-condition.bro
new file mode 100644
index 0000000000..34997b1124
--- /dev/null
+++ b/testing/btest/signatures/bad-eval-condition.bro
@@ -0,0 +1,22 @@
+# @TEST-EXEC-FAIL: bro -r $TRACES/ftp-ipv4.trace %INPUT
+# @TEST-EXEC: btest-diff .stderr
+
+@load-sigs blah.sig
+
+@TEST-START-FILE blah.sig
+signature blah
+	{
+	ip-proto == tcp
+	src-port == 21
+	payload /.*/
+	eval mark_conn
+	}
+@TEST-END-FILE
+
+# wrong function signature for use with signature 'eval' conditions
+# needs to be reported
+function mark_conn(state: signature_state): bool
+	{
+	add state$conn$service["blah"];
+	return T;
+	}
diff --git a/testing/btest/signatures/eval-condition.bro b/testing/btest/signatures/eval-condition.bro
new file mode 100644
index 0000000000..f3f1171da6
--- /dev/null
+++ b/testing/btest/signatures/eval-condition.bro
@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace %INPUT
+# @TEST-EXEC: btest-diff conn.log
+
+@load-sigs blah.sig
+
+@TEST-START-FILE blah.sig
+signature blah
+	{
+	ip-proto == tcp
+	src-port == 21
+	payload /.*/
+	eval mark_conn
+	}
+@TEST-END-FILE
+
+function mark_conn(state: signature_state, data: string): bool
+	{
+	add state$conn$service["blah"];
+	return T;
+	}
diff --git a/testing/btest/core/load-sigs.bro b/testing/btest/signatures/load-sigs.bro
similarity index 100%
rename from testing/btest/core/load-sigs.bro
rename to testing/btest/signatures/load-sigs.bro

From ff60b0bb4bf9a1d6da38bd273b0ec34eb2f37f60 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 11:59:51 -0500
Subject: [PATCH 154/238] Remove orphaned unit tests.

Looks like they're maybe from 1.5 and not applicable/updateable.
---
 testing/btest/Baseline/analyzers.conn-size-cc/conn.log | 5 -----
 testing/btest/Baseline/analyzers.conn-size/conn.log    | 5 -----
 testing/btest/analyzers/conn-size-cc.bro               | 2 --
 testing/btest/analyzers/conn-size.bro                  | 2 --
 4 files changed, 14 deletions(-)
 delete mode 100644 testing/btest/Baseline/analyzers.conn-size-cc/conn.log
 delete mode 100644 testing/btest/Baseline/analyzers.conn-size/conn.log
 delete mode 100644 testing/btest/analyzers/conn-size-cc.bro
 delete mode 100644 testing/btest/analyzers/conn-size.bro

diff --git a/testing/btest/Baseline/analyzers.conn-size-cc/conn.log b/testing/btest/Baseline/analyzers.conn-size-cc/conn.log
deleted file mode 100644
index 2f703cbcd6..0000000000
--- a/testing/btest/Baseline/analyzers.conn-size-cc/conn.log
+++ /dev/null
@@ -1,5 +0,0 @@
-1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0 cc=1
-1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0
-1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 677 5 713
-1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0
-1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0
diff --git a/testing/btest/Baseline/analyzers.conn-size/conn.log b/testing/btest/Baseline/analyzers.conn-size/conn.log
deleted file mode 100644
index 8129bc37f8..0000000000
--- a/testing/btest/Baseline/analyzers.conn-size/conn.log
+++ /dev/null
@@ -1,5 +0,0 @@
-1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0
-1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0
-1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 697 5 713
-1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0
-1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0
diff --git a/testing/btest/analyzers/conn-size-cc.bro b/testing/btest/analyzers/conn-size-cc.bro
deleted file mode 100644
index 0ba7977cf5..0000000000
--- a/testing/btest/analyzers/conn-size-cc.bro
+++ /dev/null
@@ -1,2 +0,0 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
-# @TEST-EXEC: btest-diff conn.log
diff --git a/testing/btest/analyzers/conn-size.bro b/testing/btest/analyzers/conn-size.bro
deleted file mode 100644
index 0ba7977cf5..0000000000
--- a/testing/btest/analyzers/conn-size.bro
+++ /dev/null
@@ -1,2 +0,0 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
-# @TEST-EXEC: btest-diff conn.log

From 558ca2867c873073d30522073049a35f5cc52111 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 12:29:42 -0500
Subject: [PATCH 155/238] Doc fixes for signature 'eval' conditions.

---
 doc/signatures.rst | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/doc/signatures.rst b/doc/signatures.rst
index f65215eceb..36099ba40f 100644
--- a/doc/signatures.rst
+++ b/doc/signatures.rst
@@ -229,20 +229,10 @@ matched. The following context conditions are defined:
     confirming the match. If false is returned, no signature match is
     going to be triggered. The function has to be of type ``function
     cond(state: signature_state, data: string): bool``. Here,
-    ``content`` may contain the most recent content chunk available at
+    ``data`` may contain the most recent content chunk available at
     the time the signature was matched. If no such chunk is available,
-    ``content`` will be the empty string. ``signature_state`` is
-    defined as follows:
-
-    .. code:: bro
-
-        type signature_state: record {
-            id: string;          # ID of the signature
-            conn: connection;    # Current connection
-            is_orig: bool;       # True if current endpoint is originator
-            payload_size: count; # Payload size of the first packet
-            };
-
+    ``data`` will be the empty string. See :bro:type:`signature_state`
+    for its definition.
 
 ``payload-size <cmp> <integer>``
     Compares the integer to the size of the payload of a packet. For

From 5f40e153a87b37e7621809a38545504b696202a0 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 23 Aug 2012 13:55:04 -0400
Subject: [PATCH 156/238] Adding an identifier to the SMTP blocklist notices
 for duplicate suppression.

 - Slight addition and revision to inline docs.
---
 scripts/policy/protocols/smtp/blocklists.bro | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/policy/protocols/smtp/blocklists.bro b/scripts/policy/protocols/smtp/blocklists.bro
index a3e75318bb..b1fb0e498d 100644
--- a/scripts/policy/protocols/smtp/blocklists.bro
+++ b/scripts/policy/protocols/smtp/blocklists.bro
@@ -1,3 +1,4 @@
+##! Watch for various SPAM blocklist URLs in SMTP error messages.  
 
 @load base/protocols/smtp
 
@@ -5,9 +6,11 @@ module SMTP;
 
 export {
 	redef enum Notice::Type += { 
-		## Indicates that the server sent a reply mentioning an SMTP block list.
+		## An SMTP server sent a reply mentioning an SMTP block list.
 		Blocklist_Error_Message,
-		## Indicates the client's address is seen in the block list error message.
+		## The originator's address is seen in the block list error message.
+		## This is useful to detect local hosts sending SPAM with a high
+		## positive rate.
 		Blocklist_Blocked_Host,
 	};
 
@@ -52,7 +55,8 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 				message = fmt("%s is on an SMTP block list", c$id$orig_h);
 				}
 			
-			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
+			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg, 
+			        $identifier=cat(c$id$orig_h)]);
 			}
 		}
 	}

From c1c9c9e34af571c5f204b4608b849823922c228f Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 13:04:18 -0500
Subject: [PATCH 157/238] Update documentation for builtin types

Add missing description of interval "msec" unit.
Improved description of pattern by clarifying the issue of operand
order and difference between exact and embedded matching.
---
 doc/scripts/builtins.rst | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst
index 32908f71fd..0501067409 100644
--- a/doc/scripts/builtins.rst
+++ b/doc/scripts/builtins.rst
@@ -55,8 +55,8 @@ The Bro scripting language supports the following built-in types.
 
     A temporal type representing a relative time.  An ``interval``
     constant can be written as a numeric constant followed by a time
-    unit where the time unit is one of ``usec``, ``sec``, ``min``,
-    ``hr``, or ``day`` which respectively represent microseconds,
+    unit where the time unit is one of ``usec``, ``msec``, ``sec``, ``min``,
+    ``hr``, or ``day`` which respectively represent microseconds, milliseconds,
     seconds, minutes, hours, and days.  Whitespace between the numeric
     constant and time unit is optional.  Appending the letter "s" to the
     time unit in order to pluralize it is also optional (to no semantic
@@ -95,14 +95,14 @@ The Bro scripting language supports the following built-in types.
     and embedded.
 
     In exact matching the ``==`` equality relational operator is used
-    with one :bro:type:`string` operand and one :bro:type:`pattern`
-    operand to check whether the full string exactly matches the
-    pattern.  In this case, the ``^`` beginning-of-line and ``$``
-    end-of-line anchors are redundant since pattern is implicitly
-    anchored to the beginning and end of the line to facilitate an exact
-    match.  For example::
+    with one :bro:type:`pattern` operand and one :bro:type:`string`
+    operand (order of operands does not matter) to check whether the full
+    string exactly matches the pattern.  In exact matching, the ``^``
+    beginning-of-line and ``$`` end-of-line anchors are redundant since
+    the pattern is implicitly anchored to the beginning and end of the
+    line to facilitate an exact match.  For example::
 
-        "foo" == /foo|bar/
+        /foo|bar/ == "foo"
 
     yields true, while::
 
@@ -110,9 +110,9 @@ The Bro scripting language supports the following built-in types.
 
     yields false.  The ``!=`` operator would yield the negation of ``==``.
 
-    In embedded matching the ``in`` operator is again used with one
-    :bro:type:`string` operand and one :bro:type:`pattern` operand
-    (which must be on the left-hand side), but tests whether the pattern
+    In embedded matching the ``in`` operator is used with one
+    :bro:type:`pattern` operand (which must be on the left-hand side) and
+    one :bro:type:`string` operand, but tests whether the pattern
     appears anywhere within the given string.  For example::
 
         /foo|bar/ in "foobar"

From 90281a2423230671c8a022cac4dcd509aeb233cd Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 24 Aug 2012 11:32:49 -0500
Subject: [PATCH 158/238] Add tests of the Bro scripting language

Added tests of all built-in Bro data types (including different
representations of constant values, and max./min. values), keywords, and
operators (including special properties of certain operators, such as
short-circuit evaluation and associativity).
---
 testing/btest/Baseline/language.addr/out      |  13 ++
 testing/btest/Baseline/language.bool/out      |   7 +
 testing/btest/Baseline/language.count/out     |  16 +++
 testing/btest/Baseline/language.double/out    |  25 ++++
 testing/btest/Baseline/language.enum/out      |   4 +
 testing/btest/Baseline/language.event/out     |   4 +
 testing/btest/Baseline/language.file/out1     |   2 +
 testing/btest/Baseline/language.file/out2     |   1 +
 testing/btest/Baseline/language.for/out       |   3 +
 testing/btest/Baseline/language.function/out  |  11 ++
 testing/btest/Baseline/language.if/out        |  12 ++
 testing/btest/Baseline/language.int/out       |  21 +++
 testing/btest/Baseline/language.interval/out  |  23 +++
 .../Baseline/language.null-statement/out      |   1 +
 testing/btest/Baseline/language.pattern/out   |   6 +
 testing/btest/Baseline/language.port/out      |   8 ++
 testing/btest/Baseline/language.set/out       |  36 +++++
 .../btest/Baseline/language.short-circuit/out |   4 +
 testing/btest/Baseline/language.string/out    |  24 ++++
 testing/btest/Baseline/language.subnet/out    |  10 ++
 testing/btest/Baseline/language.table/out     |  37 +++++
 testing/btest/Baseline/language.time/out      |   7 +
 testing/btest/Baseline/language.timeout/out   |   1 +
 testing/btest/Baseline/language.vector/out    |  31 +++++
 testing/btest/Baseline/language.when/out      |   2 +
 testing/btest/language/addr.bro               |  46 ++++++
 testing/btest/language/bool.bro               |  28 ++++
 testing/btest/language/count.bro              |  42 ++++++
 testing/btest/language/double.bro             |  66 +++++++++
 testing/btest/language/enum.bro               |  32 +++++
 testing/btest/language/event.bro              |  49 +++++++
 testing/btest/language/file.bro               |  19 +++
 testing/btest/language/for.bro                |  44 ++++++
 testing/btest/language/function.bro           |  73 ++++++++++
 testing/btest/language/if.bro                 |  71 ++++++++++
 testing/btest/language/int.bro                |  54 ++++++++
 testing/btest/language/interval.bro           |  77 ++++++++++
 testing/btest/language/null-statement.bro     |  34 +++++
 testing/btest/language/pattern.bro            |  28 ++++
 testing/btest/language/port.bro               |  35 +++++
 testing/btest/language/set.bro                | 121 ++++++++++++++++
 testing/btest/language/short-circuit.bro      |  48 +++++++
 testing/btest/language/string.bro             |  59 ++++++++
 testing/btest/language/subnet.bro             |  48 +++++++
 testing/btest/language/table.bro              | 131 ++++++++++++++++++
 testing/btest/language/time.bro               |  28 ++++
 testing/btest/language/timeout.bro            |  19 +++
 testing/btest/language/vector.bro             | 104 ++++++++++++++
 testing/btest/language/when.bro               |  15 ++
 49 files changed, 1580 insertions(+)
 create mode 100644 testing/btest/Baseline/language.addr/out
 create mode 100644 testing/btest/Baseline/language.bool/out
 create mode 100644 testing/btest/Baseline/language.count/out
 create mode 100644 testing/btest/Baseline/language.double/out
 create mode 100644 testing/btest/Baseline/language.enum/out
 create mode 100644 testing/btest/Baseline/language.event/out
 create mode 100644 testing/btest/Baseline/language.file/out1
 create mode 100644 testing/btest/Baseline/language.file/out2
 create mode 100644 testing/btest/Baseline/language.for/out
 create mode 100644 testing/btest/Baseline/language.function/out
 create mode 100644 testing/btest/Baseline/language.if/out
 create mode 100644 testing/btest/Baseline/language.int/out
 create mode 100644 testing/btest/Baseline/language.interval/out
 create mode 100644 testing/btest/Baseline/language.null-statement/out
 create mode 100644 testing/btest/Baseline/language.pattern/out
 create mode 100644 testing/btest/Baseline/language.port/out
 create mode 100644 testing/btest/Baseline/language.set/out
 create mode 100644 testing/btest/Baseline/language.short-circuit/out
 create mode 100644 testing/btest/Baseline/language.string/out
 create mode 100644 testing/btest/Baseline/language.subnet/out
 create mode 100644 testing/btest/Baseline/language.table/out
 create mode 100644 testing/btest/Baseline/language.time/out
 create mode 100644 testing/btest/Baseline/language.timeout/out
 create mode 100644 testing/btest/Baseline/language.vector/out
 create mode 100644 testing/btest/Baseline/language.when/out
 create mode 100644 testing/btest/language/addr.bro
 create mode 100644 testing/btest/language/bool.bro
 create mode 100644 testing/btest/language/count.bro
 create mode 100644 testing/btest/language/double.bro
 create mode 100644 testing/btest/language/enum.bro
 create mode 100644 testing/btest/language/event.bro
 create mode 100644 testing/btest/language/file.bro
 create mode 100644 testing/btest/language/for.bro
 create mode 100644 testing/btest/language/function.bro
 create mode 100644 testing/btest/language/if.bro
 create mode 100644 testing/btest/language/int.bro
 create mode 100644 testing/btest/language/interval.bro
 create mode 100644 testing/btest/language/null-statement.bro
 create mode 100644 testing/btest/language/pattern.bro
 create mode 100644 testing/btest/language/port.bro
 create mode 100644 testing/btest/language/set.bro
 create mode 100644 testing/btest/language/short-circuit.bro
 create mode 100644 testing/btest/language/string.bro
 create mode 100644 testing/btest/language/subnet.bro
 create mode 100644 testing/btest/language/table.bro
 create mode 100644 testing/btest/language/time.bro
 create mode 100644 testing/btest/language/timeout.bro
 create mode 100644 testing/btest/language/vector.bro
 create mode 100644 testing/btest/language/when.bro

diff --git a/testing/btest/Baseline/language.addr/out b/testing/btest/Baseline/language.addr/out
new file mode 100644
index 0000000000..79a88d6dcb
--- /dev/null
+++ b/testing/btest/Baseline/language.addr/out
@@ -0,0 +1,13 @@
+IPv4 address inequality (PASS)
+IPv4 address equality (PASS)
+IPv4 address comparison (PASS)
+IPv4 address comparison (PASS)
+size of IPv4 address (PASS)
+IPv6 address inequality (PASS)
+IPv6 address equality (PASS)
+IPv6 address equality (PASS)
+IPv6 address comparison (PASS)
+IPv6 address comparison (PASS)
+IPv6 address not case-sensitive (PASS)
+size of IPv6 address (PASS)
+IPv4 and IPv6 address inequality (PASS)
diff --git a/testing/btest/Baseline/language.bool/out b/testing/btest/Baseline/language.bool/out
new file mode 100644
index 0000000000..177c6795ef
--- /dev/null
+++ b/testing/btest/Baseline/language.bool/out
@@ -0,0 +1,7 @@
+equality operator (PASS)
+inequality operator (PASS)
+logical or operator (PASS)
+logical and operator (PASS)
+negation operator (PASS)
+absolute value (PASS)
+absolute value (PASS)
diff --git a/testing/btest/Baseline/language.count/out b/testing/btest/Baseline/language.count/out
new file mode 100644
index 0000000000..7dba9ea24c
--- /dev/null
+++ b/testing/btest/Baseline/language.count/out
@@ -0,0 +1,16 @@
+inequality operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+hexadecimal (PASS)
+counter alias (PASS)
+absolute value (PASS)
+absolute value (PASS)
+pre-increment operator (PASS)
+pre-decrement operator (PASS)
+modulus operator (PASS)
+division operator (PASS)
+assignment operator (PASS)
+assignment operator (PASS)
+max count value = 4294967295 (PASS)
diff --git a/testing/btest/Baseline/language.double/out b/testing/btest/Baseline/language.double/out
new file mode 100644
index 0000000000..01e3047743
--- /dev/null
+++ b/testing/btest/Baseline/language.double/out
@@ -0,0 +1,25 @@
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+double representations (PASS)
+inequality operator (PASS)
+absolute value (PASS)
+assignment operator (PASS)
+assignment operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+division operator (PASS)
+max double value = 1.7e+308 (PASS)
diff --git a/testing/btest/Baseline/language.enum/out b/testing/btest/Baseline/language.enum/out
new file mode 100644
index 0000000000..1bafdd73b0
--- /dev/null
+++ b/testing/btest/Baseline/language.enum/out
@@ -0,0 +1,4 @@
+enum equality comparison (PASS)
+enum equality comparison (PASS)
+enum equality comparison (PASS)
+type inference (PASS)
diff --git a/testing/btest/Baseline/language.event/out b/testing/btest/Baseline/language.event/out
new file mode 100644
index 0000000000..d5a22b3745
--- /dev/null
+++ b/testing/btest/Baseline/language.event/out
@@ -0,0 +1,4 @@
+event statement
+event part1
+event part2
+schedule statement
diff --git a/testing/btest/Baseline/language.file/out1 b/testing/btest/Baseline/language.file/out1
new file mode 100644
index 0000000000..5ff4194027
--- /dev/null
+++ b/testing/btest/Baseline/language.file/out1
@@ -0,0 +1,2 @@
+20
+12
diff --git a/testing/btest/Baseline/language.file/out2 b/testing/btest/Baseline/language.file/out2
new file mode 100644
index 0000000000..12be2d6723
--- /dev/null
+++ b/testing/btest/Baseline/language.file/out2
@@ -0,0 +1 @@
+test, 123, 456
diff --git a/testing/btest/Baseline/language.for/out b/testing/btest/Baseline/language.for/out
new file mode 100644
index 0000000000..dccc00ce3e
--- /dev/null
+++ b/testing/btest/Baseline/language.for/out
@@ -0,0 +1,3 @@
+for loop (PASS)
+for loop with break (PASS)
+for loop with next (PASS)
diff --git a/testing/btest/Baseline/language.function/out b/testing/btest/Baseline/language.function/out
new file mode 100644
index 0000000000..f530024370
--- /dev/null
+++ b/testing/btest/Baseline/language.function/out
@@ -0,0 +1,11 @@
+no args without return value (PASS)
+no args no return value, empty return (PASS)
+no args with return value (PASS)
+args without return value (PASS)
+args with return value (PASS)
+multiple args with return value (PASS)
+anonymous function without args or return value (PASS)
+anonymous function with return value (PASS)
+anonymous function with args and return value (PASS)
+assign function variable (PASS)
+reassign function variable (PASS)
diff --git a/testing/btest/Baseline/language.if/out b/testing/btest/Baseline/language.if/out
new file mode 100644
index 0000000000..510b66b0cf
--- /dev/null
+++ b/testing/btest/Baseline/language.if/out
@@ -0,0 +1,12 @@
+if T (PASS)
+if T else (PASS)
+if F else (PASS)
+if T else if F (PASS)
+if F else if T (PASS)
+if T else if T (PASS)
+if T else if F else (PASS)
+if F else if T else (PASS)
+if T else if T else (PASS)
+if F else if F else (PASS)
+if F else if F else if T else (PASS)
+if F else if F else if F else (PASS)
diff --git a/testing/btest/Baseline/language.int/out b/testing/btest/Baseline/language.int/out
new file mode 100644
index 0000000000..a50887999a
--- /dev/null
+++ b/testing/btest/Baseline/language.int/out
@@ -0,0 +1,21 @@
+optional '+' sign (PASS)
+negative vs. positive (PASS)
+negative vs. positive (PASS)
+hexadecimal (PASS)
+hexadecimal (PASS)
+hexadecimal (PASS)
+relational operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+relational operator (PASS)
+absolute value (PASS)
+absolute value (PASS)
+pre-increment operator (PASS)
+pre-decrement operator (PASS)
+modulus operator (PASS)
+division operator (PASS)
+assignment operator (PASS)
+assignment operator (PASS)
+max int value = 4294967295 (PASS)
+min int value = -4294967295 (PASS)
+type inference (PASS)
diff --git a/testing/btest/Baseline/language.interval/out b/testing/btest/Baseline/language.interval/out
new file mode 100644
index 0000000000..3eb135de52
--- /dev/null
+++ b/testing/btest/Baseline/language.interval/out
@@ -0,0 +1,23 @@
+optional space (PASS)
+different units with same numeric value (PASS)
+plural/singular interval are same (PASS)
+compare different time units (PASS)
+compare different time units (PASS)
+compare different time units (PASS)
+compare different time units (PASS)
+compare different time units (PASS)
+compare different time units (PASS)
+compare different time units (PASS)
+add different time units (PASS)
+subtract different time units (PASS)
+absolute value (PASS)
+absolute value (PASS)
+assignment operator (PASS)
+multiplication operator (PASS)
+division operator (PASS)
+division operator (PASS)
+relative size of units (PASS)
+relative size of units (PASS)
+relative size of units (PASS)
+relative size of units (PASS)
+relative size of units (PASS)
diff --git a/testing/btest/Baseline/language.null-statement/out b/testing/btest/Baseline/language.null-statement/out
new file mode 100644
index 0000000000..19f86f493a
--- /dev/null
+++ b/testing/btest/Baseline/language.null-statement/out
@@ -0,0 +1 @@
+done
diff --git a/testing/btest/Baseline/language.pattern/out b/testing/btest/Baseline/language.pattern/out
new file mode 100644
index 0000000000..5a31e4eacb
--- /dev/null
+++ b/testing/btest/Baseline/language.pattern/out
@@ -0,0 +1,6 @@
+equality operator (PASS)
+equality operator (order of operands) (PASS)
+inequality operator (PASS)
+in operator (PASS)
+in operator (PASS)
+!in operator (PASS)
diff --git a/testing/btest/Baseline/language.port/out b/testing/btest/Baseline/language.port/out
new file mode 100644
index 0000000000..9dd7ba03c2
--- /dev/null
+++ b/testing/btest/Baseline/language.port/out
@@ -0,0 +1,8 @@
+protocol ordering (PASS)
+protocol ordering (PASS)
+protocol ordering (PASS)
+protocol ordering (PASS)
+protocol ordering (PASS)
+different protocol but same numeric value (PASS)
+different protocol but same numeric value (PASS)
+equality operator (PASS)
diff --git a/testing/btest/Baseline/language.set/out b/testing/btest/Baseline/language.set/out
new file mode 100644
index 0000000000..b4801ac799
--- /dev/null
+++ b/testing/btest/Baseline/language.set/out
@@ -0,0 +1,36 @@
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+iterate over set (PASS)
+iterate over set (PASS)
+iterate over set (PASS)
+iterate over set (PASS)
+add element (PASS)
+in operator (PASS)
+add element (PASS)
+add element (PASS)
+in operator (PASS)
+in operator (PASS)
+add element (PASS)
+in operator (PASS)
+add element (PASS)
+in operator (PASS)
+add element (PASS)
+in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
diff --git a/testing/btest/Baseline/language.short-circuit/out b/testing/btest/Baseline/language.short-circuit/out
new file mode 100644
index 0000000000..c92995ea7c
--- /dev/null
+++ b/testing/btest/Baseline/language.short-circuit/out
@@ -0,0 +1,4 @@
+&& operator (eval. both operands) (PASS)
+&& operator (eval. 1st operand) (PASS)
+|| operator (eval. 1st operand) (PASS)
+|| operator (eval. both operands) (PASS)
diff --git a/testing/btest/Baseline/language.string/out b/testing/btest/Baseline/language.string/out
new file mode 100644
index 0000000000..623d1cd3ba
--- /dev/null
+++ b/testing/btest/Baseline/language.string/out
@@ -0,0 +1,24 @@
+empty string (PASS)
+nonempty string (PASS)
+string comparison (PASS)
+string comparison (PASS)
+string comparison (PASS)
+string comparison (PASS)
+null escape sequence (PASS)
+tab escape sequence (PASS)
+newline escape sequence (PASS)
+hex escape sequence (PASS)
+hex escape sequence (PASS)
+hex escape sequence (PASS)
+octal escape sequence (PASS)
+quote escape sequence (PASS)
+backslash escape sequence (PASS)
+null escape sequence (PASS)
+newline escape sequence (PASS)
+tab escape sequence (PASS)
+string concatenation (PASS)
+string concatenation (PASS)
+long string initialization (PASS)
+in operator (PASS)
+!in operator (PASS)
+type inference (PASS)
diff --git a/testing/btest/Baseline/language.subnet/out b/testing/btest/Baseline/language.subnet/out
new file mode 100644
index 0000000000..f753d65c68
--- /dev/null
+++ b/testing/btest/Baseline/language.subnet/out
@@ -0,0 +1,10 @@
+IPv4 subnet equality (PASS)
+IPv4 subnet inequality (PASS)
+IPv4 subnet in operator (PASS)
+IPv4 subnet !in operator (PASS)
+IPv6 subnet equality (PASS)
+IPv6 subnet inequality (PASS)
+IPv6 subnet in operator (PASS)
+IPv6 subnet !in operator (PASS)
+IPv4 and IPv6 subnet inequality (PASS)
+IPv4 address and IPv6 subnet (PASS)
diff --git a/testing/btest/Baseline/language.table/out b/testing/btest/Baseline/language.table/out
new file mode 100644
index 0000000000..8a45707e2d
--- /dev/null
+++ b/testing/btest/Baseline/language.table/out
@@ -0,0 +1,37 @@
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+iterate over table (PASS)
+iterate over table (PASS)
+iterate over table (PASS)
+iterate over table (PASS)
+iterate over table (PASS)
+add element (PASS)
+in operator (PASS)
+add element (PASS)
+add element (PASS)
+in operator (PASS)
+in operator (PASS)
+add element (PASS)
+in operator (PASS)
+add element (PASS)
+in operator (PASS)
+add element (PASS)
+in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
diff --git a/testing/btest/Baseline/language.time/out b/testing/btest/Baseline/language.time/out
new file mode 100644
index 0000000000..3615a17c53
--- /dev/null
+++ b/testing/btest/Baseline/language.time/out
@@ -0,0 +1,7 @@
+add interval (PASS)
+subtract interval (PASS)
+inequality (PASS)
+equality (PASS)
+subtract time (PASS)
+size operator (PASS)
+type inference (PASS)
diff --git a/testing/btest/Baseline/language.timeout/out b/testing/btest/Baseline/language.timeout/out
new file mode 100644
index 0000000000..790851a6bb
--- /dev/null
+++ b/testing/btest/Baseline/language.timeout/out
@@ -0,0 +1 @@
+timeout
diff --git a/testing/btest/Baseline/language.vector/out b/testing/btest/Baseline/language.vector/out
new file mode 100644
index 0000000000..4196b36141
--- /dev/null
+++ b/testing/btest/Baseline/language.vector/out
@@ -0,0 +1,31 @@
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+iterate over vector (PASS)
+iterate over vector (PASS)
+iterate over vector (PASS)
+add element (PASS)
+access element (PASS)
+add element (PASS)
+add element (PASS)
+access element (PASS)
+access element (PASS)
+add element (PASS)
+access element (PASS)
+add element (PASS)
+access element (PASS)
+add element (PASS)
+access element (PASS)
+overwrite element (PASS)
+access element (PASS)
+overwrite element (PASS)
+access element (PASS)
+access element (PASS)
+overwrite element (PASS)
+access element (PASS)
+overwrite element (PASS)
+access element (PASS)
+overwrite element (PASS)
+access element (PASS)
diff --git a/testing/btest/Baseline/language.when/out b/testing/btest/Baseline/language.when/out
new file mode 100644
index 0000000000..3a052217ab
--- /dev/null
+++ b/testing/btest/Baseline/language.when/out
@@ -0,0 +1,2 @@
+done
+lookup successful
diff --git a/testing/btest/language/addr.bro b/testing/btest/language/addr.bro
new file mode 100644
index 0000000000..b97710ce22
--- /dev/null
+++ b/testing/btest/language/addr.bro
@@ -0,0 +1,46 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	# IPv4 addresses
+	local a1: addr = 0.0.0.0;
+	local a2: addr = 10.0.0.11;
+	local a3: addr = 255.255.255.255;
+
+	test_case( "IPv4 address inequality", a1 != a2 );
+	test_case( "IPv4 address equality", a1 == 0.0.0.0 );
+	test_case( "IPv4 address comparison", a1 < a2 );
+	test_case( "IPv4 address comparison", a3 > a2 );
+	test_case( "size of IPv4 address", |a1| == 32 );
+
+	# IPv6 addresses
+	local b1: addr = [::];
+	local b2: addr = [::255.255.255.255];
+	local b3: addr = [::ffff:ffff];
+	local b4: addr = [ffff::ffff];
+	local b5: addr = [0000:0000:0000:0000:0000:0000:0000:0000];
+	local b6: addr = [aaaa:bbbb:cccc:dddd:eeee:ffff:1111:2222];
+	local b7: addr = [AAAA:BBBB:CCCC:DDDD:EEEE:FFFF:1111:2222];
+
+	test_case( "IPv6 address inequality", b1 != b2 );
+	test_case( "IPv6 address equality", b1 == b5 );
+	test_case( "IPv6 address equality", b2 == b3 );
+	test_case( "IPv6 address comparison", b1 < b2 );
+	test_case( "IPv6 address comparison", b4 > b2 );
+	test_case( "IPv6 address not case-sensitive", b6 == b7 );
+	test_case( "size of IPv6 address", |b1| == 128 );
+
+	test_case( "IPv4 and IPv6 address inequality", a1 != b1 );
+
+	# type inference
+	local x = 192.1.2.3;
+	local y = [a::b];
+}
+
diff --git a/testing/btest/language/bool.bro b/testing/btest/language/bool.bro
new file mode 100644
index 0000000000..09614b516e
--- /dev/null
+++ b/testing/btest/language/bool.bro
@@ -0,0 +1,28 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local b1: bool = T; 
+	local b2: bool = F;
+	local b3: bool = T;
+
+	test_case( "equality operator", b1 == b3 );
+	test_case( "inequality operator", b1 != b2 );
+	test_case( "logical or operator", b1 || b2 );
+	test_case( "logical and operator", b1 && b3 );
+	test_case( "negation operator", !b2 );
+	test_case( "absolute value", |b1| == 1 );
+	test_case( "absolute value", |b2| == 0 );
+
+	# type inference
+	local x = T;
+	local y = F;
+}
+
diff --git a/testing/btest/language/count.bro b/testing/btest/language/count.bro
new file mode 100644
index 0000000000..f2c248eae9
--- /dev/null
+++ b/testing/btest/language/count.bro
@@ -0,0 +1,42 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local c1: count = 0;
+	local c2: count = 5;
+	local c3: count = 0xff;
+	local c4: count = 255;
+	local c5: count = 4294967295;  # maximum allowed value
+	local c6: counter = 5;
+
+	test_case( "inequality operator", c1 != c2 );
+	test_case( "relational operator", c1 < c2 );
+	test_case( "relational operator", c1 <= c2 );
+	test_case( "relational operator", c2 > c1 );
+	test_case( "relational operator", c2 >= c1 );
+	test_case( "hexadecimal", c3 == c4 );
+	test_case( "counter alias", c2 == c6 );
+	test_case( "absolute value", |c1| == 0 );
+	test_case( "absolute value", |c2| == 5 );
+	test_case( "pre-increment operator", ++c2 == 6 );
+	test_case( "pre-decrement operator", --c2 == 5 );
+	test_case( "modulus operator", c2%2 == 1 );
+	test_case( "division operator", c2/2 == 2 );
+	c2 += 3;
+	test_case( "assignment operator", c2 == 8 );
+	c2 -= 2;
+	test_case( "assignment operator", c2 == 6 );
+	local str1 = fmt("max count value = %d", c5);
+	test_case( str1, str1 == "max count value = 4294967295" );
+
+	# type inference
+	local x = 1;
+}
+
diff --git a/testing/btest/language/double.bro b/testing/btest/language/double.bro
new file mode 100644
index 0000000000..bee7e41a94
--- /dev/null
+++ b/testing/btest/language/double.bro
@@ -0,0 +1,66 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local d1: double = 3;
+	local d2: double = +3;
+	local d3: double = 3.;
+	local d4: double = 3.0;
+	local d5: double = +3.0;
+	local d6: double = 3e0;
+	local d7: double = 3E0;
+	local d8: double = 3e+0;
+	local d9: double = 3e-0;
+	local d10: double = 3.0e0;
+	local d11: double = +3.0e0;
+	local d12: double = +3.0e+0;
+	local d13: double = +3.0E+0;
+	local d14: double = +3.0E-0;
+	local d15: double = .03E+2;
+	local d16: double = .03E2;
+	local d17: double = 3.0001;
+	local d18: double = -3.0001;
+	local d19: double = 1.7e308;  # almost maximum allowed value
+
+	test_case( "double representations", d1 == d2 );
+	test_case( "double representations", d1 == d3 );
+	test_case( "double representations", d1 == d4 );
+	test_case( "double representations", d1 == d5 );
+	test_case( "double representations", d1 == d6 );
+	test_case( "double representations", d1 == d7 );
+	test_case( "double representations", d1 == d8 );
+	test_case( "double representations", d1 == d9 );
+	test_case( "double representations", d1 == d10 );
+	test_case( "double representations", d1 == d11 );
+	test_case( "double representations", d1 == d12 );
+	test_case( "double representations", d1 == d13 );
+	test_case( "double representations", d1 == d14 );
+	test_case( "double representations", d1 == d15 );
+	test_case( "double representations", d1 == d16 );
+	test_case( "inequality operator", d18 != d17 );
+	test_case( "absolute value", |d18| == d17 );
+	d4 += 2;
+	test_case( "assignment operator", d4 == 5.0 );
+	d4 -= 3;
+	test_case( "assignment operator", d4 == 2.0 );
+	test_case( "relational operator", d4 <= d3 );
+	test_case( "relational operator", d4 < d3 );
+	test_case( "relational operator", d17 >= d3 );
+	test_case( "relational operator", d17 > d3 );
+	test_case( "division operator", d3/2 == 1.5 );
+	local str1 = fmt("max double value = %.1e", d19);
+	test_case( str1, str1 == "max double value = 1.7e+308" );
+
+	# type inference
+	local x = 7.0;
+	local y = 7e0;
+	local z = 7e+1;
+}
+
diff --git a/testing/btest/language/enum.bro b/testing/btest/language/enum.bro
new file mode 100644
index 0000000000..5cafb323a6
--- /dev/null
+++ b/testing/btest/language/enum.bro
@@ -0,0 +1,32 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+# enum with optional comma at end of definition
+type color: enum { Red, White, Blue, };
+
+# enum without optional comma
+type city: enum { Rome, Paris };
+
+
+event bro_init()
+{
+	local e1: color = Blue;
+	local e2: color = White;
+	local e3: color = Blue;
+	local e4: city = Rome;
+
+	test_case( "enum equality comparison", e1 != e2 );
+	test_case( "enum equality comparison", e1 == e3 );
+	test_case( "enum equality comparison", e1 != e4 );
+
+	# type inference
+	local x = Blue;
+	test_case( "type inference", x == e1 );
+}
+
diff --git a/testing/btest/language/event.bro b/testing/btest/language/event.bro
new file mode 100644
index 0000000000..1ea5c7b6d8
--- /dev/null
+++ b/testing/btest/language/event.bro
@@ -0,0 +1,49 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+
+event e1()
+	{
+	print "event statement";
+	return;
+	print "Error: this should not happen";
+	}
+
+event e2()
+	{
+	print "schedule statement";
+	}
+
+event e3(test: string)
+	{
+	print "event part1";
+	}
+
+event e4(num: count)
+	{
+	print "assign event variable";
+	}
+
+# Note: the name of this event is intentionally the same as one above
+event e3(test: string)
+	{
+	print "event part2";
+	}
+
+event bro_init()
+{
+	# Test calling an event with "event" statement
+	event e1();
+
+	# Test calling an event with "schedule" statement
+	schedule 1 sec { e2() };
+
+	# Test calling an event that has two separate definitions
+	event e3("foo");
+
+	# Test assigning an event variable to an event
+	local e5: event(num: count);
+	e5 = e4;
+	event e5(6);  # TODO: this does not do anything
+}
+
diff --git a/testing/btest/language/file.bro b/testing/btest/language/file.bro
new file mode 100644
index 0000000000..77650a6082
--- /dev/null
+++ b/testing/btest/language/file.bro
@@ -0,0 +1,19 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: btest-diff out2
+
+
+event bro_init()
+{
+	# Test using "print" statement to output directly to a file
+	local f1: file = open( "out1" );
+	print f1, 20;
+	print f1, 12;
+	close(f1);
+
+	# Test again, but without explicitly using the type name in declaration
+	local f2 = open( "out2" );
+	print f2, "test", 123, 456;
+	close(f2);
+}
+
diff --git a/testing/btest/language/for.bro b/testing/btest/language/for.bro
new file mode 100644
index 0000000000..f10ef0eb1b
--- /dev/null
+++ b/testing/btest/language/for.bro
@@ -0,0 +1,44 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+
+event bro_init()
+{
+	local vv: vector of string = vector( "a", "b", "c" );
+	local ct: count = 0;
+
+	# Test a "for" loop without "break" or "next" 
+
+	ct = 0;
+	for ( i in vv ) ++ct;
+	test_case("for loop", ct == 3 );
+
+	# Test the "break" statement
+
+	ct = 0;
+	for ( i in vv )
+	{
+		++ct;
+		break;
+		test_case("Error: this should not happen", F);
+	}
+	test_case("for loop with break", ct == 1 );
+
+	# Test the "next" statement
+
+	ct = 0;
+	for ( i in vv )
+	{
+		++ct;
+		next;
+		test_case("Error: this should not happen", F);
+	}
+	test_case("for loop with next", ct == 3 );
+}
+
diff --git a/testing/btest/language/function.bro b/testing/btest/language/function.bro
new file mode 100644
index 0000000000..13efbb91f8
--- /dev/null
+++ b/testing/btest/language/function.bro
@@ -0,0 +1,73 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+function f1()
+	{
+	test_case("no args without return value", T );
+	}
+
+function f2()
+	{
+	test_case("no args no return value, empty return", T );
+	return;
+	}
+
+function f3(): bool 
+	{
+	return T;
+	}
+
+function f4(test: string)
+	{
+	test_case("args without return value", T );
+	}
+
+function f5(test: string): bool 
+	{
+	return T;
+	}
+
+function f6(test: string, num: count): bool 
+	{
+	local val: int = -num;
+	if ( test == "bar" && num == 3 && val < 0 ) return T;
+	return F;
+	}
+
+function f7(test: string): bool 
+	{
+	return F;
+	}
+
+event bro_init()
+{
+	f1();
+	f2();
+	test_case("no args with return value", f3() );
+	f4("foo");
+	test_case("args with return value", f5("foo") );
+	test_case("multiple args with return value", f6("bar", 3) );
+
+	local f10 = function() { test_case("anonymous function without args or return value", T ); };
+	f10();
+
+	local f11 = function(): bool { return T; };
+	test_case("anonymous function with return value", f11() );
+
+	local f12 = function(val: int): bool { if (val > 0) return T; else return F; };
+	test_case("anonymous function with args and return value", f12(2) );
+
+	# Test that a function variable can later be assigned to a function
+	local f13: function(test: string): bool;
+	f13 = f5;
+	test_case("assign function variable", f13("foo") );
+	f13 = f7;
+	test_case("reassign function variable", !f13("bar") );
+}
+
diff --git a/testing/btest/language/if.bro b/testing/btest/language/if.bro
new file mode 100644
index 0000000000..e9acea865f
--- /dev/null
+++ b/testing/btest/language/if.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+
+event bro_init()
+{
+	# Test "if" without "else"
+
+	if ( T ) test_case( "if T", T);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+
+	# Test "if" with only an "else"
+
+	if ( T ) test_case( "if T else", T);
+	else test_case( "Error: this should not happen", F);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else test_case( "if F else", T);
+
+	# Test "if" with only an "else if"
+
+	if ( T ) test_case( "if T else if F", T);
+	else if ( F ) test_case( "Error: this should not happen", F);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else if ( T ) test_case( "if F else if T", T);
+
+	if ( T ) test_case( "if T else if T", T);
+	else if ( T ) test_case( "Error: this should not happen", F);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else if ( F ) test_case( "Error: this should not happen", F);
+
+	# Test "if" with both "else if" and "else"
+
+	if ( T ) test_case( "if T else if F else", T);
+	else if ( F ) test_case( "Error: this should not happen", F);
+	else test_case( "Error: this should not happen", F);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else if ( T ) test_case( "if F else if T else", T);
+	else test_case( "Error: this should not happen", F);
+
+	if ( T ) test_case( "if T else if T else", T);
+	else if ( T ) test_case( "Error: this should not happen", F);
+	else test_case( "Error: this should not happen", F);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else if ( F ) test_case( "Error: this should not happen", F);
+	else test_case( "if F else if F else", T);
+
+	# Test "if" with multiple "else if" and an "else"
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else if ( F ) test_case( "Error: this should not happen", F);
+	else if ( T ) test_case( "if F else if F else if T else", T);
+	else test_case( "Error: this should not happen", F);
+
+	if ( F ) test_case( "Error: this should not happen", F);
+	else if ( F ) test_case( "Error: this should not happen", F);
+	else if ( F ) test_case( "Error: this should not happen", F);
+	else test_case( "if F else if F else if F else", T);
+}
+
diff --git a/testing/btest/language/int.bro b/testing/btest/language/int.bro
new file mode 100644
index 0000000000..0c11b94235
--- /dev/null
+++ b/testing/btest/language/int.bro
@@ -0,0 +1,54 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local i1: int = 3;
+	local i2: int = +3;
+	local i3: int = -3;
+	local i4: int = +0;
+	local i5: int = -0;
+	local i6: int = 12;
+	local i7: int = 0xc;
+	local i8: int = 0xC;
+	local i9: int = -0xC;
+	local i10: int = -12;
+	local i11: int = 4294967295;
+	local i12: int = -4294967295;
+
+	test_case( "optional '+' sign", i1 == i2 );
+	test_case( "negative vs. positive", i1 != i3 );
+	test_case( "negative vs. positive", i4 == i5 );
+	test_case( "hexadecimal", i6 == i7 );
+	test_case( "hexadecimal", i6 == i8 );
+	test_case( "hexadecimal", i9 == i10 );
+	test_case( "relational operator", i2 > i3 );
+	test_case( "relational operator", i2 >= i3 );
+	test_case( "relational operator", i3 < i2 );
+	test_case( "relational operator", i3 <= i2 );
+	test_case( "absolute value", |i4| == 0 );
+	test_case( "absolute value", |i3| == 3 );
+	test_case( "pre-increment operator", ++i2 == 4 );
+	test_case( "pre-decrement operator", --i2 == 3 );
+	test_case( "modulus operator", i2%2 == 1 );
+	test_case( "division operator", i2/2 == 1 );
+	i2 += 4;
+	test_case( "assignment operator", i2 == 7 );
+	i2 -= 2;
+	test_case( "assignment operator", i2 == 5 );
+	local str1 = fmt("max int value = %d", i11);
+	test_case( str1, str1 == "max int value = 4294967295" );
+	local str2 = fmt("min int value = %d", i12);
+	test_case( str2, str2 == "min int value = -4294967295" );
+
+	# type inference
+	local x = +3;
+	test_case( "type inference", type_name(x) == "int" );
+}
+
diff --git a/testing/btest/language/interval.bro b/testing/btest/language/interval.bro
new file mode 100644
index 0000000000..9467db9397
--- /dev/null
+++ b/testing/btest/language/interval.bro
@@ -0,0 +1,77 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+function approx_equal(x: double, y: double): bool
+	{
+	# return T if x and y are approximately equal, and F otherwise
+	return |(x - y)/x| < 1e-6 ? T : F;
+	}
+
+event bro_init()
+{
+	# constants without space and no letter "s"
+	local in11: interval = 2usec;
+	local in12: interval = 2msec;
+	local in13: interval = 120sec;
+	local in14: interval = 2min;
+	local in15: interval = -2hr;
+	# TODO: this one causes bro to fail
+	#local in16: interval = 2.5day;
+
+	# constants with space and no letter "s"
+	local in21: interval = 2 usec;
+	local in22: interval = 2 msec;
+	local in23: interval = 120 sec;
+	local in24: interval = 2 min;
+	local in25: interval = -2 hr;
+	local in26: interval = 2.5 day;
+
+	# constants with space and letter "s"
+	local in31: interval = 2 usecs;
+	local in32: interval = 2 msecs;
+	local in33: interval = 120 secs;
+	local in34: interval = 2 mins;
+	local in35: interval = -2 hrs;
+	local in36: interval = 2.5 days;
+
+	test_case( "optional space", in11 == in21 );
+	test_case( "different units with same numeric value", in11 != in12 );
+	test_case( "plural/singular interval are same", in11 == in31 );
+	test_case( "compare different time units", in13 == in34 );
+	test_case( "compare different time units", in13 <= in34 );
+	test_case( "compare different time units", in13 >= in34 );
+	test_case( "compare different time units", in13 < in36 );
+	test_case( "compare different time units", in13 <= in36 );
+	test_case( "compare different time units", in13 > in35 );
+	test_case( "compare different time units", in13 >= in35 );
+	test_case( "add different time units", in13 + in14 == 4min );
+	test_case( "subtract different time units", in24 - in23 == 0sec );
+	test_case( "absolute value", |in25| == 2.0*3600 );
+	test_case( "absolute value", |in36| == 2.5*86400 );
+	in34 += 2hr;
+	test_case( "assignment operator", in34 == 122min );
+	# TODO: this should work (subtraction works)
+	#in34 -= 2hr;
+	#test_case( "assignment operator", in34 == 2min );
+	test_case( "multiplication operator", in33*2 == 4min );
+	test_case( "division operator", in35/2 == -1hr );
+	test_case( "division operator", approx_equal(in32/in31, 1e3) );
+
+	test_case( "relative size of units", approx_equal(1msec/1usec, 1000) );
+	test_case( "relative size of units", approx_equal(1sec/1msec, 1000) );
+	test_case( "relative size of units", approx_equal(1min/1sec, 60) );
+	test_case( "relative size of units", approx_equal(1hr/1min, 60) );
+	test_case( "relative size of units", approx_equal(1day/1hr, 24) );
+
+	# type inference
+	local x = 2 usec;
+	# TODO: this one causes bro to fail
+	#local y = 2.1usec;
+	local z = 3usecs;
+}
+
diff --git a/testing/btest/language/null-statement.bro b/testing/btest/language/null-statement.bro
new file mode 100644
index 0000000000..420ebd8a6c
--- /dev/null
+++ b/testing/btest/language/null-statement.bro
@@ -0,0 +1,34 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+
+function f1(test: string)
+	{
+	;  # null statement in function
+	}
+
+event bro_init()
+{
+	local s1: set[string] = set( "this", "test" );
+
+	; # null statement in event
+
+	for ( i in s1 )
+		;  # null statement in for loop
+
+	if ( |s1| > 0 ) ;  # null statement in if statement
+
+	f1("foo");
+
+	{ ; }  # null compound statement
+
+	if ( |s1| == 0 ) 
+		{
+		print "Error: this should not happen";
+		}
+	else
+		;  # null statement in else
+
+	print "done";
+}
+
diff --git a/testing/btest/language/pattern.bro b/testing/btest/language/pattern.bro
new file mode 100644
index 0000000000..de33e4d2b6
--- /dev/null
+++ b/testing/btest/language/pattern.bro
@@ -0,0 +1,28 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local p1: pattern = /foo|bar/; 
+	local p2: pattern = /oob/; 
+	local p3: pattern = /^oob/; 
+
+	test_case( "equality operator", "foo" == p1 );
+	test_case( "equality operator (order of operands)", p1 == "foo" );
+	test_case( "inequality operator", "foobar" != p1 );
+	test_case( "in operator", p1 in "foobar" );
+	test_case( "in operator", p2 in "foobar" );
+	test_case( "!in operator", p3 !in "foobar" );
+
+	# type inference
+	local x = /foo|bar/;
+	local y = /foo/;
+	local z = /^foo/;
+}
+
diff --git a/testing/btest/language/port.bro b/testing/btest/language/port.bro
new file mode 100644
index 0000000000..b45401da7a
--- /dev/null
+++ b/testing/btest/language/port.bro
@@ -0,0 +1,35 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local p1: port = 1/icmp;
+	local p2: port = 2/udp;
+	local p3: port = 3/tcp;
+	local p4: port = 4/unknown;
+
+	# maximum allowed values for each port type
+	local p5: port = 255/icmp;
+	local p6: port = 65535/udp;
+	local p7: port = 65535/tcp;
+	local p8: port = 255/unknown;
+
+	test_case( "protocol ordering", p1 > p2 );
+	test_case( "protocol ordering", p2 > p3 );
+	test_case( "protocol ordering", p3 > p4 );
+	test_case( "protocol ordering", p7 < p6 );
+	test_case( "protocol ordering", p8 < p5 );
+	test_case( "different protocol but same numeric value", p6 != p7 );
+	test_case( "different protocol but same numeric value", p5 != p8 );
+	test_case( "equality operator", 65535/tcp == p7 );
+
+	# type inference
+	local x = 123/tcp;
+}
+
diff --git a/testing/btest/language/set.bro b/testing/btest/language/set.bro
new file mode 100644
index 0000000000..66b2ebc3af
--- /dev/null
+++ b/testing/btest/language/set.bro
@@ -0,0 +1,121 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+# Note: only global sets can be initialized with curly braces
+global s10: set[string] = { "curly", "braces" };
+global s11: set[port, string, bool] = { [10/udp, "curly", F],
+		[11/udp, "braces", T] };
+
+event bro_init()
+{
+	local s1: set[string] = set( "test", "example" );
+	local s2: set[string] = set();
+	local s3: set[string];
+	local s4 = set( "type inference" );
+	local s5: set[port, string, bool] = set( [1/tcp, "test", T],
+			 [2/tcp, "example", F] );
+	local s6: set[port, string, bool] = set();
+	local s7: set[port, string, bool];
+	local s8 = set( [8/tcp, "type inference", T] );
+
+	# Test the size of each set
+	test_case( "cardinality", |s1| == 2 );
+	test_case( "cardinality", |s2| == 0 );
+	test_case( "cardinality", |s3| == 0 );
+	test_case( "cardinality", |s4| == 1 );
+	test_case( "cardinality", |s5| == 2 );
+	test_case( "cardinality", |s6| == 0 );
+	test_case( "cardinality", |s7| == 0 );
+	test_case( "cardinality", |s8| == 1 );
+	test_case( "cardinality", |s10| == 2 );
+	test_case( "cardinality", |s11| == 2 );
+
+	# Test iterating over each set
+	local ct: count;
+	ct = 0;
+	for ( c in s1 )
+	{
+		if ( type_name(c) != "string" )
+			print "Error: wrong set element type";
+		++ct;
+	}
+	test_case( "iterate over set", ct == 2 );
+
+	ct = 0;
+	for ( c in s2 )
+	{
+		++ct;
+	}
+	test_case( "iterate over set", ct == 0 );
+
+	ct = 0;
+	for ( [c1,c2,c3] in s5 )
+	{
+		++ct;
+	}
+	test_case( "iterate over set", ct == 2 );
+
+	ct = 0;
+	for ( [c1,c2,c3] in s11 )
+	{
+		++ct;
+	}
+	test_case( "iterate over set", ct == 2 );
+
+	# Test adding elements to each set
+	add s1["added"];
+	test_case( "add element", |s1| == 3 );
+	test_case( "in operator", "added" in s1 );
+
+	add s2["another"];
+	test_case( "add element", |s2| == 1 );
+	add s2["test"];
+	test_case( "add element", |s2| == 2 );
+	test_case( "in operator", "another" in s2 );
+	test_case( "in operator", "test" in s2 );
+
+	add s3["foo"];
+	test_case( "add element", |s3| == 1 );
+	test_case( "in operator", "foo" in s3 );
+
+	add s4["local"];
+	test_case( "add element", |s4| == 2 );
+	test_case( "in operator", "local" in s4 );
+
+	# Note: cannot add elements to sets of multiple types
+
+	add s10["global"];
+	test_case( "add element", |s10| == 3 );
+	test_case( "in operator", "global" in s10 );
+
+	# Test removing elements from each set
+	delete s1["test"];
+	delete s1["foobar"];  # element does not exist
+	test_case( "remove element", |s1| == 2 );
+	test_case( "!in operator", "test" !in s1 );
+
+	delete s2["test"];
+	test_case( "remove element", |s2| == 1 );
+	test_case( "!in operator", "test" !in s2 );
+
+	delete s3["foo"];
+	test_case( "remove element", |s3| == 0 );
+	test_case( "!in operator", "foo" !in s3 );
+
+	delete s4["type inference"];
+	test_case( "remove element", |s4| == 1 );
+	test_case( "!in operator", "type inference" !in s4 );
+
+	# Note: cannot remove elements from sets of multiple types
+
+	delete s10["braces"];
+	test_case( "remove element", |s10| == 2 );
+	test_case( "!in operator", "braces" !in s10 );
+}
+
diff --git a/testing/btest/language/short-circuit.bro b/testing/btest/language/short-circuit.bro
new file mode 100644
index 0000000000..f0ba585cea
--- /dev/null
+++ b/testing/btest/language/short-circuit.bro
@@ -0,0 +1,48 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+global ct: count;
+
+function t_func(): bool
+        {
+	ct += 1;
+	return T;
+        }
+
+function f_func(): bool
+        {
+	ct += 2;
+	return F;
+        }
+
+
+event bro_init()
+{
+	local res: bool;
+
+	# both functions should be called
+	ct = 0;
+	res = t_func() && f_func();
+	test_case("&& operator (eval. both operands)", res == F && ct == 3 );
+
+	# only first function should be called
+	ct = 0;
+	res = f_func() && t_func();
+	test_case("&& operator (eval. 1st operand)", res == F && ct == 2 );
+
+	# only first function should be called
+	ct = 0;
+	res = t_func() || f_func();
+	test_case("|| operator (eval. 1st operand)", res == T && ct == 1 );
+
+	# both functions should be called
+	ct = 0;
+	res = f_func() || t_func();
+	test_case("|| operator (eval. both operands)", res == T && ct == 3 );
+}
+
diff --git a/testing/btest/language/string.bro b/testing/btest/language/string.bro
new file mode 100644
index 0000000000..b9a17e3645
--- /dev/null
+++ b/testing/btest/language/string.bro
@@ -0,0 +1,59 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local s1: string = "";        # empty string
+	local s2: string = "x";       # no escape sequences
+	local s3: string = "a\0b";    # null character
+	local s4: string = "a\tb";    # tab
+	local s5: string = "a\nb";    # newline
+	local s6: string = "a\xffb";  # hex value
+	local s7: string = "a\x00b";  # hex value
+	local s8: string = "a\x0ab";  # hex value
+	local s9: string = "a\011b";  # octal value
+	local s10: string = "a\"b";   # double quote
+	local s11: string = "a\\b";   # backslash
+	local s12: string = s2 + s3;  # string concatenation
+	local s13: string = "test";
+	local s14: string = "this is a very long string" +
+				"which continues on the next line" +
+				"the end";
+	local s15: string = "on";
+
+	test_case( "empty string", |s1| == 0 );
+	test_case( "nonempty string", |s2| == 1 );
+	test_case( "string comparison", s2 > s3 );
+	test_case( "string comparison", s2 >= s3 );
+	test_case( "string comparison", s3 < s2 );
+	test_case( "string comparison", s3 <= s2 );
+	test_case( "null escape sequence", |s3| == 3 );
+	test_case( "tab escape sequence", |s4| == 3 );
+	test_case( "newline escape sequence", |s5| == 3 );
+	test_case( "hex escape sequence", |s6| == 3 );
+	test_case( "hex escape sequence", |s7| == 3 );
+	test_case( "hex escape sequence", |s8| == 3 );
+	test_case( "octal escape sequence", |s9| == 3 );
+	test_case( "quote escape sequence", |s10| == 3 );
+	test_case( "backslash escape sequence", |s11| == 3 );
+	test_case( "null escape sequence", s3 == s7 );
+	test_case( "newline escape sequence", s5 == s8 );
+	test_case( "tab escape sequence", s4 == s9 );
+	test_case( "string concatenation", |s12| == 4 );
+	s13 += s2;
+	test_case( "string concatenation", s13 == "testx" );
+	test_case( "long string initialization", |s14| == 65 );
+	test_case( "in operator", s15 in s14 );
+	test_case( "!in operator", s15 !in s13 );
+
+	# type inference
+	local x = "x";
+	test_case( "type inference", x == s2 );
+}
+
diff --git a/testing/btest/language/subnet.bro b/testing/btest/language/subnet.bro
new file mode 100644
index 0000000000..63d09f916b
--- /dev/null
+++ b/testing/btest/language/subnet.bro
@@ -0,0 +1,48 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+# TODO: "subnet inequality" tests (i.e., tests with "!=") always fail 
+
+event bro_init()
+{
+	# IPv4 addr
+	local a1: addr = 192.1.2.3;
+
+	# IPv4 subnets 
+	local s1: subnet = 0.0.0.0/0;
+	local s2: subnet = 192.0.0.0/8;
+	local s3: subnet = 255.255.255.255/32;
+
+	test_case( "IPv4 subnet equality", a1/8 == s2 );
+	test_case( "IPv4 subnet inequality", a1/4 != s2 );
+	test_case( "IPv4 subnet in operator", a1 in s2 );
+	test_case( "IPv4 subnet !in operator", a1 !in s3 );
+
+	# IPv6 addr
+	local b1: addr = [ffff::];
+	local b2: addr = [ffff::1];
+	local b3: addr = [ffff:1::1];
+
+	# IPv6 subnets
+	local t1: subnet = [::]/0;
+	local t2: subnet = [ffff::]/64;
+
+	test_case( "IPv6 subnet equality", b1/64 == t2 );
+	test_case( "IPv6 subnet inequality", b3/64 != t2 );
+	test_case( "IPv6 subnet in operator", b2 in t2 );
+	test_case( "IPv6 subnet !in operator", b3 !in t2 );
+
+	test_case( "IPv4 and IPv6 subnet inequality", s1 != t1 );
+	test_case( "IPv4 address and IPv6 subnet", a1 !in t2 );
+
+	# type inference
+	local x = 10.0.0.0/16;
+	local y = [a::]/32;
+}
+
diff --git a/testing/btest/language/table.bro b/testing/btest/language/table.bro
new file mode 100644
index 0000000000..d7fc677a6d
--- /dev/null
+++ b/testing/btest/language/table.bro
@@ -0,0 +1,131 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local t1: table[count] of string = table( [5] = "test", [0] = "example" );
+	local t2: table[count] of string = table();
+	local t3: table[count] of string;
+	local t4 = table( [1] = "type inference" );
+	local t5: table[count] of string = { [1] = "curly", [3] = "braces" };
+	local t6: table[port, string, bool] of string = table(
+			 [1/tcp, "test", T] = "test1",
+			 [2/tcp, "example", F] = "test2" );
+	local t7: table[port, string, bool] of string = table();
+	local t8: table[port, string, bool] of string;
+	local t9 = table( [8/tcp, "type inference", T] = "this" );
+	local t10: table[port, string, bool] of string = { 
+		[10/udp, "curly", F] = "first",
+		[11/udp, "braces", T] = "second" };
+
+	# Test the size of each table
+	test_case( "cardinality", |t1| == 2 );
+	test_case( "cardinality", |t2| == 0 );
+	test_case( "cardinality", |t3| == 0 );
+	test_case( "cardinality", |t4| == 1 );
+	test_case( "cardinality", |t5| == 2 );
+	test_case( "cardinality", |t6| == 2 );
+	test_case( "cardinality", |t7| == 0 );
+	test_case( "cardinality", |t8| == 0 );
+	test_case( "cardinality", |t9| == 1 );
+	test_case( "cardinality", |t10| == 2 );
+
+	# Test iterating over each table
+	local ct: count;
+	ct = 0;
+	for ( c in t1 )
+	{
+		if ( type_name(c) != "count" )
+			print "Error: wrong index type";
+		if ( type_name(t1[c]) != "string" )
+			print "Error: wrong table type";
+		++ct;
+	}
+	test_case( "iterate over table", ct == 2 );
+
+	ct = 0;
+	for ( c in t2 )
+	{
+		++ct;		
+	}
+	test_case( "iterate over table", ct == 0 );
+
+	ct = 0;
+	for ( c in t3 )
+	{
+		++ct;		
+	}
+	test_case( "iterate over table", ct == 0 );
+
+	ct = 0;
+	for ( [c1, c2, c3] in t6 )
+	{
+		++ct;		
+	}
+	test_case( "iterate over table", ct == 2 );
+
+	ct = 0;
+	for ( [c1, c2, c3] in t7 )
+	{
+		++ct;		
+	}
+	test_case( "iterate over table", ct == 0 );
+
+	# Test adding elements to each table
+	t1[1] = "added";
+	test_case( "add element", |t1| == 3 );
+	test_case( "in operator", 1 in t1 );
+
+	t2[11] = "another";
+	test_case( "add element", |t2| == 1 );
+	t2[0] = "test";
+	test_case( "add element", |t2| == 2 );
+	test_case( "in operator", 11 in t2 );
+	test_case( "in operator", 0 in t2 );
+
+	t3[3] = "foo";
+	test_case( "add element", |t3| == 1 );
+	test_case( "in operator", 3 in t3 );
+
+	t4[4] = "local";
+	test_case( "add element", |t4| == 2 );
+	test_case( "in operator", 4 in t4 );
+
+	t5[10] = "local2";
+	test_case( "add element", |t5| == 3 );
+	test_case( "in operator", 10 in t5 );
+
+	# Note: cannot add elements to tables of multiple types
+
+	# Test removing elements from each table
+	delete t1[0];
+	delete t1[17];  # element does not exist
+	test_case( "remove element", |t1| == 2 );
+	test_case( "!in operator", 0 !in t1 );
+
+	delete t2[0];
+	test_case( "remove element", |t2| == 1 );
+	test_case( "!in operator", 0 !in t2 );
+
+	delete t3[3];
+	test_case( "remove element", |t3| == 0 );
+	test_case( "!in operator", 3 !in t3 );
+
+	delete t4[1];
+	test_case( "remove element", |t4| == 1 );
+	test_case( "!in operator", 1 !in t4 );
+
+	delete t5[1];
+	test_case( "remove element", |t5| == 2 );
+	test_case( "!in operator", 1 !in t5 );
+
+	# Note: cannot remove elements from tables of multiple types
+
+}
+
diff --git a/testing/btest/language/time.bro b/testing/btest/language/time.bro
new file mode 100644
index 0000000000..588cbf8887
--- /dev/null
+++ b/testing/btest/language/time.bro
@@ -0,0 +1,28 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local t1: time = current_time();
+	local t2: time = t1 + 3 sec;
+	local t3: time = t2 - 10 sec;
+	local t4: time = t1;
+	local t5: interval = t2 - t1;
+
+	test_case( "add interval", t1 < t2 );
+	test_case( "subtract interval", t1 > t3 );
+	test_case( "inequality", t1 != t3 );
+	test_case( "equality", t1 == t4 );
+	test_case( "subtract time", t5 == 3sec);
+	test_case( "size operator", |t1| > 1.0);
+
+	local x = current_time();	
+	test_case( "type inference", x > t1 );
+}
+
diff --git a/testing/btest/language/timeout.bro b/testing/btest/language/timeout.bro
new file mode 100644
index 0000000000..6bc0419b2f
--- /dev/null
+++ b/testing/btest/language/timeout.bro
@@ -0,0 +1,19 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+
+event bro_init()
+{
+	local h1: addr = 1.2.3.4;
+
+	when ( local h1name = lookup_addr(h1) )
+		{ 
+		print "lookup successful";
+		}
+	timeout 3 secs
+		{
+		print "timeout";
+		}
+
+}
+
diff --git a/testing/btest/language/vector.bro b/testing/btest/language/vector.bro
new file mode 100644
index 0000000000..320736238e
--- /dev/null
+++ b/testing/btest/language/vector.bro
@@ -0,0 +1,104 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+# Note: only global vectors can be initialized with curly braces
+global v5: vector of string = { "curly", "braces" };
+
+event bro_init()
+{
+	local v1: vector of string = vector( "test", "example" );
+	local v2: vector of string = vector();
+	local v3: vector of string;
+	local v4 = vector( "type inference" );
+
+	# Test the size of each vector
+
+	test_case( "cardinality", |v1| == 2 );
+	test_case( "cardinality", |v2| == 0 );
+	test_case( "cardinality", |v3| == 0 );
+	test_case( "cardinality", |v4| == 1 );
+	test_case( "cardinality", |v5| == 2 );
+
+	# Test iterating over each vector
+
+	local ct: count;
+	ct = 0;
+	for ( c in v1 )
+	{
+		if ( type_name(c) != "int" )
+			print "Error: wrong index type";
+		if ( type_name(v1[c]) != "string" )
+			print "Error: wrong vector type";
+		++ct;
+	}
+	test_case( "iterate over vector", ct == 2 );
+
+	ct = 0;
+	for ( c in v2 )
+	{
+		++ct;
+	}
+	test_case( "iterate over vector", ct == 0 );
+
+	ct = 0;
+	for ( c in v5 )
+	{
+		++ct;
+	}
+	test_case( "iterate over vector", ct == 2 );
+
+	# Test adding elements to each vector
+
+	v1[2] = "added";
+	test_case( "add element", |v1| == 3 );
+	test_case( "access element", v1[2] == "added" );
+
+	v2[0] = "another";
+	test_case( "add element", |v2| == 1 );
+	v2[1] = "test";
+	test_case( "add element", |v2| == 2 );
+	test_case( "access element", v2[0] == "another" );
+	test_case( "access element", v2[1] == "test" );
+
+	v3[0] = "foo";
+	test_case( "add element", |v3| == 1 );
+	test_case( "access element", v3[0] == "foo" );
+
+	v4[1] = "local";
+	test_case( "add element", |v4| == 2 );
+	test_case( "access element", v4[1] == "local" );
+
+	v5[2] = "global";
+	test_case( "add element", |v5| == 3 );
+	test_case( "access element", v5[2] == "global" );
+
+	# Test overwriting elements of each vector
+
+	v1[0] = "new1";
+	test_case( "overwrite element", |v1| == 3 );
+	test_case( "access element", v1[0] == "new1" );
+
+	v2[1] = "new2";
+	test_case( "overwrite element", |v2| == 2 );
+	test_case( "access element", v2[0] == "another" );
+	test_case( "access element", v2[1] == "new2" );
+
+	v3[0] = "new3";
+	test_case( "overwrite element", |v3| == 1 );
+	test_case( "access element", v3[0] == "new3" );
+
+	v4[0] = "new4";
+	test_case( "overwrite element", |v4| == 2 );
+	test_case( "access element", v4[0] == "new4" );
+
+	v5[1] = "new5";
+	test_case( "overwrite element", |v5| == 3 );
+	test_case( "access element", v5[1] == "new5" );
+}
+
diff --git a/testing/btest/language/when.bro b/testing/btest/language/when.bro
new file mode 100644
index 0000000000..9ad45ab49b
--- /dev/null
+++ b/testing/btest/language/when.bro
@@ -0,0 +1,15 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+
+event bro_init()
+{
+	local h1: addr = 1.2.3.4;
+
+	when ( local h1name = lookup_addr(h1) )
+		{ 
+		print "lookup successful";
+		}
+	print "done";
+}
+

From 70f1403f1420b738d559c0675bd94703cd5af9aa Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 24 Aug 2012 13:18:51 -0700
Subject: [PATCH 159/238] Updating submodule(s).

 [nomail]
---
 CHANGES    | 2 +-
 VERSION    | 2 +-
 aux/broctl | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 066ee784a8..87da7378b0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1-beta-54 | 2012-08-23 11:58:50 -0700
+2.1 | 2012-08-24 13:18:51 -0700
 
   * Update documentation for builtin types. (Daniel Thayer)
     
diff --git a/VERSION b/VERSION
index fd6e9996db..879b416e60 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta-54
+2.1
diff --git a/aux/broctl b/aux/broctl
index 5b3f9e5906..6b24757768 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 5b3f9e5906c90b76c5aa1626e112d4c991cb3fd8
+Subproject commit 6b24757768cd9aa742fd678d6864235519740ee8

From b5c694518904a5f122bc643c02f0518e11c3dade Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 24 Aug 2012 15:11:49 -0700
Subject: [PATCH 160/238] Updating submodule(s).

 [nomail]
---
 CHANGES     | 2 +-
 aux/bro-aux | 2 +-
 aux/broctl  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 87da7378b0..1c6e9dfafe 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1 | 2012-08-24 13:18:51 -0700
+2.1 | 2012-08-24 15:11:49 -0700
 
   * Update documentation for builtin types. (Daniel Thayer)
     
diff --git a/aux/bro-aux b/aux/bro-aux
index 4bc1a6f6a8..6748ec3a96 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 4bc1a6f6a8816dfacd8288fcf182ba35520e589b
+Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff
diff --git a/aux/broctl b/aux/broctl
index 6b24757768..2fb9ff62bf 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 6b24757768cd9aa742fd678d6864235519740ee8
+Subproject commit 2fb9ff62bf08f78071753016863640022fbfe338

From 124c985d7af91a98eb8a7aff8f66b0300849e854 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 14:49:37 -0700
Subject: [PATCH 161/238] Bug found bei Keith & Seth: input framework was not
 handling counts and ints out of 32-bit-range correctly.

Note - another bugfix will be coming later (problem reading sets containing zero-length-strings & un-escaping-bug in sets)
---
 src/input/readers/Ascii.cc                    |  6 +--
 .../out                                       |  3 ++
 .../base/frameworks/input/bignumber.bro       | 44 +++++++++++++++++++
 3 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/bignumber.bro

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index fd936b07b6..28b1ed29c9 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -238,7 +238,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		break;
 
 	case TYPE_INT:
-		val->val.int_val = atoi(s.c_str());
+		val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10);
 		break;
 
 	case TYPE_DOUBLE:
@@ -249,7 +249,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 
 	case TYPE_COUNT:
 	case TYPE_COUNTER:
-		val->val.uint_val = atoi(s.c_str());
+		val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10);
 		break;
 
 	case TYPE_PORT:
@@ -344,7 +344,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 
 		if ( pos != length )
 			{
-			Error("Internal error while parsing set: did not find all elements");
+			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
 			return 0;
 			}
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
new file mode 100644
index 0000000000..ab095ca36c
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
@@ -0,0 +1,3 @@
+{
+[9223372036854775800] = [c=18446744073709551612]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro
new file mode 100644
index 0000000000..519992be05
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro
@@ -0,0 +1,44 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+9223372036854775800	18446744073709551612
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From 977c1d7c5adbf1b3bb2be55a99c4bd018e78a524 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 17:52:07 -0700
Subject: [PATCH 162/238] make set_separators different from , work for input
 framework.

1-line-patch + test.
---
 .../out                                       | 10 ++++
 .../base/frameworks/input/setseparator.bro    | 46 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/setseparator.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out
new file mode 100644
index 0000000000..d0e0f53310
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out
@@ -0,0 +1,10 @@
+{
+[1] = [s={
+b,
+e,
+d,
+c,
+f,
+a
+}, ss=[1, 2, 3, 4, 5, 6]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro
new file mode 100644
index 0000000000..44b9d08d54
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro
@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	a|b|c|d|e|f	1|2|3|4|5|6
+@TEST-END-FILE
+
+redef InputAscii::set_separator = "|";
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	ss:vector of count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From 6bf733ce513a39804ba73b1e281adba5322f2de6 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 17:53:34 -0700
Subject: [PATCH 163/238] sorry. the patch for the set_separator.

---
 src/input/readers/Ascii.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 28b1ed29c9..e0be235700 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -288,7 +288,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		// how many entries do we have...
 		unsigned int length = 1;
 		for ( unsigned int i = 0; i < s.size(); i++ )
-			if ( s[i] == ',' ) length++;
+			if ( s[i] == set_separator[0] ) length++;
 
 		unsigned int pos = 0;
 

From a9e6d9ae8154eecb415f86ca9f786f21886fff94 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 19:17:21 -0700
Subject: [PATCH 164/238] Fix two little bugs:

Escaped ,'s in sets and vectors were unescaped before tokenization

Handling of zero-length-strings as last element in a set was broken (sets ending with a ,).

Hashing of lines just containing zero-length-strings was broken (now a \0 is appended to each
string before it is hashed - giving us a hash of something for a line just consisting of \0s.
This also allows to differentiate between vectors with varying numbers of zero-length-strings).
---
 src/input/Manager.cc                          |  6 ++-
 src/input/readers/Ascii.cc                    | 18 ++++++-
 .../out                                       | 20 ++++++++
 .../base/frameworks/input/setspecialcases.bro | 49 +++++++++++++++++++
 4 files changed, 90 insertions(+), 3 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/setspecialcases.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 3c29f14928..07ce5b20fc 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1718,7 +1718,7 @@ int Manager::GetValueLength(const Value* val) {
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		length += val->val.string_val.length;
+		length += val->val.string_val.length+1;
 		break;
 		}
 
@@ -1818,7 +1818,9 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 	case TYPE_ENUM:
 		{
 		memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length);
-		return val->val.string_val.length;
+		// and add a \0 to the end. To be able to hash zero-length strings and differentiate from !present
+		memset(data+startpos+val->val.string_val.length, 0, 1); 		
+		return val->val.string_val.length+1;
 		}
 
 	case TYPE_ADDR:
diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index e0be235700..4bf82c6a13 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -220,6 +220,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 	switch ( field.type ) {
 	case TYPE_ENUM:
 	case TYPE_STRING:
+		s = get_unescaped_string(s);
 		val->val.string_val.length = s.size();
 		val->val.string_val.data = copy_string(s.c_str());
 		break;
@@ -259,6 +260,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 
 	case TYPE_SUBNET:
 		{
+		s = get_unescaped_string(s);
 		size_t pos = s.find("/");
 		if ( pos == s.npos )
 			{
@@ -275,6 +277,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		}
 
 	case TYPE_ADDR:
+		s = get_unescaped_string(s);
 		val->val.addr_val = StringToAddr(s);
 		break;
 
@@ -342,6 +345,20 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			pos++;
 			}
 
+		// test if the string ends with a set_separator... if it does we have to push an zero-lenght
+		// val on top of it.
+		if ( *s.rbegin() == set_separator[0] )
+			{
+			lvals[pos] = EntryToVal("", field.subType());
+			if ( lvals[pos] == 0 )
+				{
+				Error("Error while trying to add empty set element");
+				return 0;
+				}
+
+			pos++;
+			}
+
 		if ( pos != length )
 			{
 			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
@@ -438,7 +455,6 @@ bool Ascii::DoUpdate()
 			if ( ! getline(splitstream, s, separator[0]) )
 				break;
 
-			s = get_unescaped_string(s);
 
 			stringfields[pos] = s;
 			pos++;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
new file mode 100644
index 0000000000..28d1e025bf
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
@@ -0,0 +1,20 @@
+{
+[2] = [s={
+,
+testing
+}, s=[testing, , testing]],
+[4] = [s={
+,
+testing
+}, s=[testing, ]],
+[1] = [s={
+testing,testing,testing,
+}, s=[testing,testing,testing,]],
+[5] = [s={
+
+}, s=[, , , ]],
+[3] = [s={
+,
+testing
+}, s=[, testing]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
new file mode 100644
index 0000000000..29819a795f
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
@@ -0,0 +1,49 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	testing\x2ctesting\x2ctesting\x2c	testing\x2ctesting\x2ctesting\x2c
+2	testing,,testing	testing,,testing
+3	,testing	,testing
+4	testing,	testing,
+5	,,,	,,,
+@TEST-END-FILE
+
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	s: vector of string;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From fbe464ffa348c59b980584ad321e206d9a794ac2 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 20:26:08 -0700
Subject: [PATCH 165/238] another small bug found while searching for something
 else...

...one of the change events got the wrong parameters. This actually is a bit embarassing...
---
 src/input/Manager.cc                          |   2 +-
 .../scripts.base.frameworks.input.reread/out  | 240 ++++++++++++++++--
 2 files changed, 223 insertions(+), 19 deletions(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 07ce5b20fc..44d7140485 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1210,7 +1210,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 			Ref(predidx);
 			Ref(val);
 			Ref(ev);
-			SendEvent(stream->event, 3, ev, predidx, val);
+			SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, val);
 			}
 
 		if ( predidx )  // if we have a stream or an event...
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
index 8b55ced2ac..acc9bfe846 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
@@ -1174,10 +1174,45 @@ BB
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-43]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1190,13 +1225,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-46]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1209,13 +1278,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-44]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1228,13 +1331,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-47]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1247,13 +1384,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-45]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1266,13 +1437,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
-[i=-42]
+Input::EVENT_REMOVED
 Left
+[i=-42]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1285,7 +1490,6 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ==========SERVERS============
 {
 [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={

From 7e46936728f08b1214a6610e194793eb145a1f37 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 20:49:21 -0700
Subject: [PATCH 166/238] Ok, this one is not really necessary for 2.1 and more
 of a nice-to-have

Before this patch, empty values were not hashed at all. Which had the unfortunate side-effect
that e.g. the lines

TEST	-
and
-	TEST

have the same hash values. On re-reads that means that the change will
be ignored.

This is probably pretty academic, but this patch changes it and adds a testcase.

Output of the reread test changes due to re-ordering of the output (probably
due to the fact that the internal hash values are changed and thus transferred
in a different order)
---
 src/input/Manager.cc                          |  17 +-
 .../out                                       | 155 +++++++++++
 .../scripts.base.frameworks.input.reread/out  | 248 +++++++++---------
 .../frameworks/input/empty-values-hashing.bro |  89 +++++++
 4 files changed, 382 insertions(+), 127 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 44d7140485..e230c0e489 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1911,11 +1911,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
 		const Value* val = vals[i];
 		if ( val->present )
 			length += GetValueLength(val);
-		}
 
-	if ( length == 0 )
+		// and in any case add 1 for the end-of-field-identifier
+		length++;
+		}
+	
+
+	assert ( length >= num_elements );
+
+	if ( length == num_elements )
 		{
-		reporter->Error("Input reader sent line where all elements are null values. Ignoring line");
 		return NULL;
 		}
 
@@ -1929,6 +1934,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
 		const Value* val = vals[i];
 		if ( val->present )
 			position += CopyValue(data, position, val);
+
+		memset(data+position, 1, 1); // add end-of-field-marker. does not really matter which value it is,
+					// it just has to be... something
+
+		position++;
+
 		}
 
 	HashKey *key = new HashKey(data, length);
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out
new file mode 100644
index 0000000000..474ef45cc2
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out
@@ -0,0 +1,155 @@
+============PREDICATE============
+Input::EVENT_NEW
+[i=1]
+[s=<uninitialized>, ss=TEST]
+============PREDICATE============
+Input::EVENT_NEW
+[i=2]
+[s=<uninitialized>, ss=<uninitialized>]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=1]
+Right
+[s=<uninitialized>, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=2]
+Right
+[s=<uninitialized>, ss=<uninitialized>]
+==========SERVERS============
+{
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}
+============PREDICATE============
+Input::EVENT_CHANGED
+[i=1]
+[s=TEST, ss=<uninitialized>]
+============PREDICATE============
+Input::EVENT_CHANGED
+[i=2]
+[s=TEST, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_CHANGED
+Left
+[i=1]
+Right
+[s=<uninitialized>, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_CHANGED
+Left
+[i=2]
+Right
+[s=<uninitialized>, ss=<uninitialized>]
+==========SERVERS============
+{
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}
+done
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
index acc9bfe846..538a6dec18 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
@@ -1084,7 +1084,7 @@ BB
 }
 ============PREDICATE============
 Input::EVENT_REMOVED
-[i=-43]
+[i=-44]
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1096,6 +1096,21 @@ AA,
 BB
 }, se={
 
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_REMOVED
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
@@ -1111,21 +1126,6 @@ AA,
 BB
 }, se={
 
-}, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
-Input::EVENT_REMOVED
-[i=-44]
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
@@ -1159,7 +1159,113 @@ BB
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
+[i=-43]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
+[i=-44]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
 [i=-42]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1207,59 +1313,6 @@ print A::outfile, A::right;
 return (T);
 }, config={
 
-}]
-Type
-Input::EVENT_REMOVED
-Left
-[i=-43]
-Right
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-============EVENT============
-Description
-[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
-[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-{ 
-print A::outfile, ============EVENT============;
-print A::outfile, Description;
-print A::outfile, A::description;
-print A::outfile, Type;
-print A::outfile, A::tpe;
-print A::outfile, Left;
-print A::outfile, A::left;
-print A::outfile, Right;
-print A::outfile, A::right;
-}, pred=anonymous-function
-{ 
-print A::outfile, ============PREDICATE============;
-print A::outfile, A::typ;
-print A::outfile, A::left;
-print A::outfile, A::right;
-return (T);
-}, config={
-
 }]
 Type
 Input::EVENT_REMOVED
@@ -1313,59 +1366,6 @@ print A::outfile, A::right;
 return (T);
 }, config={
 
-}]
-Type
-Input::EVENT_REMOVED
-Left
-[i=-44]
-Right
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-============EVENT============
-Description
-[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
-[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-{ 
-print A::outfile, ============EVENT============;
-print A::outfile, Description;
-print A::outfile, A::description;
-print A::outfile, Type;
-print A::outfile, A::tpe;
-print A::outfile, Left;
-print A::outfile, A::left;
-print A::outfile, Right;
-print A::outfile, A::right;
-}, pred=anonymous-function
-{ 
-print A::outfile, ============PREDICATE============;
-print A::outfile, A::typ;
-print A::outfile, A::left;
-print A::outfile, A::right;
-return (T);
-}, config={
-
 }]
 Type
 Input::EVENT_REMOVED
@@ -1476,9 +1476,9 @@ return (T);
 Type
 Input::EVENT_REMOVED
 Left
-[i=-42]
+[i=-43]
 Right
-[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
new file mode 100644
index 0000000000..b66febba82
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
@@ -0,0 +1,89 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	-	TEST
+2	-	-
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	TEST	-
+2	TEST	TEST
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: string;
+	ss: string;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	}
+
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 2 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}

From f133e8808a0f8b199f47141f497cb33ed6a6955f Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 22:00:37 -0700
Subject: [PATCH 167/238] ok, this one might really be a bit too big for 2.1

Give all kinds of errors when encountering invalid numbers (like out-of-range-warnings, etc).
---
 src/input/readers/Ascii.cc                    | 57 ++++++++++++++++---
 src/input/readers/Ascii.h                     |  1 +
 .../out                                       |  3 +-
 .../.stderrwithoutfirstline                   |  8 +++
 .../out                                       |  5 ++
 .../base/frameworks/input/bignumber.bro       |  1 +
 .../base/frameworks/input/invalidnumbers.bro  | 55 ++++++++++++++++++
 7 files changed, 122 insertions(+), 8 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/invalidnumbers.bro

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 4bf82c6a13..1923532103 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -11,6 +11,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <errno.h>
 
 using namespace input::reader;
 using threading::Value;
@@ -209,6 +210,34 @@ bool Ascii::GetLine(string& str)
 	return false;
 	}
 
+bool Ascii::CheckNumberError(const string & s, const char * end)
+	{
+
+	if ( s.length() == 0 )
+		{
+		Error("Got empty string for number field");
+		return true;
+		}
+
+	if ( end == s.c_str() ) {
+		Error(Fmt("String '%s' contained no parseable number", s.c_str()));
+		return true;
+	}
+
+	if ( *end != '\0' )
+		Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+
+	if ( errno == EINVAL )
+		{
+		Error(Fmt("String '%s' could not be converted to a number", s.c_str()));
+		return true;
+		}
+	else if ( errno == ERANGE ) 
+		Error(Fmt("Number '%s' out of supported range. Number was truncated", s.c_str()));
+
+	return false;
+	}
+
 
 Value* Ascii::EntryToVal(string s, FieldMapping field)
 	{
@@ -216,6 +245,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		return new Value(field.type, false);
 
 	Value* val = new Value(field.type, true);
+	char* end;
+	errno = 0;
 
 	switch ( field.type ) {
 	case TYPE_ENUM:
@@ -239,22 +270,31 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		break;
 
 	case TYPE_INT:
-		val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10);
+		val->val.int_val = strtoll(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 		break;
 
 	case TYPE_DOUBLE:
 	case TYPE_TIME:
 	case TYPE_INTERVAL:
-		val->val.double_val = atof(s.c_str());
+		val->val.double_val = strtod(s.c_str(), &end);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 		break;
 
 	case TYPE_COUNT:
 	case TYPE_COUNTER:
-		val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10);
+		val->val.uint_val = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 		break;
-
+		
 	case TYPE_PORT:
-		val->val.port_val.port = atoi(s.c_str());
+		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
+
 		val->val.port_val.proto = TRANSPORT_UNKNOWN;
 		break;
 
@@ -268,8 +308,11 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			return 0;
 			}
 
-		int width = atoi(s.substr(pos+1).c_str());
+		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
 		string addr = s.substr(0, pos);
+		
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 
 		val->val.subnet_val.prefix = StringToAddr(addr);
 		val->val.subnet_val.length = width;
@@ -490,7 +533,7 @@ bool Ascii::DoUpdate()
 			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
 			if ( val == 0 )
 				{
-				Error("Could not convert String value to Val");
+				Error(Fmt("Could not convert line '%s' to Val. Aborting file read.", line.c_str()));
 				return false;
 				}
 
diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h
index e1506cbe82..2228e491b0 100644
--- a/src/input/readers/Ascii.h
+++ b/src/input/readers/Ascii.h
@@ -48,6 +48,7 @@ private:
 	bool ReadHeader(bool useCached);
 	bool GetLine(string& str);
 	threading::Value* EntryToVal(string s, FieldMapping type);
+	bool CheckNumberError(const string & s, const char * end);
 
 	ifstream* file;
 	time_t mtime;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
index ab095ca36c..8b95ed8b19 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
@@ -1,3 +1,4 @@
 {
-[9223372036854775800] = [c=18446744073709551612]
+[9223372036854775800] = [c=18446744073709551612],
+[-9223372036854775800] = [c=18446744073709551612]
 }
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
new file mode 100644
index 0000000000..bd32495a6f
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
@@ -0,0 +1,8 @@
+error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. Number was truncated
+error: ../input.log/Input::READER_ASCII: Number '121218446744073709551612' out of supported range. Number was truncated
+error: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
+error: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
+error: ../input2.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
+error: ../input2.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Aborting file read.
+received termination signal
+>>>
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
new file mode 100644
index 0000000000..9be82c13a9
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
@@ -0,0 +1,5 @@
+{
+[9223372036854775807] = [c=18446744073709551615],
+[9223372036854775800] = [c=4],
+[9223372036854775801] = [c=1]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro
index 519992be05..250f84bbb2 100644
--- a/testing/btest/scripts/base/frameworks/input/bignumber.bro
+++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro
@@ -10,6 +10,7 @@
 #fields	i	c
 #types	int	count
 9223372036854775800	18446744073709551612
+-9223372036854775800	18446744073709551612
 @TEST-END-FILE
 
 @load frameworks/communication/listen
diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
new file mode 100644
index 0000000000..7914b53d94
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
@@ -0,0 +1,55 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+12129223372036854775800	121218446744073709551612
+9223372036854775801TEXTHERE	1Justtext
+9223372036854775800	-18446744073709551612
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+#separator \x09
+#fields	i	c
+#types	int	count
+Justtext	1
+@TEST-END-FILE
+
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	Input::add_table([$source="../input2.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers]);
+	}

From a4ca5b0d829fa61a706913848620d85f2b125dd6 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 27 Aug 2012 09:49:57 -0700
Subject: [PATCH 168/238] fix handline of sets only containing a zero-length
 string.

Thank you Robin...
---
 src/input/readers/Ascii.cc                                 | 7 ++++---
 .../scripts.base.frameworks.input.setspecialcases/out      | 3 +++
 .../scripts/base/frameworks/input/setspecialcases.bro      | 1 +
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 4bf82c6a13..f1664a555a 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -345,9 +345,10 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			pos++;
 			}
 
-		// test if the string ends with a set_separator... if it does we have to push an zero-lenght
-		// val on top of it.
-		if ( *s.rbegin() == set_separator[0] )
+		// test if the string ends with a set_separator... or if the complete string is
+		// empty.
+		// In either of these cases we have to push an empty val on top of it.
+		if ( s.empty() ||  *s.rbegin() == set_separator[0] )
 			{
 			lvals[pos] = EntryToVal("", field.subType());
 			if ( lvals[pos] == 0 )
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
index 28d1e025bf..62229f7f37 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
@@ -7,6 +7,9 @@ testing
 ,
 testing
 }, s=[testing, ]],
+[6] = [s={
+
+}, s=[]],
 [1] = [s={
 testing,testing,testing,
 }, s=[testing,testing,testing,]],
diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
index 29819a795f..239bdfe7e7 100644
--- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
+++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
@@ -13,6 +13,7 @@
 3	,testing	,testing
 4	testing,	testing,
 5	,,,	,,,
+6		
 @TEST-END-FILE
 
 

From 5c486dae7e82ce308a6553a5dc53afb2fcae9ed8 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 27 Aug 2012 10:54:33 -0700
Subject: [PATCH 169/238] Ok, this one was a little bit sneaky.

If I understand things correctly, calling other string functions
on an stl string may alter the contents of the buffer to which
earlier .c_str()-calls pointed. Kind of makes sense when thinking
about it.

Basically moving around a few lines should fix this.

(And thank you again Robin)
---
 src/input/readers/Ascii.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 1923532103..276391ef84 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -213,6 +213,9 @@ bool Ascii::GetLine(string& str)
 bool Ascii::CheckNumberError(const string & s, const char * end)
 	{
 
+	bool endnotnull =  (*end != '\0'); // do this check first, before executing s.c_str() or similar.
+	// otherwise the value to which *end is pointing at the moment might be gone...
+
 	if ( s.length() == 0 )
 		{
 		Error("Got empty string for number field");
@@ -224,7 +227,7 @@ bool Ascii::CheckNumberError(const string & s, const char * end)
 		return true;
 	}
 
-	if ( *end != '\0' )
+	if ( endnotnull )
 		Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
 
 	if ( errno == EINVAL )
@@ -309,11 +312,12 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			}
 
 		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
-		string addr = s.substr(0, pos);
-		
+
 		if ( CheckNumberError(s, end) ) 
 			return 0;
 
+		string addr = s.substr(0, pos);
+
 		val->val.subnet_val.prefix = StringToAddr(addr);
 		val->val.subnet_val.length = width;
 		break;

From 56fa56ffa946581d7b4806b494821fe79f9974dc Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 27 Aug 2012 11:38:20 -0700
Subject: [PATCH 170/238] ...and another small change to error handling -> now
 errors in single lines do not kill processing, but simply ignore the line,
 log it, and continue.

---
 src/input/readers/Ascii.cc                    | 28 ++++++++++++++++---
 .../.stderrwithoutfirstline                   | 12 ++++----
 .../out                                       |  1 -
 .../base/frameworks/input/invalidnumbers.bro  | 11 ++------
 4 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 276391ef84..9c25953864 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -228,7 +228,7 @@ bool Ascii::CheckNumberError(const string & s, const char * end)
 	}
 
 	if ( endnotnull )
-		Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+		Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
 
 	if ( errno == EINVAL )
 		{
@@ -236,7 +236,10 @@ bool Ascii::CheckNumberError(const string & s, const char * end)
 		return true;
 		}
 	else if ( errno == ERANGE ) 
-		Error(Fmt("Number '%s' out of supported range. Number was truncated", s.c_str()));
+		{
+		Error(Fmt("Number '%s' out of supported range.", s.c_str()));
+		return true;
+		}
 
 	return false;
 	}
@@ -492,6 +495,7 @@ bool Ascii::DoUpdate()
 	while ( GetLine(line ) )
 		{
 		// split on tabs
+		bool error = false;
 		istringstream splitstream(line);
 
 		map<int, string> stringfields;
@@ -537,8 +541,9 @@ bool Ascii::DoUpdate()
 			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
 			if ( val == 0 )
 				{
-				Error(Fmt("Could not convert line '%s' to Val. Aborting file read.", line.c_str()));
-				return false;
+				Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str()));
+				error = true;
+				break;
 				}
 
 			if ( (*fit).secondary_position != -1 )
@@ -555,6 +560,21 @@ bool Ascii::DoUpdate()
 			fpos++;
 			}
 
+
+		if ( error ) 
+			{
+			// encountered non-fatal error. ignoring line.
+			// first - delete all successfully read fields and the array structure.
+			
+			for ( int i = 0; i < fpos; i++ )
+				delete fields[fpos];
+
+			delete[] fields;
+			continue;
+			}
+
+
+
 		//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
 		assert ( fpos == NumFields() );
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
index bd32495a6f..3ef51e40f2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
@@ -1,8 +1,8 @@
-error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. Number was truncated
-error: ../input.log/Input::READER_ASCII: Number '121218446744073709551612' out of supported range. Number was truncated
-error: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
-error: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
-error: ../input2.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
-error: ../input2.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Aborting file read.
+error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range.
+error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800	121218446744073709551612' to Val. Ignoring line.
+warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
+warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
+error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
+error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Ignoring line.
 received termination signal
 >>>
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
index 9be82c13a9..56b2736006 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
@@ -1,5 +1,4 @@
 {
-[9223372036854775807] = [c=18446744073709551615],
 [9223372036854775800] = [c=4],
 [9223372036854775801] = [c=1]
 }
diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
index 7914b53d94..3c755f1d08 100644
--- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
+++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
@@ -13,17 +13,10 @@
 #types	int	count
 12129223372036854775800	121218446744073709551612
 9223372036854775801TEXTHERE	1Justtext
+Justtext	1
 9223372036854775800	-18446744073709551612
 @TEST-END-FILE
 
-@TEST-START-FILE input2.log
-#separator \x09
-#fields	i	c
-#types	int	count
-Justtext	1
-@TEST-END-FILE
-
-
 @load frameworks/communication/listen
 
 global outfile: file;
@@ -51,5 +44,5 @@ event bro_init()
 event Input::update_finished(name: string, source:string)
 	{
 	print outfile, servers;
-	Input::add_table([$source="../input2.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers]);
+	terminate();
 	}

From 26f5aee7f6376d65031517efa78a1a6e7cbf1b46 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 28 Aug 2012 00:44:39 -0700
Subject: [PATCH 171/238] on 32-bit machines only unsigned long longs are
 64-bits long. Not just unsigned longs...

Note that this means that up to now all outputs (including logs) of counts > 32 bits were broken on 32-bit systems.
---
 src/modp_numtoa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modp_numtoa.c b/src/modp_numtoa.c
index 6deb8a70ed..6fa49b460f 100644
--- a/src/modp_numtoa.c
+++ b/src/modp_numtoa.c
@@ -56,7 +56,7 @@ void modp_uitoa10(uint32_t value, char* str)
 void modp_litoa10(int64_t value, char* str)
 {
     char* wstr=str;
-    unsigned long uvalue = (value < 0) ? -value : value;
+    unsigned long long uvalue = (value < 0) ? -value : value;
 
     // Conversion. Number is reversed.
     do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);

From 03f5795095642f89e11265ed36fda17f97a97ea9 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 28 Aug 2012 07:33:05 -0700
Subject: [PATCH 172/238] parse 64-bit consts correctly.

---
 src/scan.l | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/scan.l b/src/scan.l
index 645ce659cd..3148ba58ad 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -439,7 +439,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 {D}		{
  		// TODO: check if we can use strtoull instead of atol,
 		// and similarly for {HEX}.
-		RET_CONST(new Val(static_cast<unsigned int>(atol(yytext)),
+		RET_CONST(new Val(static_cast<bro_uint_t>(strtoll(yytext, (char**) NULL, 10)),
 			  TYPE_COUNT))
 		}
 {FLOAT}		RET_CONST(new Val(atof(yytext), TYPE_DOUBLE))
@@ -483,7 +483,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 
 ({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
 
-"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtol(yytext, 0, 16)), TYPE_COUNT))
+"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
 
 {H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
 

From b815b7ca5c133960102409d32bb492080112dde0 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 28 Aug 2012 10:57:21 -0500
Subject: [PATCH 173/238] Fix uninitialized value for 'is_partial' in TCP
 analyzer.

This led to non-deterministic behavior in cases where the first packet
analyzed wasn't from the originator side (see the conditionals in
TCP_Analyzer::CheckFlagCombos()).

The 'short' test in private test suite showed this behavior most often.
---
 src/TCP.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/TCP.cc b/src/TCP.cc
index 57e4449bf8..555adf1b57 100644
--- a/src/TCP.cc
+++ b/src/TCP.cc
@@ -46,6 +46,7 @@ TCP_Analyzer::TCP_Analyzer(Connection* conn)
 	finished = 0;
 	reassembling = 0;
 	first_packet_seen = 0;
+	is_partial = 0;
 
 	orig = new TCP_Endpoint(this, 1);
 	resp = new TCP_Endpoint(this, 0);

From 393ded1efe378a3f2109ccf49623e5050c12e048 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 28 Aug 2012 09:19:33 -0700
Subject: [PATCH 174/238] Set VERSION to 2.1-rc3 so that we don't get confused.

---
 CHANGES | 2 +-
 VERSION | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 02d7d74046..7df00f352c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1 | 2012-08-24 15:11:49 -0700
+2.1-rc3 | 2012-08-24 15:11:49 -0700
 
   * Input framework fixes, including:  (Bernhard Amann)
 
diff --git a/VERSION b/VERSION
index 879b416e60..1537f322a9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1
+2.1-rc3

From cc49193f93ba8c60b65b61047a0874982ad93db3 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 28 Aug 2012 13:11:12 -0500
Subject: [PATCH 175/238] Remove automatic use of gperftools on non-Linux
 systems.

--enable-perftools must now explicity be supplied to ./configure on
non-Linux systems to link against the tcmalloc library that a
gperftools installation provides.  Linux systems still automatically
link it if it's found.  The rationale is that gperftools was developed
and most throroughly tested on Linux so it's safer there.

There especially seems to be potential problems with gperftools on OS X
(e.g. see http://code.google.com/p/gperftools/issues/detail?id=413),
and Bro currently doesn't work with gpertools there using clang or gcc.
---
 CMakeLists.txt | 29 ++++++++++++++++++-----------
 configure      |  7 +++++++
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f667c0cfe0..2c8a726a1a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -88,24 +88,30 @@ if (LIBGEOIP_FOUND)
     list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
 endif ()
 
-set(USE_PERFTOOLS false)
+set(HAVE_PERFTOOLS false)
 set(USE_PERFTOOLS_DEBUG false)
+set(USE_PERFTOOLS_TCMALLOC false)
 
 if (NOT DISABLE_PERFTOOLS)
    find_package(GooglePerftools)
 endif ()
 
 if (GOOGLEPERFTOOLS_FOUND)
-    include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
-    set(USE_PERFTOOLS true)
+    set(HAVE_PERFTOOLS true)
+    # Non-Linux systems may not be well-supported by gperftools, so
+    # require explicit request from user to enable it in that case.
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS)
+        set(USE_PERFTOOLS_TCMALLOC true)
 
-    if (ENABLE_PERFTOOLS_DEBUG)
-        # Enable heap debugging with perftools.
-        set(USE_PERFTOOLS_DEBUG true)
-        list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
-    else ()
-        # Link in tcmalloc for better performance.
-        list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
+        if (ENABLE_PERFTOOLS_DEBUG)
+            # Enable heap debugging with perftools.
+            set(USE_PERFTOOLS_DEBUG true)
+            include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+            list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
+        else ()
+            # Link in tcmalloc for better performance.
+            list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
+        endif ()
     endif ()
 endif ()
 
@@ -224,7 +230,8 @@ message(
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
-    "\nGoogle perftools:  ${USE_PERFTOOLS}"
+    "\ngperftools found:  ${HAVE_PERFTOOLS}"
+    "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
     "\ncURL:              ${USE_CURL}"
     "\n"
diff --git a/configure b/configure
index b4ca606103..8e4aaa8425 100755
--- a/configure
+++ b/configure
@@ -29,6 +29,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
   Optional Features:
     --enable-debug         compile in debugging mode
     --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
+    --enable-perftools     force use of Google perftools on non-Linux systems
+                           (automatically on when perftools is present on Linux)
     --enable-perftools-debug use Google's perftools for debugging
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
@@ -98,6 +100,7 @@ append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/broctl
 append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
+append_cache_entry ENABLE_PERFTOOLS     BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
@@ -146,7 +149,11 @@ while [ $# -ne 0 ]; do
         --enable-mobile-ipv6)
             append_cache_entry ENABLE_MOBILE_IPV6         BOOL   true
             ;;
+        --enable-perftools)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
+            ;;
         --enable-perftools-debug)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
         --disable-broccoli)

From e4b7ffa8ac0718ace6d37371c8283efc50502c4f Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 28 Aug 2012 16:44:30 -0700
Subject: [PATCH 176/238] Updating CHANGES and VERSION.

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 2097bb1d94..9459d4ba2a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.1-rc3-5
+1.1

From b915db86d5c7b30c7d50d8b5ddfbbbdadd32107d Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 28 Aug 2012 16:46:37 -0700
Subject: [PATCH 177/238] Updating CHANGES and VERSION.

---
 CHANGES | 2 +-
 VERSION | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 232e2faa19..516c36974e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-1.1 | 2012-08-28 16:29:30 -0700
+2.1 | 2012-08-28 16:46:42 -0700
 
   * Remove automatic use of gperftools on non-Linux systems.
     --enable-perftools must now explicity be supplied to ./configure
diff --git a/VERSION b/VERSION
index 9459d4ba2a..879b416e60 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.1
+2.1

From 22cf75dae553dc2aa2a103bf7721cd466b764d64 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 29 Aug 2012 08:09:44 -0700
Subject: [PATCH 178/238] Two fixes.

    - Typo in recent scanner fix.

    - Make bif.identify_magic robust against FreeBSD's libmagic config.
---
 CHANGES                                       | 3 +++
 src/scan.l                                    | 4 +---
 testing/btest/Baseline/bifs.identify_data/out | 2 +-
 testing/btest/bifs/identify_data.bro          | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/CHANGES b/CHANGES
index 516c36974e..f8e4444f1d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,9 @@
 
 2.1 | 2012-08-28 16:46:42 -0700
 
+  * Make bif.identify_magic robust against FreeBSD's libmagic config.
+    (Robin Sommer)
+
   * Remove automatic use of gperftools on non-Linux systems.
     --enable-perftools must now explicity be supplied to ./configure
     on non-Linux systems to link against the tcmalloc library.
diff --git a/src/scan.l b/src/scan.l
index 3148ba58ad..1b3d09f879 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -437,9 +437,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 	}
 
 {D}		{
- 		// TODO: check if we can use strtoull instead of atol,
-		// and similarly for {HEX}.
-		RET_CONST(new Val(static_cast<bro_uint_t>(strtoll(yytext, (char**) NULL, 10)),
+		RET_CONST(new Val(static_cast<bro_uint_t>(strtoul(yytext, (char**) NULL, 10)),
 			  TYPE_COUNT))
 		}
 {FLOAT}		RET_CONST(new Val(atof(yytext), TYPE_DOUBLE))
diff --git a/testing/btest/Baseline/bifs.identify_data/out b/testing/btest/Baseline/bifs.identify_data/out
index a2872877f9..1cadefbf6e 100644
--- a/testing/btest/Baseline/bifs.identify_data/out
+++ b/testing/btest/Baseline/bifs.identify_data/out
@@ -1,4 +1,4 @@
 ASCII text, with no line terminators
 text/plain; charset=us-ascii
-PNG image data
+PNG image
 image/png; charset=binary
diff --git a/testing/btest/bifs/identify_data.bro b/testing/btest/bifs/identify_data.bro
index 11824b5e85..39f289d40b 100644
--- a/testing/btest/bifs/identify_data.bro
+++ b/testing/btest/bifs/identify_data.bro
@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro %INPUT | sed 's/PNG image data/PNG image/g' >out
 # @TEST-EXEC: btest-diff out
 
 event bro_init()

From 621a90d24821f5dafd4939e6b67248d0c1e98a8c Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 29 Aug 2012 17:14:03 -0500
Subject: [PATCH 179/238] Add more language tests

Added tests for the conditional operator, operator precedence,
modules ("module" and "export" keywords, and the "::" operator), and
for the "copy" keyword.

Also improved tests of max/min values of int, count, and double constants.
---
 .../language.conditional-expression/out       |   7 ++
 testing/btest/Baseline/language.copy/out      |   2 +
 testing/btest/Baseline/language.count/out     |   1 +
 testing/btest/Baseline/language.double/out    |   2 +-
 testing/btest/Baseline/language.int/out       |   2 +
 testing/btest/Baseline/language.module/out    |   4 +
 .../btest/Baseline/language.precedence/out    |  31 +++++
 .../btest/language/conditional-expression.bro |  66 +++++++++++
 testing/btest/language/copy.bro               |  30 +++++
 testing/btest/language/count.bro              |   9 +-
 testing/btest/language/double.bro             |   6 +-
 testing/btest/language/int.bro                |  10 +-
 testing/btest/language/module.bro             |  41 +++++++
 testing/btest/language/precedence.bro         | 110 ++++++++++++++++++
 14 files changed, 312 insertions(+), 9 deletions(-)
 create mode 100644 testing/btest/Baseline/language.conditional-expression/out
 create mode 100644 testing/btest/Baseline/language.copy/out
 create mode 100644 testing/btest/Baseline/language.module/out
 create mode 100644 testing/btest/Baseline/language.precedence/out
 create mode 100644 testing/btest/language/conditional-expression.bro
 create mode 100644 testing/btest/language/copy.bro
 create mode 100644 testing/btest/language/module.bro
 create mode 100644 testing/btest/language/precedence.bro

diff --git a/testing/btest/Baseline/language.conditional-expression/out b/testing/btest/Baseline/language.conditional-expression/out
new file mode 100644
index 0000000000..0dcbdbd7c7
--- /dev/null
+++ b/testing/btest/Baseline/language.conditional-expression/out
@@ -0,0 +1,7 @@
+true condition (PASS)
+false condition (PASS)
+true condition (PASS)
+false condition (PASS)
+associativity (PASS)
+associativity (PASS)
+associativity (PASS)
diff --git a/testing/btest/Baseline/language.copy/out b/testing/btest/Baseline/language.copy/out
new file mode 100644
index 0000000000..675d38aa5d
--- /dev/null
+++ b/testing/btest/Baseline/language.copy/out
@@ -0,0 +1,2 @@
+direct assignment (PASS)
+using copy (PASS)
diff --git a/testing/btest/Baseline/language.count/out b/testing/btest/Baseline/language.count/out
index 7dba9ea24c..a5de991e22 100644
--- a/testing/btest/Baseline/language.count/out
+++ b/testing/btest/Baseline/language.count/out
@@ -14,3 +14,4 @@ division operator (PASS)
 assignment operator (PASS)
 assignment operator (PASS)
 max count value = 4294967295 (PASS)
+max count value = 9223372036854775807 (PASS)
diff --git a/testing/btest/Baseline/language.double/out b/testing/btest/Baseline/language.double/out
index 01e3047743..9711e70d9b 100644
--- a/testing/btest/Baseline/language.double/out
+++ b/testing/btest/Baseline/language.double/out
@@ -22,4 +22,4 @@ relational operator (PASS)
 relational operator (PASS)
 relational operator (PASS)
 division operator (PASS)
-max double value = 1.7e+308 (PASS)
+max double value = 1.7976931348623157e+308 (PASS)
diff --git a/testing/btest/Baseline/language.int/out b/testing/btest/Baseline/language.int/out
index a50887999a..223d520e25 100644
--- a/testing/btest/Baseline/language.int/out
+++ b/testing/btest/Baseline/language.int/out
@@ -18,4 +18,6 @@ assignment operator (PASS)
 assignment operator (PASS)
 max int value = 4294967295 (PASS)
 min int value = -4294967295 (PASS)
+max int value = 9223372036854775807 (PASS)
+min int value = -9223372036854775807 (PASS)
 type inference (PASS)
diff --git a/testing/btest/Baseline/language.module/out b/testing/btest/Baseline/language.module/out
new file mode 100644
index 0000000000..5b011543b5
--- /dev/null
+++ b/testing/btest/Baseline/language.module/out
@@ -0,0 +1,4 @@
+function (PASS)
+global variable (PASS)
+const (PASS)
+event (PASS)
diff --git a/testing/btest/Baseline/language.precedence/out b/testing/btest/Baseline/language.precedence/out
new file mode 100644
index 0000000000..263ca83529
--- /dev/null
+++ b/testing/btest/Baseline/language.precedence/out
@@ -0,0 +1,31 @@
+++ and * (PASS)
+++ and * (PASS)
+* and ++ (PASS)
+* and % (PASS)
+* and % (PASS)
+* and % (PASS)
+% and * (PASS)
+% and * (PASS)
+% and * (PASS)
++ and * (PASS)
++ and * (PASS)
++ and * (PASS)
+< and + (PASS)
+< and + (PASS)
++ and < (PASS)
++ and < (PASS)
++= and + (PASS)
++= and + (PASS)
++= and + (PASS)
+&& and || (PASS)
+&& and || (PASS)
+&& and || (PASS)
+|| and && (PASS)
+|| and && (PASS)
+|| and && (PASS)
+|| and conditional operator (PASS)
+|| and conditional operator (PASS)
+|| and conditional operator (PASS)
+conditional operator and || (PASS)
+conditional operator and || (PASS)
+conditional operator and || (PASS)
diff --git a/testing/btest/language/conditional-expression.bro b/testing/btest/language/conditional-expression.bro
new file mode 100644
index 0000000000..74648b6ce8
--- /dev/null
+++ b/testing/btest/language/conditional-expression.bro
@@ -0,0 +1,66 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+global ct: count;
+
+function f1(): bool
+	{
+	ct += 1;
+	return T;
+	}
+
+function f2(): bool
+	{
+	ct += 4;
+	return F;
+	}
+
+
+event bro_init()
+{
+	local a: count;
+	local b: count;
+	local res: count;
+	local res2: bool;
+
+	# Test that the correct operand is evaluated
+
+	a = b = 0;
+	res = T ? ++a : ++b;
+	test_case( "true condition", a == 1 && b == 0 && res == 1);
+
+	a = b = 0;
+	res = F ? ++a : ++b;
+	test_case( "false condition", a == 0 && b == 1 && res == 1);
+
+	# Test again using function calls as operands
+
+	ct = 0;
+	res2 = ct == 0 ? f1() : f2();
+	test_case( "true condition", ct == 1 && res2 == T);
+
+	ct = 0;
+	res2 = ct != 0 ? f1() : f2();
+	test_case( "false condition", ct == 4 && res2 == F);
+
+	# Test that the conditional operator is right-associative
+
+	ct = 0;
+	T ? f1() : T ? f1() : f2();
+	test_case( "associativity", ct == 1 );
+
+	ct = 0;
+	T ? f1() : (T ? f1() : f2());
+	test_case( "associativity", ct == 1 );
+
+	ct = 0;
+	(T ? f1() : T) ? f1() : f2();
+	test_case( "associativity", ct == 2 );
+
+}
+
diff --git a/testing/btest/language/copy.bro b/testing/btest/language/copy.bro
new file mode 100644
index 0000000000..6740a080c7
--- /dev/null
+++ b/testing/btest/language/copy.bro
@@ -0,0 +1,30 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+
+event bro_init()
+{
+	# "b" is not a copy of "a"
+	local a: set[string] = set("this", "test");
+	local b: set[string] = a;
+
+	delete a["this"];
+
+	test_case( "direct assignment", |b| == 1 && "this" !in b );
+
+	# "d" is a copy of "c"
+	local c: set[string] = set("this", "test");
+	local d: set[string] = copy(c);
+
+	delete c["this"];
+
+	test_case( "using copy", |d| == 2 && "this" in d);
+
+}
+
diff --git a/testing/btest/language/count.bro b/testing/btest/language/count.bro
index f2c248eae9..97fb13ce51 100644
--- a/testing/btest/language/count.bro
+++ b/testing/btest/language/count.bro
@@ -11,10 +11,11 @@ event bro_init()
 {
 	local c1: count = 0;
 	local c2: count = 5;
-	local c3: count = 0xff;
+	local c3: count = 0xFF;
 	local c4: count = 255;
 	local c5: count = 4294967295;  # maximum allowed value
-	local c6: counter = 5;
+	local c6: count = 0x7fffffffffffffff;  # maximum allowed value
+	local c7: counter = 5;
 
 	test_case( "inequality operator", c1 != c2 );
 	test_case( "relational operator", c1 < c2 );
@@ -22,7 +23,7 @@ event bro_init()
 	test_case( "relational operator", c2 > c1 );
 	test_case( "relational operator", c2 >= c1 );
 	test_case( "hexadecimal", c3 == c4 );
-	test_case( "counter alias", c2 == c6 );
+	test_case( "counter alias", c2 == c7 );
 	test_case( "absolute value", |c1| == 0 );
 	test_case( "absolute value", |c2| == 5 );
 	test_case( "pre-increment operator", ++c2 == 6 );
@@ -35,6 +36,8 @@ event bro_init()
 	test_case( "assignment operator", c2 == 6 );
 	local str1 = fmt("max count value = %d", c5);
 	test_case( str1, str1 == "max count value = 4294967295" );
+	local str2 = fmt("max count value = %d", c6);
+	test_case( str2, str2 == "max count value = 9223372036854775807" );
 
 	# type inference
 	local x = 1;
diff --git a/testing/btest/language/double.bro b/testing/btest/language/double.bro
index bee7e41a94..f56d291631 100644
--- a/testing/btest/language/double.bro
+++ b/testing/btest/language/double.bro
@@ -27,7 +27,7 @@ event bro_init()
 	local d16: double = .03E2;
 	local d17: double = 3.0001;
 	local d18: double = -3.0001;
-	local d19: double = 1.7e308;  # almost maximum allowed value
+	local d19: double = 1.7976931348623157e308;  # maximum allowed value
 
 	test_case( "double representations", d1 == d2 );
 	test_case( "double representations", d1 == d3 );
@@ -55,8 +55,8 @@ event bro_init()
 	test_case( "relational operator", d17 >= d3 );
 	test_case( "relational operator", d17 > d3 );
 	test_case( "division operator", d3/2 == 1.5 );
-	local str1 = fmt("max double value = %.1e", d19);
-	test_case( str1, str1 == "max double value = 1.7e+308" );
+	local str1 = fmt("max double value = %.16e", d19);
+	test_case( str1, str1 == "max double value = 1.7976931348623157e+308" );
 
 	# type inference
 	local x = 7.0;
diff --git a/testing/btest/language/int.bro b/testing/btest/language/int.bro
index 0c11b94235..7cc91dd9d8 100644
--- a/testing/btest/language/int.bro
+++ b/testing/btest/language/int.bro
@@ -19,8 +19,10 @@ event bro_init()
 	local i8: int = 0xC;
 	local i9: int = -0xC;
 	local i10: int = -12;
-	local i11: int = 4294967295;
-	local i12: int = -4294967295;
+	local i11: int = 4294967295;   # max. allowed value
+	local i12: int = -4294967295;  # min. allowed value
+	local i13: int = 0x7fffffffffffffff;   # max. allowed value
+	local i14: int = -0x7fffffffffffffff;  # min. allowed value
 
 	test_case( "optional '+' sign", i1 == i2 );
 	test_case( "negative vs. positive", i1 != i3 );
@@ -46,6 +48,10 @@ event bro_init()
 	test_case( str1, str1 == "max int value = 4294967295" );
 	local str2 = fmt("min int value = %d", i12);
 	test_case( str2, str2 == "min int value = -4294967295" );
+	local str3 = fmt("max int value = %d", i13);
+	test_case( str3, str3 == "max int value = 9223372036854775807" );
+	local str4 = fmt("min int value = %d", i14);
+	test_case( str4, str4 == "min int value = -9223372036854775807" );
 
 	# type inference
 	local x = +3;
diff --git a/testing/btest/language/module.bro b/testing/btest/language/module.bro
new file mode 100644
index 0000000000..4c70546406
--- /dev/null
+++ b/testing/btest/language/module.bro
@@ -0,0 +1,41 @@
+# @TEST-EXEC: bro %INPUT secondtestfile >out
+# @TEST-EXEC: btest-diff out
+
+# In this source file, we define a module and export some objects
+
+module thisisatest;
+
+export {
+	global test_case: function(msg: string, expect: bool);
+
+	global testevent: event(msg: string);
+
+	global num: count = 123;
+
+	const daysperyear: count = 365;
+}
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+event testevent(msg: string)
+        {
+	test_case( "event", T );
+        }
+
+
+# @TEST-START-FILE secondtestfile
+
+# In this source file, we try to access each exported object from the module
+
+event bro_init()
+{
+	thisisatest::test_case( "function", T );
+	thisisatest::test_case( "global variable", thisisatest::num == 123 );
+	thisisatest::test_case( "const", thisisatest::daysperyear == 365 );
+	event thisisatest::testevent( "foo" );
+}
+
+# @TEST-END-FILE
diff --git a/testing/btest/language/precedence.bro b/testing/btest/language/precedence.bro
new file mode 100644
index 0000000000..da8fef311c
--- /dev/null
+++ b/testing/btest/language/precedence.bro
@@ -0,0 +1,110 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+# This is an incomplete set of tests to demonstrate the order of precedence
+# of bro script operators
+
+event bro_init()
+{
+	local n1: int;
+	local n2: int;
+	local n3: int;
+
+	# Tests that show "++" has higher precedence than "*"
+
+	n1 = n2 = 5;
+	n1 = ++n1 * 3;
+	n2 = (++n2) * 3;
+	test_case( "++ and *", n1 == 18 );
+	test_case( "++ and *", n2 == 18 );
+
+	n1 = 5;
+	n1 = 3 * ++n1;
+	test_case( "* and ++", n1 == 18 );
+
+	# Tests that show "*" has same precedence as "%"
+
+	n1 = 3 * 5 % 2;
+	n2 = (3 * 5) % 2;
+	n3 = 3 * (5 % 2);
+	test_case( "* and %", n1 == 1 );
+	test_case( "* and %", n2 == 1 );
+	test_case( "* and %", n3 == 3 );
+
+	n1 = 7 % 3 * 2;
+	n2 = (7 % 3) * 2;
+	n3 = 7 % (3 * 2);
+	test_case( "% and *", n1 == 2 );
+	test_case( "% and *", n2 == 2 );
+	test_case( "% and *", n3 == 1 );
+
+	# Tests that show "*" has higher precedence than "+"
+
+	n1 = 1 + 2 * 3;
+	n2 = 1 + (2 * 3);
+	n3 = (1 + 2) * 3;
+	test_case( "+ and *", n1 == 7 );
+	test_case( "+ and *", n2 == 7 );
+	test_case( "+ and *", n3 == 9 );
+
+	# Tests that show "+" has higher precedence than "<"
+
+	test_case( "< and +", 5 < 3 + 7 );
+	test_case( "< and +", 5 < (3 + 7) );
+
+	test_case( "+ and <", 7 + 3 > 5 );
+	test_case( "+ and <", (7 + 3) > 5 );
+
+	# Tests that show "+" has higher precedence than "+="
+
+	n1 = n2 = n3 = 0;
+	n1 += 1 + 2;
+	n2 += (1 + 2);
+	(n3 += 1) + 2;
+	test_case( "+= and +", n1 == 3 );
+	test_case( "+= and +", n2 == 3 );
+	test_case( "+= and +", n3 == 1 );
+
+	local r1: bool;
+	local r2: bool;
+	local r3: bool;
+
+	# Tests that show "&&" has higher precedence than "||"
+
+	r1 = F && F || T;
+	r2 = (F && F) || T;
+	r3 = F && (F || T);
+	test_case( "&& and ||", r1 );
+	test_case( "&& and ||", r2 );
+	test_case( "&& and ||", !r3 );
+
+	r1 = T || F && F;
+	r2 = T || (F && F);
+	r3 = (T || F) && F;
+	test_case( "|| and &&", r1 );
+	test_case( "|| and &&", r2 );
+	test_case( "|| and &&", !r3 );
+
+	# Tests that show "||" has higher precedence than conditional operator
+
+	r1 = T || T ? F : F;
+	r2 = (T || T) ? F : F;
+	r3 = T || (T ? F : F);
+	test_case( "|| and conditional operator", !r1 );
+	test_case( "|| and conditional operator", !r2 );
+	test_case( "|| and conditional operator", r3 );
+
+	r1 = T ? F : F || T;
+	r2 = T ? F : (F || T);
+	r3 = (T ? F : F) || T;
+	test_case( "conditional operator and ||", !r1 );
+	test_case( "conditional operator and ||", !r2 );
+	test_case( "conditional operator and ||", r3 );
+
+}
+

From 44c6ed5e8cb216028377c071902956b68ba48f9e Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 29 Aug 2012 17:53:37 -0500
Subject: [PATCH 180/238] Update language tests

Updated the int and count max/min constant value tests based on latest fixes
in master.
---
 testing/btest/Baseline/language.count/out |  4 ++--
 testing/btest/Baseline/language.int/out   |  6 +++---
 testing/btest/language/count.bro          |  8 ++++----
 testing/btest/language/int.bro            | 12 ++++++------
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/testing/btest/Baseline/language.count/out b/testing/btest/Baseline/language.count/out
index a5de991e22..cab1ff90df 100644
--- a/testing/btest/Baseline/language.count/out
+++ b/testing/btest/Baseline/language.count/out
@@ -13,5 +13,5 @@ modulus operator (PASS)
 division operator (PASS)
 assignment operator (PASS)
 assignment operator (PASS)
-max count value = 4294967295 (PASS)
-max count value = 9223372036854775807 (PASS)
+max count value = 18446744073709551615 (PASS)
+max count value = 18446744073709551615 (PASS)
diff --git a/testing/btest/Baseline/language.int/out b/testing/btest/Baseline/language.int/out
index 223d520e25..6defb35b20 100644
--- a/testing/btest/Baseline/language.int/out
+++ b/testing/btest/Baseline/language.int/out
@@ -16,8 +16,8 @@ modulus operator (PASS)
 division operator (PASS)
 assignment operator (PASS)
 assignment operator (PASS)
-max int value = 4294967295 (PASS)
-min int value = -4294967295 (PASS)
 max int value = 9223372036854775807 (PASS)
-min int value = -9223372036854775807 (PASS)
+min int value = -9223372036854775808 (PASS)
+max int value = 9223372036854775807 (PASS)
+min int value = -9223372036854775808 (PASS)
 type inference (PASS)
diff --git a/testing/btest/language/count.bro b/testing/btest/language/count.bro
index 97fb13ce51..e58fb47b54 100644
--- a/testing/btest/language/count.bro
+++ b/testing/btest/language/count.bro
@@ -13,8 +13,8 @@ event bro_init()
 	local c2: count = 5;
 	local c3: count = 0xFF;
 	local c4: count = 255;
-	local c5: count = 4294967295;  # maximum allowed value
-	local c6: count = 0x7fffffffffffffff;  # maximum allowed value
+	local c5: count = 18446744073709551615;  # maximum allowed value
+	local c6: count = 0xffffffffffffffff;    # maximum allowed value
 	local c7: counter = 5;
 
 	test_case( "inequality operator", c1 != c2 );
@@ -35,9 +35,9 @@ event bro_init()
 	c2 -= 2;
 	test_case( "assignment operator", c2 == 6 );
 	local str1 = fmt("max count value = %d", c5);
-	test_case( str1, str1 == "max count value = 4294967295" );
+	test_case( str1, str1 == "max count value = 18446744073709551615" );
 	local str2 = fmt("max count value = %d", c6);
-	test_case( str2, str2 == "max count value = 9223372036854775807" );
+	test_case( str2, str2 == "max count value = 18446744073709551615" );
 
 	# type inference
 	local x = 1;
diff --git a/testing/btest/language/int.bro b/testing/btest/language/int.bro
index 7cc91dd9d8..03dd52b404 100644
--- a/testing/btest/language/int.bro
+++ b/testing/btest/language/int.bro
@@ -19,10 +19,10 @@ event bro_init()
 	local i8: int = 0xC;
 	local i9: int = -0xC;
 	local i10: int = -12;
-	local i11: int = 4294967295;   # max. allowed value
-	local i12: int = -4294967295;  # min. allowed value
+	local i11: int = 9223372036854775807;   # max. allowed value
+	local i12: int = -9223372036854775808;  # min. allowed value
 	local i13: int = 0x7fffffffffffffff;   # max. allowed value
-	local i14: int = -0x7fffffffffffffff;  # min. allowed value
+	local i14: int = -0x8000000000000000;  # min. allowed value
 
 	test_case( "optional '+' sign", i1 == i2 );
 	test_case( "negative vs. positive", i1 != i3 );
@@ -45,13 +45,13 @@ event bro_init()
 	i2 -= 2;
 	test_case( "assignment operator", i2 == 5 );
 	local str1 = fmt("max int value = %d", i11);
-	test_case( str1, str1 == "max int value = 4294967295" );
+	test_case( str1, str1 == "max int value = 9223372036854775807" );
 	local str2 = fmt("min int value = %d", i12);
-	test_case( str2, str2 == "min int value = -4294967295" );
+	test_case( str2, str2 == "min int value = -9223372036854775808" );
 	local str3 = fmt("max int value = %d", i13);
 	test_case( str3, str3 == "max int value = 9223372036854775807" );
 	local str4 = fmt("min int value = %d", i14);
-	test_case( str4, str4 == "min int value = -9223372036854775807" );
+	test_case( str4, str4 == "min int value = -9223372036854775808" );
 
 	# type inference
 	local x = +3;

From 05ad3f95afd1e27e8899c582ecc17d722080ad45 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 31 Aug 2012 14:05:02 -0500
Subject: [PATCH 181/238] Add more language tests

Added more tests and fixed a broken test.
---
 testing/btest/Baseline/language.at-if/out     |  3 ++
 testing/btest/Baseline/language.at-ifdef/out  |  3 ++
 testing/btest/Baseline/language.at-ifndef/out |  3 ++
 testing/btest/Baseline/language.at-load/out   |  4 ++
 testing/btest/Baseline/language.no-module/out |  4 ++
 testing/btest/Baseline/language.set/out       |  8 +++
 testing/btest/Baseline/language.table/out     |  4 ++
 testing/btest/Baseline/language.vector/out    |  1 +
 testing/btest/Baseline/language.when/out      |  1 -
 testing/btest/language/at-if.bro              | 49 ++++++++++++++++++
 testing/btest/language/at-ifdef.bro           | 50 +++++++++++++++++++
 testing/btest/language/at-ifndef.bro          | 50 +++++++++++++++++++
 testing/btest/language/at-load.bro            | 43 ++++++++++++++++
 testing/btest/language/no-module.bro          | 34 +++++++++++++
 testing/btest/language/set.bro                | 15 ++++++
 testing/btest/language/table.bro              |  7 +++
 testing/btest/language/vector.bro             |  4 ++
 testing/btest/language/when.bro               |  2 +-
 18 files changed, 283 insertions(+), 2 deletions(-)
 create mode 100644 testing/btest/Baseline/language.at-if/out
 create mode 100644 testing/btest/Baseline/language.at-ifdef/out
 create mode 100644 testing/btest/Baseline/language.at-ifndef/out
 create mode 100644 testing/btest/Baseline/language.at-load/out
 create mode 100644 testing/btest/Baseline/language.no-module/out
 create mode 100644 testing/btest/language/at-if.bro
 create mode 100644 testing/btest/language/at-ifdef.bro
 create mode 100644 testing/btest/language/at-ifndef.bro
 create mode 100644 testing/btest/language/at-load.bro
 create mode 100644 testing/btest/language/no-module.bro

diff --git a/testing/btest/Baseline/language.at-if/out b/testing/btest/Baseline/language.at-if/out
new file mode 100644
index 0000000000..b63cbbb714
--- /dev/null
+++ b/testing/btest/Baseline/language.at-if/out
@@ -0,0 +1,3 @@
+@if (PASS)
+@if...@else (PASS)
+@if...@else (PASS)
diff --git a/testing/btest/Baseline/language.at-ifdef/out b/testing/btest/Baseline/language.at-ifdef/out
new file mode 100644
index 0000000000..644a42d407
--- /dev/null
+++ b/testing/btest/Baseline/language.at-ifdef/out
@@ -0,0 +1,3 @@
+@ifdef (PASS)
+@ifdef...@else (PASS)
+@ifdef...@else (PASS)
diff --git a/testing/btest/Baseline/language.at-ifndef/out b/testing/btest/Baseline/language.at-ifndef/out
new file mode 100644
index 0000000000..70abba9b3f
--- /dev/null
+++ b/testing/btest/Baseline/language.at-ifndef/out
@@ -0,0 +1,3 @@
+@ifndef (PASS)
+@ifndef...@else (PASS)
+@ifndef...@else (PASS)
diff --git a/testing/btest/Baseline/language.at-load/out b/testing/btest/Baseline/language.at-load/out
new file mode 100644
index 0000000000..5b011543b5
--- /dev/null
+++ b/testing/btest/Baseline/language.at-load/out
@@ -0,0 +1,4 @@
+function (PASS)
+global variable (PASS)
+const (PASS)
+event (PASS)
diff --git a/testing/btest/Baseline/language.no-module/out b/testing/btest/Baseline/language.no-module/out
new file mode 100644
index 0000000000..5b011543b5
--- /dev/null
+++ b/testing/btest/Baseline/language.no-module/out
@@ -0,0 +1,4 @@
+function (PASS)
+global variable (PASS)
+const (PASS)
+event (PASS)
diff --git a/testing/btest/Baseline/language.set/out b/testing/btest/Baseline/language.set/out
index b4801ac799..fc157cf7d9 100644
--- a/testing/btest/Baseline/language.set/out
+++ b/testing/btest/Baseline/language.set/out
@@ -1,3 +1,7 @@
+type inference (PASS)
+type inference (PASS)
+type inference (PASS)
+cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
@@ -24,6 +28,10 @@ add element (PASS)
 in operator (PASS)
 add element (PASS)
 in operator (PASS)
+add element (PASS)
+in operator (PASS)
+remove element (PASS)
+!in operator (PASS)
 remove element (PASS)
 !in operator (PASS)
 remove element (PASS)
diff --git a/testing/btest/Baseline/language.table/out b/testing/btest/Baseline/language.table/out
index 8a45707e2d..5d32cb29fd 100644
--- a/testing/btest/Baseline/language.table/out
+++ b/testing/btest/Baseline/language.table/out
@@ -1,3 +1,7 @@
+type inference (PASS)
+type inference (PASS)
+type inference (PASS)
+cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
diff --git a/testing/btest/Baseline/language.vector/out b/testing/btest/Baseline/language.vector/out
index 4196b36141..4bf909725c 100644
--- a/testing/btest/Baseline/language.vector/out
+++ b/testing/btest/Baseline/language.vector/out
@@ -1,3 +1,4 @@
+type inference (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
diff --git a/testing/btest/Baseline/language.when/out b/testing/btest/Baseline/language.when/out
index 3a052217ab..19f86f493a 100644
--- a/testing/btest/Baseline/language.when/out
+++ b/testing/btest/Baseline/language.when/out
@@ -1,2 +1 @@
 done
-lookup successful
diff --git a/testing/btest/language/at-if.bro b/testing/btest/language/at-if.bro
new file mode 100644
index 0000000000..979ed0bb9a
--- /dev/null
+++ b/testing/btest/language/at-if.bro
@@ -0,0 +1,49 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+
+event bro_init()
+{
+	local xyz = 0;
+
+	# Test "if" without "else"
+
+	@if ( F )
+		xyz += 1;
+	@endif
+
+	@if ( T )
+		xyz += 2;
+	@endif
+
+	test_case( "@if", xyz == 2 );
+
+	# Test "if" with an "else"
+
+	xyz = 0;
+
+	@if ( F )
+		xyz += 1;
+	@else
+		xyz += 2;
+	@endif
+
+	test_case( "@if...@else", xyz == 2 );
+
+	xyz = 0;
+
+	@if ( T )
+		xyz += 1;
+	@else
+		xyz += 2;
+	@endif
+
+	test_case( "@if...@else", xyz == 1 );
+
+}
+
diff --git a/testing/btest/language/at-ifdef.bro b/testing/btest/language/at-ifdef.bro
new file mode 100644
index 0000000000..c30236f204
--- /dev/null
+++ b/testing/btest/language/at-ifdef.bro
@@ -0,0 +1,50 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+global thisisdefined = 123;
+
+event bro_init()
+{
+	local xyz = 0;
+
+	# Test "ifdef" without "else"
+
+	@ifdef ( notdefined )
+		xyz += 1;
+	@endif
+
+	@ifdef ( thisisdefined )
+		xyz += 2;
+	@endif
+
+	test_case( "@ifdef", xyz == 2 );
+
+	# Test "ifdef" with an "else"
+
+	xyz = 0;
+
+	@ifdef ( doesnotexist )
+		xyz += 1;
+	@else
+		xyz += 2;
+	@endif
+
+	test_case( "@ifdef...@else", xyz == 2 );
+
+	xyz = 0;
+
+	@ifdef ( thisisdefined )
+		xyz += 1;
+	@else
+		xyz += 2;
+	@endif
+
+	test_case( "@ifdef...@else", xyz == 1 );
+
+}
+
diff --git a/testing/btest/language/at-ifndef.bro b/testing/btest/language/at-ifndef.bro
new file mode 100644
index 0000000000..c98287590f
--- /dev/null
+++ b/testing/btest/language/at-ifndef.bro
@@ -0,0 +1,50 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+global thisisdefined = 123;
+
+event bro_init()
+{
+	local xyz = 0;
+
+	# Test "ifndef" without "else"
+
+	@ifndef ( notdefined )
+		xyz += 1;
+	@endif
+
+	@ifndef ( thisisdefined )
+		xyz += 2;
+	@endif
+
+	test_case( "@ifndef", xyz == 1 );
+
+	# Test "ifndef" with an "else"
+
+	xyz = 0;
+
+	@ifndef ( doesnotexist )
+		xyz += 1;
+	@else
+		xyz += 2;
+	@endif
+
+	test_case( "@ifndef...@else", xyz == 1 );
+
+	xyz = 0;
+
+	@ifndef ( thisisdefined )
+		xyz += 1;
+	@else
+		xyz += 2;
+	@endif
+
+	test_case( "@ifndef...@else", xyz == 2 );
+
+}
+
diff --git a/testing/btest/language/at-load.bro b/testing/btest/language/at-load.bro
new file mode 100644
index 0000000000..b51594be16
--- /dev/null
+++ b/testing/btest/language/at-load.bro
@@ -0,0 +1,43 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+# In this script, we try to access each object defined in a "@load"ed script
+
+@load secondtestfile
+
+event bro_init()
+{
+	test_case( "function", T );
+	test_case( "global variable", num == 123 );
+	test_case( "const", daysperyear == 365 );
+	event testevent( "foo" );
+}
+
+
+# @TEST-START-FILE secondtestfile
+
+# In this script, we define some objects to be used in another script
+
+# Note: this script is not listed on the bro command-line (instead, it
+# is "@load"ed from the other script)
+
+global test_case: function(msg: string, expect: bool);
+
+global testevent: event(msg: string);
+
+global num: count = 123;
+
+const daysperyear: count = 365;
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+event testevent(msg: string)
+        {
+	test_case( "event", T );
+        }
+
+# @TEST-END-FILE
+
diff --git a/testing/btest/language/no-module.bro b/testing/btest/language/no-module.bro
new file mode 100644
index 0000000000..eadce66c18
--- /dev/null
+++ b/testing/btest/language/no-module.bro
@@ -0,0 +1,34 @@
+# @TEST-EXEC: bro %INPUT secondtestfile >out
+# @TEST-EXEC: btest-diff out
+
+# This is the same test as "module.bro", but here we omit the module definition
+
+
+global num: count = 123;
+
+const daysperyear: count = 365;
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+event testevent(msg: string)
+        {
+	test_case( "event", T );
+        }
+
+
+# @TEST-START-FILE secondtestfile
+
+# In this script, we try to access each object defined in the other script
+
+event bro_init()
+{
+	test_case( "function", T );
+	test_case( "global variable", num == 123 );
+	test_case( "const", daysperyear == 365 );
+	event testevent( "foo" );
+}
+
+# @TEST-END-FILE
diff --git a/testing/btest/language/set.bro b/testing/btest/language/set.bro
index 66b2ebc3af..bfea2b729b 100644
--- a/testing/btest/language/set.bro
+++ b/testing/btest/language/set.bro
@@ -11,6 +11,7 @@ function test_case(msg: string, expect: bool)
 global s10: set[string] = { "curly", "braces" };
 global s11: set[port, string, bool] = { [10/udp, "curly", F],
 		[11/udp, "braces", T] };
+global s12 = { "more", "curly", "braces" };
 
 event bro_init()
 {
@@ -24,6 +25,11 @@ event bro_init()
 	local s7: set[port, string, bool];
 	local s8 = set( [8/tcp, "type inference", T] );
 
+	# Type inference test
+	test_case( "type inference", type_name(s4) == "set[string]" );
+	test_case( "type inference", type_name(s8) == "set[port,string,bool]" );
+	test_case( "type inference", type_name(s12) == "set[string]" );
+
 	# Test the size of each set
 	test_case( "cardinality", |s1| == 2 );
 	test_case( "cardinality", |s2| == 0 );
@@ -35,6 +41,7 @@ event bro_init()
 	test_case( "cardinality", |s8| == 1 );
 	test_case( "cardinality", |s10| == 2 );
 	test_case( "cardinality", |s11| == 2 );
+	test_case( "cardinality", |s12| == 3 );
 
 	# Test iterating over each set
 	local ct: count;
@@ -94,6 +101,10 @@ event bro_init()
 	test_case( "add element", |s10| == 3 );
 	test_case( "in operator", "global" in s10 );
 
+	add s12["more global"];
+	test_case( "add element", |s12| == 4 );
+	test_case( "in operator", "more global" in s12 );
+
 	# Test removing elements from each set
 	delete s1["test"];
 	delete s1["foobar"];  # element does not exist
@@ -117,5 +128,9 @@ event bro_init()
 	delete s10["braces"];
 	test_case( "remove element", |s10| == 2 );
 	test_case( "!in operator", "braces" !in s10 );
+
+	delete s12["curly"];
+	test_case( "remove element", |s12| == 3 );
+	test_case( "!in operator", "curly" !in s12 );
 }
 
diff --git a/testing/btest/language/table.bro b/testing/btest/language/table.bro
index d7fc677a6d..83f9377d68 100644
--- a/testing/btest/language/table.bro
+++ b/testing/btest/language/table.bro
@@ -6,6 +6,7 @@ function test_case(msg: string, expect: bool)
         print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
         }
 
+global t11 = { [1] = "type", [2] = "inference", [3] = "test" };
 
 event bro_init()
 {
@@ -24,6 +25,11 @@ event bro_init()
 		[10/udp, "curly", F] = "first",
 		[11/udp, "braces", T] = "second" };
 
+	# Type inference test
+	test_case( "type inference", type_name(t4) == "table[count] of string" );
+	test_case( "type inference", type_name(t9) == "table[port,string,bool] of string" );
+	test_case( "type inference", type_name(t11) == "table[count] of string" );
+
 	# Test the size of each table
 	test_case( "cardinality", |t1| == 2 );
 	test_case( "cardinality", |t2| == 0 );
@@ -35,6 +41,7 @@ event bro_init()
 	test_case( "cardinality", |t8| == 0 );
 	test_case( "cardinality", |t9| == 1 );
 	test_case( "cardinality", |t10| == 2 );
+	test_case( "cardinality", |t11| == 3 );
 
 	# Test iterating over each table
 	local ct: count;
diff --git a/testing/btest/language/vector.bro b/testing/btest/language/vector.bro
index 320736238e..d09b474b08 100644
--- a/testing/btest/language/vector.bro
+++ b/testing/btest/language/vector.bro
@@ -17,6 +17,10 @@ event bro_init()
 	local v3: vector of string;
 	local v4 = vector( "type inference" );
 
+	# Type inference test
+
+	test_case( "type inference", type_name(v4) == "vector of string" );
+
 	# Test the size of each vector
 
 	test_case( "cardinality", |v1| == 2 );
diff --git a/testing/btest/language/when.bro b/testing/btest/language/when.bro
index 9ad45ab49b..d6b08b67e1 100644
--- a/testing/btest/language/when.bro
+++ b/testing/btest/language/when.bro
@@ -4,7 +4,7 @@
 
 event bro_init()
 {
-	local h1: addr = 1.2.3.4;
+	local h1: addr = 127.0.0.1;
 
 	when ( local h1name = lookup_addr(h1) )
 		{ 

From 76420e4b618899ba26e022fb3cb4d8ddd8612d06 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Sun, 2 Sep 2012 12:55:31 -0500
Subject: [PATCH 182/238] Add more language tests

---
 testing/btest/Baseline/language.vector/out | 26 ++++++++
 testing/btest/language/string.bro          |  6 +-
 testing/btest/language/vector.bro          | 76 +++++++++++++++++++---
 3 files changed, 96 insertions(+), 12 deletions(-)

diff --git a/testing/btest/Baseline/language.vector/out b/testing/btest/Baseline/language.vector/out
index 4bf909725c..54d19346d3 100644
--- a/testing/btest/Baseline/language.vector/out
+++ b/testing/btest/Baseline/language.vector/out
@@ -1,9 +1,22 @@
 type inference (PASS)
+type inference (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+cardinality (PASS)
+zero-based indexing (PASS)
 iterate over vector (PASS)
 iterate over vector (PASS)
 iterate over vector (PASS)
@@ -19,6 +32,8 @@ add element (PASS)
 access element (PASS)
 add element (PASS)
 access element (PASS)
+add element (PASS)
+access element (PASS)
 overwrite element (PASS)
 access element (PASS)
 overwrite element (PASS)
@@ -30,3 +45,14 @@ overwrite element (PASS)
 access element (PASS)
 overwrite element (PASS)
 access element (PASS)
+overwrite element (PASS)
+access element (PASS)
+++ operator (PASS)
+-- operator (PASS)
++ operator (PASS)
+- operator (PASS)
+* operator (PASS)
+/ operator (PASS)
+% operator (PASS)
+&& operator (PASS)
+|| operator (PASS)
diff --git a/testing/btest/language/string.bro b/testing/btest/language/string.bro
index b9a17e3645..eb3757ed70 100644
--- a/testing/btest/language/string.bro
+++ b/testing/btest/language/string.bro
@@ -15,9 +15,9 @@ event bro_init()
 	local s4: string = "a\tb";    # tab
 	local s5: string = "a\nb";    # newline
 	local s6: string = "a\xffb";  # hex value
-	local s7: string = "a\x00b";  # hex value
-	local s8: string = "a\x0ab";  # hex value
-	local s9: string = "a\011b";  # octal value
+	local s7: string = "a\x00b";  # hex value (null character)
+	local s8: string = "a\x0ab";  # hex value (newline character)
+	local s9: string = "a\011b";  # octal value (tab character)
 	local s10: string = "a\"b";   # double quote
 	local s11: string = "a\\b";   # backslash
 	local s12: string = s2 + s3;  # string concatenation
diff --git a/testing/btest/language/vector.bro b/testing/btest/language/vector.bro
index d09b474b08..2e3ecb8eee 100644
--- a/testing/btest/language/vector.bro
+++ b/testing/btest/language/vector.bro
@@ -8,7 +8,7 @@ function test_case(msg: string, expect: bool)
 
 
 # Note: only global vectors can be initialized with curly braces
-global v5: vector of string = { "curly", "braces" };
+global v20: vector of string = { "curly", "braces" };
 
 event bro_init()
 {
@@ -16,10 +16,22 @@ event bro_init()
 	local v2: vector of string = vector();
 	local v3: vector of string;
 	local v4 = vector( "type inference" );
+	local v5 = vector( 1, 2, 3 );
+	local v6 = vector( 10, 20, 30 );
+	local v7 = v5 + v6;
+	local v8 = v6 - v5;
+	local v9 = v5 * v6;
+	local v10 = v6 / v5;
+	local v11 = v6 % v5;
+	local v12 = vector( T, F, T );
+	local v13 = vector( F, F, T );
+	local v14 = v12 && v13;
+	local v15 = v12 || v13;
 
 	# Type inference test
 
 	test_case( "type inference", type_name(v4) == "vector of string" );
+	test_case( "type inference", type_name(v5) == "vector of count" );
 
 	# Test the size of each vector
 
@@ -27,7 +39,22 @@ event bro_init()
 	test_case( "cardinality", |v2| == 0 );
 	test_case( "cardinality", |v3| == 0 );
 	test_case( "cardinality", |v4| == 1 );
-	test_case( "cardinality", |v5| == 2 );
+	test_case( "cardinality", |v5| == 3 );
+	test_case( "cardinality", |v6| == 3 );
+	test_case( "cardinality", |v7| == 3 );
+	test_case( "cardinality", |v8| == 3 );
+	test_case( "cardinality", |v9| == 3 );
+	test_case( "cardinality", |v10| == 3 );
+	test_case( "cardinality", |v11| == 3 );
+	test_case( "cardinality", |v12| == 3 );
+	test_case( "cardinality", |v13| == 3 );
+	test_case( "cardinality", |v14| == 3 );
+	test_case( "cardinality", |v15| == 3 );
+	test_case( "cardinality", |v20| == 2 );
+
+	# Test that vectors use zero-based indexing
+
+	test_case( "zero-based indexing", v1[0] == "test" && v5[0] == 1 );
 
 	# Test iterating over each vector
 
@@ -51,7 +78,7 @@ event bro_init()
 	test_case( "iterate over vector", ct == 0 );
 
 	ct = 0;
-	for ( c in v5 )
+	for ( c in v20 )
 	{
 		++ct;
 	}
@@ -78,9 +105,13 @@ event bro_init()
 	test_case( "add element", |v4| == 2 );
 	test_case( "access element", v4[1] == "local" );
 
-	v5[2] = "global";
-	test_case( "add element", |v5| == 3 );
-	test_case( "access element", v5[2] == "global" );
+	v5[3] = 77;
+	test_case( "add element", |v5| == 4 );
+	test_case( "access element", v5[3] == 77 );
+
+	v20[2] = "global";
+	test_case( "add element", |v20| == 3 );
+	test_case( "access element", v20[2] == "global" );
 
 	# Test overwriting elements of each vector
 
@@ -101,8 +132,35 @@ event bro_init()
 	test_case( "overwrite element", |v4| == 2 );
 	test_case( "access element", v4[0] == "new4" );
 
-	v5[1] = "new5";
-	test_case( "overwrite element", |v5| == 3 );
-	test_case( "access element", v5[1] == "new5" );
+	v5[0] = 0;
+	test_case( "overwrite element", |v5| == 4 );
+	test_case( "access element", v5[0] == 0 );
+
+	v20[1] = "new5";
+	test_case( "overwrite element", |v20| == 3 );
+	test_case( "access element", v20[1] == "new5" );
+
+	# Test increment/decrement operators
+
+	++v5;
+	test_case( "++ operator", |v5| == 4 && v5[0] == 1 && v5[1] == 3
+			 && v5[2] == 4 && v5[3] == 78 );
+	--v5;
+	test_case( "-- operator", |v5| == 4 && v5[0] == 0 && v5[1] == 2
+			 && v5[2] == 3 && v5[3] == 77 );
+
+	# Test +,-,*,/,% of two vectors
+
+	test_case( "+ operator", v7[0] == 11 && v7[1] == 22 && v7[2] == 33 );
+	test_case( "- operator", v8[0] == 9 && v8[1] == 18 && v8[2] == 27 );
+	test_case( "* operator", v9[0] == 10 && v9[1] == 40 && v9[2] == 90 );
+	test_case( "/ operator", v10[0] == 10 && v10[1] == 10 && v10[2] == 10 );
+	test_case( "% operator", v11[0] == 0 && v11[1] == 0 && v11[2] == 0 );
+
+	# Test &&,|| of two vectors
+
+	test_case( "&& operator", v14[0] == F && v14[1] == F && v14[2] == T );
+	test_case( "|| operator", v15[0] == T && v15[1] == F && v15[2] == T );
+
 }
 

From d5bf5eb38c56860cbcb4232c26343d8182b7634f Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 4 Sep 2012 17:39:00 -0500
Subject: [PATCH 183/238] Add more language tests

---
 testing/btest/Baseline/language.addr/out     |  2 +
 testing/btest/Baseline/language.any/out      | 14 +++
 testing/btest/Baseline/language.bool/out     |  2 +
 testing/btest/Baseline/language.count/out    |  5 +-
 testing/btest/Baseline/language.double/out   |  3 +
 testing/btest/Baseline/language.int/out      |  2 +-
 testing/btest/Baseline/language.interval/out |  4 +-
 testing/btest/Baseline/language.pattern/out  |  2 +
 testing/btest/Baseline/language.port/out     |  1 +
 testing/btest/Baseline/language.string/out   | 33 ++++---
 testing/btest/Baseline/language.subnet/out   |  2 +
 testing/btest/Baseline/language.table/out    |  1 +
 testing/btest/Baseline/language.time/out     |  2 +-
 testing/btest/Baseline/language.vector/out   |  1 +
 testing/btest/language/addr.bro              |  7 +-
 testing/btest/language/any.bro               | 40 ++++++++
 testing/btest/language/bool.bro              |  7 +-
 testing/btest/language/count.bro             | 22 ++++-
 testing/btest/language/double.bro            | 21 ++++-
 testing/btest/language/file.bro              |  4 +-
 testing/btest/language/int.bro               | 18 +++-
 testing/btest/language/interval.bro          | 38 ++++++--
 testing/btest/language/pattern.bro           | 12 ++-
 testing/btest/language/port.bro              | 27 +++---
 testing/btest/language/set.bro               | 58 ++++++------
 testing/btest/language/string.bro            | 99 +++++++++++---------
 testing/btest/language/subnet.bro            |  9 +-
 testing/btest/language/table.bro             | 31 ++++--
 testing/btest/language/time.bro              | 15 ++-
 testing/btest/language/vector.bro            | 21 +++--
 30 files changed, 341 insertions(+), 162 deletions(-)
 create mode 100644 testing/btest/Baseline/language.any/out
 create mode 100644 testing/btest/language/any.bro

diff --git a/testing/btest/Baseline/language.addr/out b/testing/btest/Baseline/language.addr/out
index 79a88d6dcb..b04aac5ce3 100644
--- a/testing/btest/Baseline/language.addr/out
+++ b/testing/btest/Baseline/language.addr/out
@@ -3,6 +3,7 @@ IPv4 address equality (PASS)
 IPv4 address comparison (PASS)
 IPv4 address comparison (PASS)
 size of IPv4 address (PASS)
+IPv4 address type inference (PASS)
 IPv6 address inequality (PASS)
 IPv6 address equality (PASS)
 IPv6 address equality (PASS)
@@ -10,4 +11,5 @@ IPv6 address comparison (PASS)
 IPv6 address comparison (PASS)
 IPv6 address not case-sensitive (PASS)
 size of IPv6 address (PASS)
+IPv6 address type inference (PASS)
 IPv4 and IPv6 address inequality (PASS)
diff --git a/testing/btest/Baseline/language.any/out b/testing/btest/Baseline/language.any/out
new file mode 100644
index 0000000000..4072ce3745
--- /dev/null
+++ b/testing/btest/Baseline/language.any/out
@@ -0,0 +1,14 @@
+count (PASS)
+string (PASS)
+pattern (PASS)
+bool (PASS)
+string (PASS)
+count (PASS)
+int (PASS)
+double (PASS)
+pattern (PASS)
+addr (PASS)
+addr (PASS)
+subnet (PASS)
+subnet (PASS)
+port (PASS)
diff --git a/testing/btest/Baseline/language.bool/out b/testing/btest/Baseline/language.bool/out
index 177c6795ef..9e4c6c3d6e 100644
--- a/testing/btest/Baseline/language.bool/out
+++ b/testing/btest/Baseline/language.bool/out
@@ -5,3 +5,5 @@ logical and operator (PASS)
 negation operator (PASS)
 absolute value (PASS)
 absolute value (PASS)
+type inference (PASS)
+type inference (PASS)
diff --git a/testing/btest/Baseline/language.count/out b/testing/btest/Baseline/language.count/out
index cab1ff90df..4ef65b6098 100644
--- a/testing/btest/Baseline/language.count/out
+++ b/testing/btest/Baseline/language.count/out
@@ -1,10 +1,11 @@
+type inference (PASS)
+counter alias (PASS)
+hexadecimal (PASS)
 inequality operator (PASS)
 relational operator (PASS)
 relational operator (PASS)
 relational operator (PASS)
 relational operator (PASS)
-hexadecimal (PASS)
-counter alias (PASS)
 absolute value (PASS)
 absolute value (PASS)
 pre-increment operator (PASS)
diff --git a/testing/btest/Baseline/language.double/out b/testing/btest/Baseline/language.double/out
index 9711e70d9b..3f70635588 100644
--- a/testing/btest/Baseline/language.double/out
+++ b/testing/btest/Baseline/language.double/out
@@ -1,3 +1,6 @@
+type inference (PASS)
+type inference (PASS)
+type inference (PASS)
 double representations (PASS)
 double representations (PASS)
 double representations (PASS)
diff --git a/testing/btest/Baseline/language.int/out b/testing/btest/Baseline/language.int/out
index 6defb35b20..01f018acbe 100644
--- a/testing/btest/Baseline/language.int/out
+++ b/testing/btest/Baseline/language.int/out
@@ -1,3 +1,4 @@
+type inference (PASS)
 optional '+' sign (PASS)
 negative vs. positive (PASS)
 negative vs. positive (PASS)
@@ -20,4 +21,3 @@ max int value = 9223372036854775807 (PASS)
 min int value = -9223372036854775808 (PASS)
 max int value = 9223372036854775807 (PASS)
 min int value = -9223372036854775808 (PASS)
-type inference (PASS)
diff --git a/testing/btest/Baseline/language.interval/out b/testing/btest/Baseline/language.interval/out
index 3eb135de52..425ae1c15c 100644
--- a/testing/btest/Baseline/language.interval/out
+++ b/testing/btest/Baseline/language.interval/out
@@ -1,6 +1,8 @@
+type inference (PASS)
+type inference (PASS)
 optional space (PASS)
-different units with same numeric value (PASS)
 plural/singular interval are same (PASS)
+different units with same numeric value (PASS)
 compare different time units (PASS)
 compare different time units (PASS)
 compare different time units (PASS)
diff --git a/testing/btest/Baseline/language.pattern/out b/testing/btest/Baseline/language.pattern/out
index 5a31e4eacb..4a5b8de670 100644
--- a/testing/btest/Baseline/language.pattern/out
+++ b/testing/btest/Baseline/language.pattern/out
@@ -1,6 +1,8 @@
+type inference (PASS)
 equality operator (PASS)
 equality operator (order of operands) (PASS)
 inequality operator (PASS)
+inequality operator (order of operands) (PASS)
 in operator (PASS)
 in operator (PASS)
 !in operator (PASS)
diff --git a/testing/btest/Baseline/language.port/out b/testing/btest/Baseline/language.port/out
index 9dd7ba03c2..b307388c35 100644
--- a/testing/btest/Baseline/language.port/out
+++ b/testing/btest/Baseline/language.port/out
@@ -1,3 +1,4 @@
+type inference (PASS)
 protocol ordering (PASS)
 protocol ordering (PASS)
 protocol ordering (PASS)
diff --git a/testing/btest/Baseline/language.string/out b/testing/btest/Baseline/language.string/out
index 623d1cd3ba..5595445ffc 100644
--- a/testing/btest/Baseline/language.string/out
+++ b/testing/btest/Baseline/language.string/out
@@ -1,24 +1,29 @@
+type inference (PASS)
+tab escape sequence (PASS)
+newline escape sequence (PASS)
+double quote escape sequence (PASS)
+backslash escape sequence (PASS)
+1-digit hex escape sequence (PASS)
+2-digit hex escape sequence (PASS)
+2-digit hex escape sequence (PASS)
+2-digit hex escape sequence (PASS)
+3-digit octal escape sequence (PASS)
+2-digit octal escape sequence (PASS)
+1-digit octal escape sequence (PASS)
+tab escape sequence (PASS)
+tab escape sequence (PASS)
+newline escape sequence (PASS)
+newline escape sequence (PASS)
+double quote escape sequence (PASS)
+null escape sequence (PASS)
 empty string (PASS)
 nonempty string (PASS)
 string comparison (PASS)
 string comparison (PASS)
 string comparison (PASS)
 string comparison (PASS)
-null escape sequence (PASS)
-tab escape sequence (PASS)
-newline escape sequence (PASS)
-hex escape sequence (PASS)
-hex escape sequence (PASS)
-hex escape sequence (PASS)
-octal escape sequence (PASS)
-quote escape sequence (PASS)
-backslash escape sequence (PASS)
-null escape sequence (PASS)
-newline escape sequence (PASS)
-tab escape sequence (PASS)
 string concatenation (PASS)
 string concatenation (PASS)
-long string initialization (PASS)
+multi-line string initialization (PASS)
 in operator (PASS)
 !in operator (PASS)
-type inference (PASS)
diff --git a/testing/btest/Baseline/language.subnet/out b/testing/btest/Baseline/language.subnet/out
index f753d65c68..45900a291e 100644
--- a/testing/btest/Baseline/language.subnet/out
+++ b/testing/btest/Baseline/language.subnet/out
@@ -2,9 +2,11 @@ IPv4 subnet equality (PASS)
 IPv4 subnet inequality (PASS)
 IPv4 subnet in operator (PASS)
 IPv4 subnet !in operator (PASS)
+IPv4 subnet type inference (PASS)
 IPv6 subnet equality (PASS)
 IPv6 subnet inequality (PASS)
 IPv6 subnet in operator (PASS)
 IPv6 subnet !in operator (PASS)
+IPv6 subnet type inference (PASS)
 IPv4 and IPv6 subnet inequality (PASS)
 IPv4 address and IPv6 subnet (PASS)
diff --git a/testing/btest/Baseline/language.table/out b/testing/btest/Baseline/language.table/out
index 5d32cb29fd..514cb6b02d 100644
--- a/testing/btest/Baseline/language.table/out
+++ b/testing/btest/Baseline/language.table/out
@@ -17,6 +17,7 @@ iterate over table (PASS)
 iterate over table (PASS)
 iterate over table (PASS)
 iterate over table (PASS)
+overwrite element (PASS)
 add element (PASS)
 in operator (PASS)
 add element (PASS)
diff --git a/testing/btest/Baseline/language.time/out b/testing/btest/Baseline/language.time/out
index 3615a17c53..5e1c8e6b26 100644
--- a/testing/btest/Baseline/language.time/out
+++ b/testing/btest/Baseline/language.time/out
@@ -1,7 +1,7 @@
+type inference (PASS)
 add interval (PASS)
 subtract interval (PASS)
 inequality (PASS)
 equality (PASS)
 subtract time (PASS)
 size operator (PASS)
-type inference (PASS)
diff --git a/testing/btest/Baseline/language.vector/out b/testing/btest/Baseline/language.vector/out
index 54d19346d3..0aa3ab0a8f 100644
--- a/testing/btest/Baseline/language.vector/out
+++ b/testing/btest/Baseline/language.vector/out
@@ -1,5 +1,6 @@
 type inference (PASS)
 type inference (PASS)
+type inference (PASS)
 cardinality (PASS)
 cardinality (PASS)
 cardinality (PASS)
diff --git a/testing/btest/language/addr.bro b/testing/btest/language/addr.bro
index b97710ce22..1cd93bad03 100644
--- a/testing/btest/language/addr.bro
+++ b/testing/btest/language/addr.bro
@@ -13,12 +13,14 @@ event bro_init()
 	local a1: addr = 0.0.0.0;
 	local a2: addr = 10.0.0.11;
 	local a3: addr = 255.255.255.255;
+	local a4 = 192.1.2.3;
 
 	test_case( "IPv4 address inequality", a1 != a2 );
 	test_case( "IPv4 address equality", a1 == 0.0.0.0 );
 	test_case( "IPv4 address comparison", a1 < a2 );
 	test_case( "IPv4 address comparison", a3 > a2 );
 	test_case( "size of IPv4 address", |a1| == 32 );
+	test_case( "IPv4 address type inference", type_name(a4) == "addr" );
 
 	# IPv6 addresses
 	local b1: addr = [::];
@@ -28,6 +30,7 @@ event bro_init()
 	local b5: addr = [0000:0000:0000:0000:0000:0000:0000:0000];
 	local b6: addr = [aaaa:bbbb:cccc:dddd:eeee:ffff:1111:2222];
 	local b7: addr = [AAAA:BBBB:CCCC:DDDD:EEEE:FFFF:1111:2222];
+	local b8 = [a::b];
 
 	test_case( "IPv6 address inequality", b1 != b2 );
 	test_case( "IPv6 address equality", b1 == b5 );
@@ -36,11 +39,9 @@ event bro_init()
 	test_case( "IPv6 address comparison", b4 > b2 );
 	test_case( "IPv6 address not case-sensitive", b6 == b7 );
 	test_case( "size of IPv6 address", |b1| == 128 );
+	test_case( "IPv6 address type inference", type_name(b8) == "addr" );
 
 	test_case( "IPv4 and IPv6 address inequality", a1 != b1 );
 
-	# type inference
-	local x = 192.1.2.3;
-	local y = [a::b];
 }
 
diff --git a/testing/btest/language/any.bro b/testing/btest/language/any.bro
new file mode 100644
index 0000000000..7437ee9851
--- /dev/null
+++ b/testing/btest/language/any.bro
@@ -0,0 +1,40 @@
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+function test_case(msg: string, expect: bool)
+        {
+        print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
+        }
+
+function anyarg(arg1: any, arg1type: string)
+	{
+	test_case( arg1type, type_name(arg1) == arg1type );
+	}
+
+event bro_init()
+{
+	local any1: any = 5;
+	local any2: any = "bar";
+	local any3: any = /bar/;
+
+	# Test using variable of type "any"
+
+	anyarg( any1, "count" );
+	anyarg( any2, "string" );
+	anyarg( any3, "pattern" );
+
+	# Test of other types
+
+	anyarg( T, "bool" );
+	anyarg( "foo", "string" );
+	anyarg( 15, "count" );
+	anyarg( +15, "int" );
+	anyarg( 15.0, "double" );
+	anyarg( /foo/, "pattern" );
+	anyarg( 127.0.0.1, "addr" );
+	anyarg( [::1], "addr" );
+	anyarg( 127.0.0.1/16, "subnet" );
+	anyarg( [ffff::1]/64, "subnet" );
+	anyarg( 123/tcp, "port" );
+}
+
diff --git a/testing/btest/language/bool.bro b/testing/btest/language/bool.bro
index 09614b516e..b75343025f 100644
--- a/testing/btest/language/bool.bro
+++ b/testing/btest/language/bool.bro
@@ -12,6 +12,8 @@ event bro_init()
 	local b1: bool = T; 
 	local b2: bool = F;
 	local b3: bool = T;
+	local b4 = T;
+	local b5 = F;
 
 	test_case( "equality operator", b1 == b3 );
 	test_case( "inequality operator", b1 != b2 );
@@ -20,9 +22,8 @@ event bro_init()
 	test_case( "negation operator", !b2 );
 	test_case( "absolute value", |b1| == 1 );
 	test_case( "absolute value", |b2| == 0 );
+	test_case( "type inference", type_name(b4) == "bool" );
+	test_case( "type inference", type_name(b5) == "bool" );
 
-	# type inference
-	local x = T;
-	local y = F;
 }
 
diff --git a/testing/btest/language/count.bro b/testing/btest/language/count.bro
index e58fb47b54..d6dcf5a97e 100644
--- a/testing/btest/language/count.bro
+++ b/testing/btest/language/count.bro
@@ -16,14 +16,27 @@ event bro_init()
 	local c5: count = 18446744073709551615;  # maximum allowed value
 	local c6: count = 0xffffffffffffffff;    # maximum allowed value
 	local c7: counter = 5;
+	local c8 = 1;
+
+	# Type inference test
+
+	test_case( "type inference", type_name(c8) == "count" );
+
+	# Counter alias test
+
+	test_case( "counter alias", c2 == c7 );
+
+	# Test various constant representations
+
+	test_case( "hexadecimal", c3 == c4 );
+
+	# Operator tests
 
 	test_case( "inequality operator", c1 != c2 );
 	test_case( "relational operator", c1 < c2 );
 	test_case( "relational operator", c1 <= c2 );
 	test_case( "relational operator", c2 > c1 );
 	test_case( "relational operator", c2 >= c1 );
-	test_case( "hexadecimal", c3 == c4 );
-	test_case( "counter alias", c2 == c7 );
 	test_case( "absolute value", |c1| == 0 );
 	test_case( "absolute value", |c2| == 5 );
 	test_case( "pre-increment operator", ++c2 == 6 );
@@ -34,12 +47,13 @@ event bro_init()
 	test_case( "assignment operator", c2 == 8 );
 	c2 -= 2;
 	test_case( "assignment operator", c2 == 6 );
+
+	# Max. value tests
+
 	local str1 = fmt("max count value = %d", c5);
 	test_case( str1, str1 == "max count value = 18446744073709551615" );
 	local str2 = fmt("max count value = %d", c6);
 	test_case( str2, str2 == "max count value = 18446744073709551615" );
 
-	# type inference
-	local x = 1;
 }
 
diff --git a/testing/btest/language/double.bro b/testing/btest/language/double.bro
index f56d291631..62ca768e22 100644
--- a/testing/btest/language/double.bro
+++ b/testing/btest/language/double.bro
@@ -28,6 +28,17 @@ event bro_init()
 	local d17: double = 3.0001;
 	local d18: double = -3.0001;
 	local d19: double = 1.7976931348623157e308;  # maximum allowed value
+	local d20 = 7.0;
+	local d21 = 7e0;
+	local d22 = 7e+1;
+
+	# Type inference tests
+
+	test_case( "type inference", type_name(d20) == "double" );
+	test_case( "type inference", type_name(d21) == "double" );
+	test_case( "type inference", type_name(d22) == "double" );
+
+	# Test various constant representations
 
 	test_case( "double representations", d1 == d2 );
 	test_case( "double representations", d1 == d3 );
@@ -44,6 +55,9 @@ event bro_init()
 	test_case( "double representations", d1 == d14 );
 	test_case( "double representations", d1 == d15 );
 	test_case( "double representations", d1 == d16 );
+
+	# Operator tests
+
 	test_case( "inequality operator", d18 != d17 );
 	test_case( "absolute value", |d18| == d17 );
 	d4 += 2;
@@ -55,12 +69,11 @@ event bro_init()
 	test_case( "relational operator", d17 >= d3 );
 	test_case( "relational operator", d17 > d3 );
 	test_case( "division operator", d3/2 == 1.5 );
+
+	# Max. value test
+
 	local str1 = fmt("max double value = %.16e", d19);
 	test_case( str1, str1 == "max double value = 1.7976931348623157e+308" );
 
-	# type inference
-	local x = 7.0;
-	local y = 7e0;
-	local z = 7e+1;
 }
 
diff --git a/testing/btest/language/file.bro b/testing/btest/language/file.bro
index 77650a6082..1f631eb4fe 100644
--- a/testing/btest/language/file.bro
+++ b/testing/btest/language/file.bro
@@ -5,13 +5,13 @@
 
 event bro_init()
 {
-	# Test using "print" statement to output directly to a file
 	local f1: file = open( "out1" );
 	print f1, 20;
 	print f1, 12;
 	close(f1);
 
-	# Test again, but without explicitly using the type name in declaration
+	# Type inference test
+
 	local f2 = open( "out2" );
 	print f2, "test", 123, 456;
 	close(f2);
diff --git a/testing/btest/language/int.bro b/testing/btest/language/int.bro
index 03dd52b404..5cfa1620bd 100644
--- a/testing/btest/language/int.bro
+++ b/testing/btest/language/int.bro
@@ -15,7 +15,7 @@ event bro_init()
 	local i4: int = +0;
 	local i5: int = -0;
 	local i6: int = 12;
-	local i7: int = 0xc;
+	local i7: int = +0xc;
 	local i8: int = 0xC;
 	local i9: int = -0xC;
 	local i10: int = -12;
@@ -23,6 +23,13 @@ event bro_init()
 	local i12: int = -9223372036854775808;  # min. allowed value
 	local i13: int = 0x7fffffffffffffff;   # max. allowed value
 	local i14: int = -0x8000000000000000;  # min. allowed value
+	local i15 = +3;
+
+	# Type inference test
+
+	test_case( "type inference", type_name(i15) == "int" );
+
+	# Test various constant representations
 
 	test_case( "optional '+' sign", i1 == i2 );
 	test_case( "negative vs. positive", i1 != i3 );
@@ -30,6 +37,9 @@ event bro_init()
 	test_case( "hexadecimal", i6 == i7 );
 	test_case( "hexadecimal", i6 == i8 );
 	test_case( "hexadecimal", i9 == i10 );
+
+	# Operator tests
+
 	test_case( "relational operator", i2 > i3 );
 	test_case( "relational operator", i2 >= i3 );
 	test_case( "relational operator", i3 < i2 );
@@ -44,6 +54,9 @@ event bro_init()
 	test_case( "assignment operator", i2 == 7 );
 	i2 -= 2;
 	test_case( "assignment operator", i2 == 5 );
+
+	# Max/min value tests
+
 	local str1 = fmt("max int value = %d", i11);
 	test_case( str1, str1 == "max int value = 9223372036854775807" );
 	local str2 = fmt("min int value = %d", i12);
@@ -53,8 +66,5 @@ event bro_init()
 	local str4 = fmt("min int value = %d", i14);
 	test_case( str4, str4 == "min int value = -9223372036854775808" );
 
-	# type inference
-	local x = +3;
-	test_case( "type inference", type_name(x) == "int" );
 }
 
diff --git a/testing/btest/language/interval.bro b/testing/btest/language/interval.bro
index 9467db9397..816dfd6416 100644
--- a/testing/btest/language/interval.bro
+++ b/testing/btest/language/interval.bro
@@ -14,7 +14,8 @@ function approx_equal(x: double, y: double): bool
 
 event bro_init()
 {
-	# constants without space and no letter "s"
+	# Constants without space and no letter "s"
+
 	local in11: interval = 2usec;
 	local in12: interval = 2msec;
 	local in13: interval = 120sec;
@@ -23,7 +24,8 @@ event bro_init()
 	# TODO: this one causes bro to fail
 	#local in16: interval = 2.5day;
 
-	# constants with space and no letter "s"
+	# Constants with space and no letter "s"
+
 	local in21: interval = 2 usec;
 	local in22: interval = 2 msec;
 	local in23: interval = 120 sec;
@@ -31,17 +33,36 @@ event bro_init()
 	local in25: interval = -2 hr;
 	local in26: interval = 2.5 day;
 
-	# constants with space and letter "s"
+	# Constants with space and letter "s"
+
 	local in31: interval = 2 usecs;
 	local in32: interval = 2 msecs;
-	local in33: interval = 120 secs;
+	local in33: interval = 1.2e2 secs;
 	local in34: interval = 2 mins;
 	local in35: interval = -2 hrs;
 	local in36: interval = 2.5 days;
 
+	# Type inference
+
+	local in41 = 2 usec;
+	# TODO: this one causes bro to fail
+	#local in42 = 2.1usec;
+	local in43 = 3usecs;
+
+	# Type inference tests
+
+	test_case( "type inference", type_name(in41) == "interval" );
+	#test_case( "type inference", type_name(in42) == "interval" );
+	test_case( "type inference", type_name(in43) == "interval" );
+
+	# Test various constant representations
+
 	test_case( "optional space", in11 == in21 );
-	test_case( "different units with same numeric value", in11 != in12 );
 	test_case( "plural/singular interval are same", in11 == in31 );
+
+	# Operator tests
+
+	test_case( "different units with same numeric value", in11 != in12 );
 	test_case( "compare different time units", in13 == in34 );
 	test_case( "compare different time units", in13 <= in34 );
 	test_case( "compare different time units", in13 >= in34 );
@@ -62,16 +83,13 @@ event bro_init()
 	test_case( "division operator", in35/2 == -1hr );
 	test_case( "division operator", approx_equal(in32/in31, 1e3) );
 
+	# Test relative size of each interval unit
+
 	test_case( "relative size of units", approx_equal(1msec/1usec, 1000) );
 	test_case( "relative size of units", approx_equal(1sec/1msec, 1000) );
 	test_case( "relative size of units", approx_equal(1min/1sec, 60) );
 	test_case( "relative size of units", approx_equal(1hr/1min, 60) );
 	test_case( "relative size of units", approx_equal(1day/1hr, 24) );
 
-	# type inference
-	local x = 2 usec;
-	# TODO: this one causes bro to fail
-	#local y = 2.1usec;
-	local z = 3usecs;
 }
 
diff --git a/testing/btest/language/pattern.bro b/testing/btest/language/pattern.bro
index de33e4d2b6..ec50dc66fe 100644
--- a/testing/btest/language/pattern.bro
+++ b/testing/btest/language/pattern.bro
@@ -12,17 +12,21 @@ event bro_init()
 	local p1: pattern = /foo|bar/; 
 	local p2: pattern = /oob/; 
 	local p3: pattern = /^oob/; 
+	local p4 = /foo/;
+
+	# Type inference tests
+
+	test_case( "type inference", type_name(p4) == "pattern" );
+
+	# Operator tests
 
 	test_case( "equality operator", "foo" == p1 );
 	test_case( "equality operator (order of operands)", p1 == "foo" );
 	test_case( "inequality operator", "foobar" != p1 );
+	test_case( "inequality operator (order of operands)", p1 != "foobar" );
 	test_case( "in operator", p1 in "foobar" );
 	test_case( "in operator", p2 in "foobar" );
 	test_case( "!in operator", p3 !in "foobar" );
 
-	# type inference
-	local x = /foo|bar/;
-	local y = /foo/;
-	local z = /^foo/;
 }
 
diff --git a/testing/btest/language/port.bro b/testing/btest/language/port.bro
index b45401da7a..1874e1dca3 100644
--- a/testing/btest/language/port.bro
+++ b/testing/btest/language/port.bro
@@ -13,23 +13,28 @@ event bro_init()
 	local p2: port = 2/udp;
 	local p3: port = 3/tcp;
 	local p4: port = 4/unknown;
+	local p5 = 123/tcp;
 
 	# maximum allowed values for each port type
-	local p5: port = 255/icmp;
-	local p6: port = 65535/udp;
-	local p7: port = 65535/tcp;
-	local p8: port = 255/unknown;
+	local p6: port = 255/icmp;
+	local p7: port = 65535/udp;
+	local p8: port = 65535/tcp;
+	local p9: port = 255/unknown;
+
+	# Type inference test
+
+	test_case( "type inference", type_name(p5) == "port" );
+
+	# Operator tests
 
 	test_case( "protocol ordering", p1 > p2 );
 	test_case( "protocol ordering", p2 > p3 );
 	test_case( "protocol ordering", p3 > p4 );
-	test_case( "protocol ordering", p7 < p6 );
-	test_case( "protocol ordering", p8 < p5 );
-	test_case( "different protocol but same numeric value", p6 != p7 );
-	test_case( "different protocol but same numeric value", p5 != p8 );
-	test_case( "equality operator", 65535/tcp == p7 );
+	test_case( "protocol ordering", p8 < p7 );
+	test_case( "protocol ordering", p9 < p6 );
+	test_case( "different protocol but same numeric value", p7 != p8 );
+	test_case( "different protocol but same numeric value", p6 != p9 );
+	test_case( "equality operator", 65535/tcp == p8 );
 
-	# type inference
-	local x = 123/tcp;
 }
 
diff --git a/testing/btest/language/set.bro b/testing/btest/language/set.bro
index bfea2b729b..5e56e3b9b8 100644
--- a/testing/btest/language/set.bro
+++ b/testing/btest/language/set.bro
@@ -8,10 +8,10 @@ function test_case(msg: string, expect: bool)
 
 
 # Note: only global sets can be initialized with curly braces
-global s10: set[string] = { "curly", "braces" };
-global s11: set[port, string, bool] = { [10/udp, "curly", F],
+global sg1: set[string] = { "curly", "braces" };
+global sg2: set[port, string, bool] = { [10/udp, "curly", F],
 		[11/udp, "braces", T] };
-global s12 = { "more", "curly", "braces" };
+global sg3 = { "more", "curly", "braces" };
 
 event bro_init()
 {
@@ -25,12 +25,14 @@ event bro_init()
 	local s7: set[port, string, bool];
 	local s8 = set( [8/tcp, "type inference", T] );
 
-	# Type inference test
+	# Type inference tests
+
 	test_case( "type inference", type_name(s4) == "set[string]" );
 	test_case( "type inference", type_name(s8) == "set[port,string,bool]" );
-	test_case( "type inference", type_name(s12) == "set[string]" );
+	test_case( "type inference", type_name(sg3) == "set[string]" );
 
 	# Test the size of each set
+
 	test_case( "cardinality", |s1| == 2 );
 	test_case( "cardinality", |s2| == 0 );
 	test_case( "cardinality", |s3| == 0 );
@@ -39,11 +41,12 @@ event bro_init()
 	test_case( "cardinality", |s6| == 0 );
 	test_case( "cardinality", |s7| == 0 );
 	test_case( "cardinality", |s8| == 1 );
-	test_case( "cardinality", |s10| == 2 );
-	test_case( "cardinality", |s11| == 2 );
-	test_case( "cardinality", |s12| == 3 );
+	test_case( "cardinality", |sg1| == 2 );
+	test_case( "cardinality", |sg2| == 2 );
+	test_case( "cardinality", |sg3| == 3 );
 
 	# Test iterating over each set
+
 	local ct: count;
 	ct = 0;
 	for ( c in s1 )
@@ -69,14 +72,17 @@ event bro_init()
 	test_case( "iterate over set", ct == 2 );
 
 	ct = 0;
-	for ( [c1,c2,c3] in s11 )
+	for ( [c1,c2,c3] in sg2 )
 	{
 		++ct;
 	}
 	test_case( "iterate over set", ct == 2 );
 
-	# Test adding elements to each set
+	# Test adding elements to each set (Note: cannot add elements to sets
+	# of multiple types)
+
 	add s1["added"];
+	add s1["added"];  # element already exists (nothing happens)
 	test_case( "add element", |s1| == 3 );
 	test_case( "in operator", "added" in s1 );
 
@@ -95,19 +101,19 @@ event bro_init()
 	test_case( "add element", |s4| == 2 );
 	test_case( "in operator", "local" in s4 );
 
-	# Note: cannot add elements to sets of multiple types
+	add sg1["global"];
+	test_case( "add element", |sg1| == 3 );
+	test_case( "in operator", "global" in sg1 );
 
-	add s10["global"];
-	test_case( "add element", |s10| == 3 );
-	test_case( "in operator", "global" in s10 );
+	add sg3["more global"];
+	test_case( "add element", |sg3| == 4 );
+	test_case( "in operator", "more global" in sg3 );
 
-	add s12["more global"];
-	test_case( "add element", |s12| == 4 );
-	test_case( "in operator", "more global" in s12 );
+	# Test removing elements from each set (Note: cannot remove elements
+	# from sets of multiple types)
 
-	# Test removing elements from each set
 	delete s1["test"];
-	delete s1["foobar"];  # element does not exist
+	delete s1["foobar"];  # element does not exist (nothing happens)
 	test_case( "remove element", |s1| == 2 );
 	test_case( "!in operator", "test" !in s1 );
 
@@ -123,14 +129,12 @@ event bro_init()
 	test_case( "remove element", |s4| == 1 );
 	test_case( "!in operator", "type inference" !in s4 );
 
-	# Note: cannot remove elements from sets of multiple types
+	delete sg1["braces"];
+	test_case( "remove element", |sg1| == 2 );
+	test_case( "!in operator", "braces" !in sg1 );
 
-	delete s10["braces"];
-	test_case( "remove element", |s10| == 2 );
-	test_case( "!in operator", "braces" !in s10 );
-
-	delete s12["curly"];
-	test_case( "remove element", |s12| == 3 );
-	test_case( "!in operator", "curly" !in s12 );
+	delete sg3["curly"];
+	test_case( "remove element", |sg3| == 3 );
+	test_case( "!in operator", "curly" !in sg3 );
 }
 
diff --git a/testing/btest/language/string.bro b/testing/btest/language/string.bro
index eb3757ed70..3b9137cda5 100644
--- a/testing/btest/language/string.bro
+++ b/testing/btest/language/string.bro
@@ -9,51 +9,66 @@ function test_case(msg: string, expect: bool)
 
 event bro_init()
 {
-	local s1: string = "";        # empty string
-	local s2: string = "x";       # no escape sequences
-	local s3: string = "a\0b";    # null character
-	local s4: string = "a\tb";    # tab
-	local s5: string = "a\nb";    # newline
-	local s6: string = "a\xffb";  # hex value
-	local s7: string = "a\x00b";  # hex value (null character)
-	local s8: string = "a\x0ab";  # hex value (newline character)
-	local s9: string = "a\011b";  # octal value (tab character)
-	local s10: string = "a\"b";   # double quote
-	local s11: string = "a\\b";   # backslash
-	local s12: string = s2 + s3;  # string concatenation
-	local s13: string = "test";
-	local s14: string = "this is a very long string" +
+	local s1: string = "a\ty";    # tab
+	local s2: string = "a\nb";    # newline
+	local s3: string = "a\"b";    # double quote
+	local s4: string = "a\\b";    # backslash
+	local s5: string = "a\x9y";   # 1-digit hex value (tab character)
+	local s6: string = "a\x0ab";  # 2-digit hex value (newline character)
+	local s7: string = "a\x22b";  # 2-digit hex value (double quote)
+	local s8: string = "a\x00b";  # 2-digit hex value (null character)
+	local s9: string = "a\011y";  # 3-digit octal value (tab character)
+	local s10: string = "a\12b";  # 2-digit octal value (newline character)
+	local s11: string = "a\0b";   # 1-digit octal value (null character)
+
+	local s20: string = "";
+	local s21: string = "x";
+	local s22: string = s21 + s11;
+	local s23: string = "test";
+	local s24: string = "this is a very long string" +
 				"which continues on the next line" +
 				"the end";
-	local s15: string = "on";
+	local s25: string = "on";
+	local s26 = "x";
 
-	test_case( "empty string", |s1| == 0 );
-	test_case( "nonempty string", |s2| == 1 );
-	test_case( "string comparison", s2 > s3 );
-	test_case( "string comparison", s2 >= s3 );
-	test_case( "string comparison", s3 < s2 );
-	test_case( "string comparison", s3 <= s2 );
-	test_case( "null escape sequence", |s3| == 3 );
-	test_case( "tab escape sequence", |s4| == 3 );
-	test_case( "newline escape sequence", |s5| == 3 );
-	test_case( "hex escape sequence", |s6| == 3 );
-	test_case( "hex escape sequence", |s7| == 3 );
-	test_case( "hex escape sequence", |s8| == 3 );
-	test_case( "octal escape sequence", |s9| == 3 );
-	test_case( "quote escape sequence", |s10| == 3 );
-	test_case( "backslash escape sequence", |s11| == 3 );
-	test_case( "null escape sequence", s3 == s7 );
-	test_case( "newline escape sequence", s5 == s8 );
-	test_case( "tab escape sequence", s4 == s9 );
-	test_case( "string concatenation", |s12| == 4 );
-	s13 += s2;
-	test_case( "string concatenation", s13 == "testx" );
-	test_case( "long string initialization", |s14| == 65 );
-	test_case( "in operator", s15 in s14 );
-	test_case( "!in operator", s15 !in s13 );
+	# Type inference test
+
+	test_case( "type inference", type_name(s26) == "string" );
+
+	# Escape sequence tests
+
+	test_case( "tab escape sequence", |s1| == 3 );
+	test_case( "newline escape sequence", |s2| == 3 );
+	test_case( "double quote escape sequence", |s3| == 3 );
+	test_case( "backslash escape sequence", |s4| == 3 );
+	test_case( "1-digit hex escape sequence", |s5| == 3 );
+	test_case( "2-digit hex escape sequence", |s6| == 3 );
+	test_case( "2-digit hex escape sequence", |s7| == 3 );
+	test_case( "2-digit hex escape sequence", |s8| == 3 );
+	test_case( "3-digit octal escape sequence", |s9| == 3 );
+	test_case( "2-digit octal escape sequence", |s10| == 3 );
+	test_case( "1-digit octal escape sequence", |s11| == 3 );
+	test_case( "tab escape sequence", s1 == s5 );
+	test_case( "tab escape sequence", s5 == s9 );
+	test_case( "newline escape sequence", s2 == s6 );
+	test_case( "newline escape sequence", s6 == s10 );
+	test_case( "double quote escape sequence", s3 == s7 );
+	test_case( "null escape sequence", s8 == s11 );
+
+	# Operator tests
+
+	test_case( "empty string", |s20| == 0 );
+	test_case( "nonempty string", |s21| == 1 );
+	test_case( "string comparison", s21 > s11 );
+	test_case( "string comparison", s21 >= s11 );
+	test_case( "string comparison", s11 < s21 );
+	test_case( "string comparison", s11 <= s21 );
+	test_case( "string concatenation", |s22| == 4 );
+	s23 += s21;
+	test_case( "string concatenation", s23 == "testx" );
+	test_case( "multi-line string initialization", |s24| == 65 );
+	test_case( "in operator", s25 in s24 );
+	test_case( "!in operator", s25 !in s23 );
 
-	# type inference
-	local x = "x";
-	test_case( "type inference", x == s2 );
 }
 
diff --git a/testing/btest/language/subnet.bro b/testing/btest/language/subnet.bro
index 63d09f916b..591a42119e 100644
--- a/testing/btest/language/subnet.bro
+++ b/testing/btest/language/subnet.bro
@@ -18,13 +18,15 @@ event bro_init()
 	local s1: subnet = 0.0.0.0/0;
 	local s2: subnet = 192.0.0.0/8;
 	local s3: subnet = 255.255.255.255/32;
+	local s4 = 10.0.0.0/16;
 
 	test_case( "IPv4 subnet equality", a1/8 == s2 );
 	test_case( "IPv4 subnet inequality", a1/4 != s2 );
 	test_case( "IPv4 subnet in operator", a1 in s2 );
 	test_case( "IPv4 subnet !in operator", a1 !in s3 );
+	test_case( "IPv4 subnet type inference", type_name(s4) == "subnet" );
 
-	# IPv6 addr
+	# IPv6 addrs
 	local b1: addr = [ffff::];
 	local b2: addr = [ffff::1];
 	local b3: addr = [ffff:1::1];
@@ -32,17 +34,16 @@ event bro_init()
 	# IPv6 subnets
 	local t1: subnet = [::]/0;
 	local t2: subnet = [ffff::]/64;
+	local t3 = [a::]/32;
 
 	test_case( "IPv6 subnet equality", b1/64 == t2 );
 	test_case( "IPv6 subnet inequality", b3/64 != t2 );
 	test_case( "IPv6 subnet in operator", b2 in t2 );
 	test_case( "IPv6 subnet !in operator", b3 !in t2 );
+	test_case( "IPv6 subnet type inference", type_name(t3) == "subnet" );
 
 	test_case( "IPv4 and IPv6 subnet inequality", s1 != t1 );
 	test_case( "IPv4 address and IPv6 subnet", a1 !in t2 );
 
-	# type inference
-	local x = 10.0.0.0/16;
-	local y = [a::]/32;
 }
 
diff --git a/testing/btest/language/table.bro b/testing/btest/language/table.bro
index 83f9377d68..d1b0751970 100644
--- a/testing/btest/language/table.bro
+++ b/testing/btest/language/table.bro
@@ -6,7 +6,9 @@ function test_case(msg: string, expect: bool)
         print fmt("%s (%s)", msg, expect ? "PASS" : "FAIL");
         }
 
-global t11 = { [1] = "type", [2] = "inference", [3] = "test" };
+# Note: only global tables can be initialized with curly braces when the table
+# type is not explicitly specified
+global tg1 = { [1] = "type", [2] = "inference", [3] = "test" };
 
 event bro_init()
 {
@@ -25,12 +27,14 @@ event bro_init()
 		[10/udp, "curly", F] = "first",
 		[11/udp, "braces", T] = "second" };
 
-	# Type inference test
+	# Type inference tests
+
 	test_case( "type inference", type_name(t4) == "table[count] of string" );
 	test_case( "type inference", type_name(t9) == "table[port,string,bool] of string" );
-	test_case( "type inference", type_name(t11) == "table[count] of string" );
+	test_case( "type inference", type_name(tg1) == "table[count] of string" );
 
 	# Test the size of each table
+
 	test_case( "cardinality", |t1| == 2 );
 	test_case( "cardinality", |t2| == 0 );
 	test_case( "cardinality", |t3| == 0 );
@@ -41,9 +45,10 @@ event bro_init()
 	test_case( "cardinality", |t8| == 0 );
 	test_case( "cardinality", |t9| == 1 );
 	test_case( "cardinality", |t10| == 2 );
-	test_case( "cardinality", |t11| == 3 );
+	test_case( "cardinality", |tg1| == 3 );
 
 	# Test iterating over each table
+
 	local ct: count;
 	ct = 0;
 	for ( c in t1 )
@@ -84,7 +89,15 @@ event bro_init()
 	}
 	test_case( "iterate over table", ct == 0 );
 
-	# Test adding elements to each table
+	# Test overwriting elements in each table (Note: cannot overwrite
+	# elements in tables of multiple types)
+
+	t1[5] = "overwrite";
+	test_case( "overwrite element", |t1| == 2 && t1[5] == "overwrite" );
+ 
+	# Test adding elements to each table (Note: cannot add elements to
+	# tables of multiple types)
+
 	t1[1] = "added";
 	test_case( "add element", |t1| == 3 );
 	test_case( "in operator", 1 in t1 );
@@ -108,11 +121,11 @@ event bro_init()
 	test_case( "add element", |t5| == 3 );
 	test_case( "in operator", 10 in t5 );
 
-	# Note: cannot add elements to tables of multiple types
+	# Test removing elements from each table (Note: cannot remove elements
+	# from tables of multiple types)
 
-	# Test removing elements from each table
 	delete t1[0];
-	delete t1[17];  # element does not exist
+	delete t1[17];  # element does not exist (nothing happens)
 	test_case( "remove element", |t1| == 2 );
 	test_case( "!in operator", 0 !in t1 );
 
@@ -132,7 +145,5 @@ event bro_init()
 	test_case( "remove element", |t5| == 2 );
 	test_case( "!in operator", 1 !in t5 );
 
-	# Note: cannot remove elements from tables of multiple types
-
 }
 
diff --git a/testing/btest/language/time.bro b/testing/btest/language/time.bro
index 588cbf8887..43b6694101 100644
--- a/testing/btest/language/time.bro
+++ b/testing/btest/language/time.bro
@@ -13,16 +13,21 @@ event bro_init()
 	local t2: time = t1 + 3 sec;
 	local t3: time = t2 - 10 sec;
 	local t4: time = t1;
-	local t5: interval = t2 - t1;
+	local t5: time = double_to_time(1234567890);
+	local t6 = current_time();	
+
+	# Type inference test
+
+	test_case( "type inference", type_name(t6) == "time" );
+
+	# Operator tests
 
 	test_case( "add interval", t1 < t2 );
 	test_case( "subtract interval", t1 > t3 );
 	test_case( "inequality", t1 != t3 );
 	test_case( "equality", t1 == t4 );
-	test_case( "subtract time", t5 == 3sec);
-	test_case( "size operator", |t1| > 1.0);
+	test_case( "subtract time", t2 - t1 == 3sec);
+	test_case( "size operator", |t5| == 1234567890.0 );
 
-	local x = current_time();	
-	test_case( "type inference", x > t1 );
 }
 
diff --git a/testing/btest/language/vector.bro b/testing/btest/language/vector.bro
index 2e3ecb8eee..928ddcb645 100644
--- a/testing/btest/language/vector.bro
+++ b/testing/btest/language/vector.bro
@@ -8,7 +8,7 @@ function test_case(msg: string, expect: bool)
 
 
 # Note: only global vectors can be initialized with curly braces
-global v20: vector of string = { "curly", "braces" };
+global vg1: vector of string = { "curly", "braces" };
 
 event bro_init()
 {
@@ -28,10 +28,11 @@ event bro_init()
 	local v14 = v12 && v13;
 	local v15 = v12 || v13;
 
-	# Type inference test
+	# Type inference tests
 
 	test_case( "type inference", type_name(v4) == "vector of string" );
 	test_case( "type inference", type_name(v5) == "vector of count" );
+	test_case( "type inference", type_name(v12) == "vector of bool" );
 
 	# Test the size of each vector
 
@@ -50,7 +51,7 @@ event bro_init()
 	test_case( "cardinality", |v13| == 3 );
 	test_case( "cardinality", |v14| == 3 );
 	test_case( "cardinality", |v15| == 3 );
-	test_case( "cardinality", |v20| == 2 );
+	test_case( "cardinality", |vg1| == 2 );
 
 	# Test that vectors use zero-based indexing
 
@@ -78,7 +79,7 @@ event bro_init()
 	test_case( "iterate over vector", ct == 0 );
 
 	ct = 0;
-	for ( c in v20 )
+	for ( c in vg1 )
 	{
 		++ct;
 	}
@@ -109,9 +110,9 @@ event bro_init()
 	test_case( "add element", |v5| == 4 );
 	test_case( "access element", v5[3] == 77 );
 
-	v20[2] = "global";
-	test_case( "add element", |v20| == 3 );
-	test_case( "access element", v20[2] == "global" );
+	vg1[2] = "global";
+	test_case( "add element", |vg1| == 3 );
+	test_case( "access element", vg1[2] == "global" );
 
 	# Test overwriting elements of each vector
 
@@ -136,9 +137,9 @@ event bro_init()
 	test_case( "overwrite element", |v5| == 4 );
 	test_case( "access element", v5[0] == 0 );
 
-	v20[1] = "new5";
-	test_case( "overwrite element", |v20| == 3 );
-	test_case( "access element", v20[1] == "new5" );
+	vg1[1] = "new5";
+	test_case( "overwrite element", |vg1| == 3 );
+	test_case( "access element", vg1[1] == "new5" );
 
 	# Test increment/decrement operators
 

From 63a550fa9e9b2c2a84b0769c683ccd183e10fefb Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 5 Sep 2012 12:00:21 -0500
Subject: [PATCH 184/238] Fix a segfault when iterating over a set

When iterating over a set with a "for" loop, bro would segfault
when the number of index variables was less than required.
Example:  for ( [c1,c2] in s1 ) ...
where s1 is defined as set[addr,port,count].
---
 src/Stmt.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Stmt.cc b/src/Stmt.cc
index 582323bf91..7d754d8e72 100644
--- a/src/Stmt.cc
+++ b/src/Stmt.cc
@@ -943,7 +943,10 @@ ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr)
 		{
 		const type_list* indices = e->Type()->AsTableType()->IndexTypes();
 		if ( indices->length() != loop_vars->length() )
+			{
 			e->Error("wrong index size");
+			return;
+			}
 
 		for ( int i = 0; i < indices->length(); i++ )
 			{

From a10093b620a1dab8d7955b43ac237c40ecfa9bcf Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 5 Sep 2012 16:20:34 -0500
Subject: [PATCH 185/238] Add sleeps to configuration_update test for better
 reliability.

Not the greatest solution, but makes the 3 bro processes more likely to
run sequentially so that the controller2 process doesn't happen to be
scheduled before the controller process.  In that case, the controllee
gets the shutdown request before the configuration update.  FreeBSD
especially seemed to schedule them the unintended way frequently.
---
 .../scripts/base/frameworks/control/configuration_update.bro    | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/testing/btest/scripts/base/frameworks/control/configuration_update.bro b/testing/btest/scripts/base/frameworks/control/configuration_update.bro
index 920a162503..d9e62efe08 100644
--- a/testing/btest/scripts/base/frameworks/control/configuration_update.bro
+++ b/testing/btest/scripts/base/frameworks/control/configuration_update.bro
@@ -1,7 +1,9 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65531/tcp 
+# @TEST-EXEC: sleep 5
 # @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
+# @TEST-EXEC: sleep 5
 # @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff controllee/.stdout

From 9357aeb6b19adc0a3ab4b72de90c347a132cc000 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 5 Sep 2012 16:52:14 -0500
Subject: [PATCH 186/238] Fix "!=" operator for subnets

Fixed a bug where the "!=" operator with subnet operands
was treated the same as the "==" operator.
---
 src/Expr.cc | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/Expr.cc b/src/Expr.cc
index b62f119bae..e58e20f671 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -871,11 +871,12 @@ Val* BinaryExpr::SubNetFold(Val* v1, Val* v2) const
 	{
 	const IPPrefix& n1 = v1->AsSubNet();
 	const IPPrefix& n2 = v2->AsSubNet();
+	bool result = ( n1 == n2 ) ? true : false;
 
-	if ( n1 == n2 )
-		return new Val(1, TYPE_BOOL);
-	else
-		return new Val(0, TYPE_BOOL);
+	if ( tag == EXPR_NE )
+		result = !result;
+
+	return new Val(result, TYPE_BOOL);
 	}
 
 void BinaryExpr::SwapOps()

From cd21eb5b6afe384d044c44a8bb98f3c163532ecb Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 5 Sep 2012 17:17:43 -0500
Subject: [PATCH 187/238] Fix the "-=" operator for intervals

Fixed a bug where "a -= b" (both operands are intervals) was not allowed in
bro scripts (although "a = a - b" is allowed).
---
 src/Expr.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Expr.cc b/src/Expr.cc
index e58e20f671..70aab46ab5 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -1516,6 +1516,8 @@ RemoveFromExpr::RemoveFromExpr(Expr* arg_op1, Expr* arg_op2)
 
 	if ( BothArithmetic(bt1, bt2) )
 		PromoteType(max_type(bt1, bt2), is_vector(op1) || is_vector(op2));
+	else if ( BothInterval(bt1, bt2) )
+		SetType(base_type(bt1));
 	else
 		ExprError("requires two arithmetic operands");
 	}

From 11f66076a18d0fb5ea07a8102c29c9b216698569 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 6 Sep 2012 23:05:57 -0700
Subject: [PATCH 188/238] Starting 2.2 release notes.

---
 NEWS | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/NEWS b/NEWS
index d93e153252..a186fea0fc 100644
--- a/NEWS
+++ b/NEWS
@@ -7,6 +7,20 @@ release. For a complete list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
 their own CHANGES.)
 
+Bro 2.2
+-------
+
+New Functionality
+~~~~~~~~~~~~~~~~~
+
+- TODO: Update.
+
+Changed Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+- TODO: Update.
+
+
 Bro 2.1
 -------
 

From 84ec139fd97114cffc2c19b6110f980a745d8679 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 7 Sep 2012 10:48:13 -0500
Subject: [PATCH 189/238] Update language tests for recent bug fixes

---
 testing/btest/Baseline/language.interval/out | 1 +
 testing/btest/language/interval.bro          | 6 +++---
 testing/btest/language/subnet.bro            | 2 --
 3 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/testing/btest/Baseline/language.interval/out b/testing/btest/Baseline/language.interval/out
index 425ae1c15c..f42082ef5d 100644
--- a/testing/btest/Baseline/language.interval/out
+++ b/testing/btest/Baseline/language.interval/out
@@ -15,6 +15,7 @@ subtract different time units (PASS)
 absolute value (PASS)
 absolute value (PASS)
 assignment operator (PASS)
+assignment operator (PASS)
 multiplication operator (PASS)
 division operator (PASS)
 division operator (PASS)
diff --git a/testing/btest/language/interval.bro b/testing/btest/language/interval.bro
index 816dfd6416..6bdbf3a8e8 100644
--- a/testing/btest/language/interval.bro
+++ b/testing/btest/language/interval.bro
@@ -52,6 +52,7 @@ event bro_init()
 	# Type inference tests
 
 	test_case( "type inference", type_name(in41) == "interval" );
+	# TODO: uncomment when bug is fixed
 	#test_case( "type inference", type_name(in42) == "interval" );
 	test_case( "type inference", type_name(in43) == "interval" );
 
@@ -76,9 +77,8 @@ event bro_init()
 	test_case( "absolute value", |in36| == 2.5*86400 );
 	in34 += 2hr;
 	test_case( "assignment operator", in34 == 122min );
-	# TODO: this should work (subtraction works)
-	#in34 -= 2hr;
-	#test_case( "assignment operator", in34 == 2min );
+	in34 -= 2hr;
+	test_case( "assignment operator", in34 == 2min );
 	test_case( "multiplication operator", in33*2 == 4min );
 	test_case( "division operator", in35/2 == -1hr );
 	test_case( "division operator", approx_equal(in32/in31, 1e3) );
diff --git a/testing/btest/language/subnet.bro b/testing/btest/language/subnet.bro
index 591a42119e..ea641f6983 100644
--- a/testing/btest/language/subnet.bro
+++ b/testing/btest/language/subnet.bro
@@ -7,8 +7,6 @@ function test_case(msg: string, expect: bool)
         }
 
 
-# TODO: "subnet inequality" tests (i.e., tests with "!=") always fail 
-
 event bro_init()
 {
 	# IPv4 addr

From 84fabf1718f85238b5de74a709c0162e526fd82c Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 7 Sep 2012 12:40:25 -0500
Subject: [PATCH 190/238] Add an item to FAQ page about broctl options

---
 doc/faq.rst | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/doc/faq.rst b/doc/faq.rst
index 8545cc57ee..1579fb6313 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -46,7 +46,7 @@ directions:
 http://securityonion.blogspot.com/2011/10/when-is-full-packet-capture-not-full.html
 
 What does an error message like ``internal error: NB-DNS error`` mean?
----------------------------------------------------------------------------------------------------------------------------------
+----------------------------------------------------------------------
 
 That often means that DNS is not set up correctly on the system
 running Bro. Try verifying from the command line that DNS lookups
@@ -65,6 +65,15 @@ Generally, please note that we do not regularly test OpenBSD builds.
 We appreciate any patches that improve Bro's support for this
 platform.
 
+How do broctl options affect Bro script variables?
+--------------------------------------------------
+
+Some (but not all) broctl options override a corresponding Bro script variable.
+For example, setting the broctl option "LogRotationInterval" will override
+the value of the Bro script variable "Log::default_rotation_interval".
+See the :doc:`broctl documentation <components/broctl/README>` to find out
+which broctl options override Bro script variables, and for more discussion
+on site-specific customization.
 
 Usage
 =====

From f6c9b69eda29913c51e09b51daa8ed5a3f416513 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Fri, 7 Sep 2012 10:57:52 -0700
Subject: [PATCH 191/238] reorder a few statements in scan.l to make 1.5msecs
 etc work.

Adresses #872
---
 src/scan.l | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/scan.l b/src/scan.l
index 1b3d09f879..377c74cc1a 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -479,12 +479,6 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 		RET_CONST(new PortVal(p, TRANSPORT_UNKNOWN))
 		}
 
-({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
-
-"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
-
-{H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
-
 {FLOAT}{OWS}day(s?)	RET_CONST(new IntervalVal(atof(yytext),Days))
 {FLOAT}{OWS}hr(s?)	RET_CONST(new IntervalVal(atof(yytext),Hours))
 {FLOAT}{OWS}min(s?)	RET_CONST(new IntervalVal(atof(yytext),Minutes))
@@ -492,6 +486,12 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 {FLOAT}{OWS}msec(s?)	RET_CONST(new IntervalVal(atof(yytext),Milliseconds))
 {FLOAT}{OWS}usec(s?)	RET_CONST(new IntervalVal(atof(yytext),Microseconds))
 
+({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
+
+"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
+
+{H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
+
 \"([^\\\n\"]|{ESCSEQ})*\"	{
 	const char* text = yytext;
 	int len = strlen(text) + 1;

From 67d01ab9e9d1edebb8d7b19795fc07d3023d5b22 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 7 Sep 2012 15:15:48 -0500
Subject: [PATCH 192/238] Small change to non-blocking DNS initialization.

The trailing dot on "localhost." circumvents use of /etc/hosts in
some environments (I saw it on FreeBSD 9.0-RELEASE-p3) and so emits
an actual DNS query.  When running the test suite, that would be
hundreds of useless queries.
---
 src/nb_dns.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/nb_dns.c b/src/nb_dns.c
index d3b3c5c4de..3051be9bc2 100644
--- a/src/nb_dns.c
+++ b/src/nb_dns.c
@@ -124,7 +124,7 @@ nb_dns_init(char *errstr)
 	nd->s = -1;
 
 	/* XXX should be able to init static hostent struct some other way */
-	(void)gethostbyname("localhost.");
+	(void)gethostbyname("localhost");
 
 	if ((_res.options & RES_INIT) == 0 && res_init() == -1) {
 		snprintf(errstr, NB_DNS_ERRSIZE, "res_init() failed");

From bd84ff2c2051ff34a4b2060cce718875e23acf8c Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 7 Sep 2012 16:25:07 -0500
Subject: [PATCH 193/238] Adjusting some unit tests that do cluster
 communication.

Added explicit synchronization and termination points to make the
tests more reliable and exit earlier in most cases.
---
 .../base/frameworks/cluster/start-it-up.bro   | 15 ++++++-
 .../base/frameworks/notice/cluster.bro        | 37 +++++++++++++++--
 .../frameworks/notice/suppression-cluster.bro | 40 +++++++++++++++++--
 3 files changed, 84 insertions(+), 8 deletions(-)

diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
index a1069d1bd0..89f8d6b168 100644
--- a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
+++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
@@ -5,7 +5,7 @@
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff proxy-1/.stdout
 # @TEST-EXEC: btest-diff proxy-2/.stdout
@@ -22,7 +22,20 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+global peer_count = 0;
+
 event remote_connection_handshake_done(p: event_peer)
 	{
 	print "Connected to a peer";
+	if ( Cluster::node == "manager-1" )
+		{
+		peer_count = peer_count + 1;
+		if ( peer_count == 4 )
+			terminate_communication();
+		}
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
 	}
diff --git a/testing/btest/scripts/base/frameworks/notice/cluster.bro b/testing/btest/scripts/base/frameworks/notice/cluster.bro
index 8d54a27eaf..47932edb8e 100644
--- a/testing/btest/scripts/base/frameworks/notice/cluster.bro
+++ b/testing/btest/scripts/base/frameworks/notice/cluster.bro
@@ -2,9 +2,9 @@
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -21,13 +21,44 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready: event();
+
+redef Cluster::manager2worker_events += /ready/;
+
 event delayed_notice()
 	{
 	if ( Cluster::node == "worker-1" )
 		NOTICE([$note=Test_Notice, $msg="test notice!"]);
 	}
 
-event bro_init()
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready()
 	{
 	schedule 1secs { delayed_notice() };
 	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	peer_count = peer_count + 1;
+	if ( peer_count == 2 )
+		event ready();
+	}
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	}
+
+@endif
diff --git a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro
index b812c6451d..5010da82cc 100644
--- a/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro
@@ -2,10 +2,10 @@
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -23,6 +23,15 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready: event();
+
+redef Cluster::manager2worker_events += /ready/;
+
 event delayed_notice()
 	{
 	NOTICE([$note=Test_Notice,
@@ -30,10 +39,33 @@ event delayed_notice()
 	        $identifier="this identifier is static"]);
 	}
 
-event bro_init() &priority=5
-	{
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready()
+    {
 	if ( Cluster::node == "worker-1" )
 		schedule 4secs { delayed_notice() };
 	if ( Cluster::node == "worker-2" )
 		schedule 1secs { delayed_notice() };
+    }
+
+event Notice::suppressed(n: Notice::Info)
+	{
+	if ( Cluster::node == "worker-1" )
+		terminate_communication();
 	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		event ready();
+	}
+
+@endif

From 292bf61ae8cbdae6773b675ad5d33884c7fc7fd4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 13 Sep 2012 12:59:40 -0500
Subject: [PATCH 194/238] Unit test reliability adjustment.

Sometimes manager node was shutting everything down before others had
a chance to generate output.  It now waits for all nodes to fully
connect with each other.
---
 .../base/frameworks/cluster/start-it-up.bro   | 30 +++++++++++++++++--
 1 file changed, 27 insertions(+), 3 deletions(-)

diff --git a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
index 89f8d6b168..acb9c3676a 100644
--- a/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
+++ b/testing/btest/scripts/base/frameworks/cluster/start-it-up.bro
@@ -1,11 +1,13 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff proxy-1/.stdout
 # @TEST-EXEC: btest-diff proxy-2/.stdout
@@ -22,17 +24,39 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+global fully_connected: event();
+
 global peer_count = 0;
 
+global fully_connected_nodes = 0;
+
+event fully_connected()
+	{
+	fully_connected_nodes = fully_connected_nodes + 1;
+	if ( Cluster::node == "manager-1" )
+		{
+		if ( peer_count == 4 && fully_connected_nodes == 4 )
+			terminate_communication();
+		}
+	}
+
+redef Cluster::worker2manager_events += /fully_connected/;
+redef Cluster::proxy2manager_events += /fully_connected/;
+
 event remote_connection_handshake_done(p: event_peer)
 	{
 	print "Connected to a peer";
+	peer_count = peer_count + 1;
 	if ( Cluster::node == "manager-1" )
 		{
-		peer_count = peer_count + 1;
-		if ( peer_count == 4 )
+		if ( peer_count == 4 && fully_connected_nodes == 4 )
 			terminate_communication();
 		}
+	else
+		{
+		if ( peer_count == 2 )
+			event fully_connected();
+		}
 	}
 
 event remote_connection_closed(p: event_peer)

From 6d1abdb661b98726c2c77e171bbab0a65e024f54 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 13 Sep 2012 16:47:40 -0500
Subject: [PATCH 195/238] Adjusting Mac binary packaging script.

Setting CMAKE_PREFIX_PATH helps link against standard system libs
instead of ones that come from other package manager (e.g. MacPorts).

Changed to allow only more recent CMake versions to create packages
due to poorer clang compiler support in older versions, important
since clang is now the default compiler instead of gcc on Macs.
---
 pkg/make-mac-packages | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/pkg/make-mac-packages b/pkg/make-mac-packages
index 829a64ca25..2930f8f393 100755
--- a/pkg/make-mac-packages
+++ b/pkg/make-mac-packages
@@ -3,7 +3,13 @@
 # This script creates binary packages for Mac OS X.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
+cmake -P /dev/stdin << "EOF"
+if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
+    message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
+endif ()
+EOF
+
+[ $? -ne 0 ] && exit 1;
 
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
@@ -34,26 +40,26 @@ prefix=/opt/bro
 cd ..
 
 # Minimum Bro
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \
     --binary-package
 ( cd build && make package )
 
 # Full Bro package
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
 cd aux/broccoli
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..
 
 # Broctl
 cd aux/broctl
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..

From 6fbbf2829023036333231ffe00f89802b1f7bee0 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 14 Sep 2012 10:28:23 -0500
Subject: [PATCH 196/238] Update compile/dependency docs for OS X.

---
 doc/quickstart.rst | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/doc/quickstart.rst b/doc/quickstart.rst
index cc18956836..68dc4561cb 100644
--- a/doc/quickstart.rst
+++ b/doc/quickstart.rst
@@ -1,5 +1,6 @@
 .. _CMake: http://www.cmake.org
 .. _SWIG: http://www.swig.org
+.. _Xcode: https://developer.apple.com/xcode/
 .. _MacPorts: http://www.macports.org
 .. _Fink: http://www.finkproject.org
 .. _Homebrew: http://mxcl.github.com/homebrew
@@ -85,17 +86,20 @@ The following dependencies are required to build Bro:
 
 * Mac OS X
 
-  Snow Leopard (10.6) comes with all required dependencies except for CMake_.
+  Compiling source code on Macs requires first downloading Xcode_,
+  then going through its "Preferences..." -> "Downloads" menus to
+  install the "Command Line Tools" component.
 
-  Lion (10.7) comes with all required dependencies except for CMake_ and SWIG_.
+  Lion (10.7) and Mountain Lion (10.7) come with all required
+  dependencies except for CMake_, SWIG_, and ``libmagic``.
 
-  Distributions of these dependencies can be obtained from the project websites
-  linked above, but they're also likely available from your preferred Mac OS X
-  package management system (e.g. MacPorts_, Fink_, or Homebrew_).
+  Distributions of these dependencies can be obtained from the project
+  websites linked above, but they're also likely available from your
+  preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
+  or Homebrew_).
 
-  Note that the MacPorts ``swig`` package may not include any specific
-  language support so you may need to also install ``swig-ruby`` and
-  ``swig-python``.
+  Specifically for MacPorts, the ``swig``, ``swig-ruby``, ``swig-python``
+  and ``file`` packages provide the required dependencies.
 
 Optional Dependencies
 ~~~~~~~~~~~~~~~~~~~~~

From 392b99b2fa4b7bdda267eca55d4cc57d85e88641 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 18 Sep 2012 16:52:12 -0500
Subject: [PATCH 197/238] Fix construction of ip6_ah (Authentication Header)
 record values.

Authentication Headers with a Payload Len field set to zero would cause
a crash due to invalid memory allocation because the previous code
assumed Payload Len would always be great enough to contain all
mandatory fields of the header.  This changes it so the length of
the header is explicitly checked before attempting to extract fields
located past the minimum length (8 bytes) of an Authentication Header.

Crashes due to this are only possible when handling script-layer events
ipv6_ext_headers, new_packet, esp_packet, or teredo_*.  Or also when
implementing one of the discarder_check_* family of functions.
Otherwise, Bro correctly parses past such a header.
---
 scripts/base/init-bare.bro                       |   8 ++++----
 src/IP.cc                                        |  11 ++++++++---
 .../btest/Baseline/core.ipv6_zero_len_ah/output  |   2 ++
 testing/btest/Traces/ipv6_zero_len_ah.trace      | Bin 0 -> 1320 bytes
 testing/btest/core/ipv6_zero_len_ah.test         |  11 +++++++++++
 5 files changed, 25 insertions(+), 7 deletions(-)
 create mode 100644 testing/btest/Baseline/core.ipv6_zero_len_ah/output
 create mode 100644 testing/btest/Traces/ipv6_zero_len_ah.trace
 create mode 100644 testing/btest/core/ipv6_zero_len_ah.test

diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro
index ec75c76beb..cc3a40f54b 100644
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@@ -1135,10 +1135,10 @@ type ip6_ah: record {
 	rsv: count;
 	## Security Parameter Index.
 	spi: count;
-	## Sequence number.
-	seq: count;
-	## Authentication data.
-	data: string;
+	## Sequence number, unset in the case that *len* field is zero.
+	seq: count &optional;
+	## Authentication data, unset in the case that *len* field is zero.
+	data: string &optional;
 };
 
 ## Values extracted from an IPv6 ESP extension header.
diff --git a/src/IP.cc b/src/IP.cc
index 45afd593a9..398aacf1ee 100644
--- a/src/IP.cc
+++ b/src/IP.cc
@@ -148,9 +148,14 @@ RecordVal* IPv6_Hdr::BuildRecordVal(VectorVal* chain) const
 		rv->Assign(1, new Val(((ip6_ext*)data)->ip6e_len, TYPE_COUNT));
 		rv->Assign(2, new Val(ntohs(((uint16*)data)[1]), TYPE_COUNT));
 		rv->Assign(3, new Val(ntohl(((uint32*)data)[1]), TYPE_COUNT));
-		rv->Assign(4, new Val(ntohl(((uint32*)data)[2]), TYPE_COUNT));
-		uint16 off = 3 * sizeof(uint32);
-		rv->Assign(5, new StringVal(new BroString(data + off, Length() - off, 1)));
+		if ( Length() >= 12 )
+			{
+			// Sequence Number and ICV fields can only be extracted if
+			// Payload Len was non-zero for this header.
+			rv->Assign(4, new Val(ntohl(((uint32*)data)[2]), TYPE_COUNT));
+			uint16 off = 3 * sizeof(uint32);
+			rv->Assign(5, new StringVal(new BroString(data + off, Length() - off, 1)));
+			}
 		}
 		break;
 
diff --git a/testing/btest/Baseline/core.ipv6_zero_len_ah/output b/testing/btest/Baseline/core.ipv6_zero_len_ah/output
new file mode 100644
index 0000000000..d8db6a4c48
--- /dev/null
+++ b/testing/btest/Baseline/core.ipv6_zero_len_ah/output
@@ -0,0 +1,2 @@
+[orig_h=2000:1300::1, orig_p=128/icmp, resp_h=2000:1300::2, resp_p=129/icmp]
+[ip=<uninitialized>, ip6=[class=0, flow=0, len=166, nxt=51, hlim=255, src=2000:1300::1, dst=2000:1300::2, exts=[[id=51, hopopts=<uninitialized>, dstopts=<uninitialized>, routing=<uninitialized>, fragment=<uninitialized>, ah=[nxt=58, len=0, rsv=0, spi=0, seq=<uninitialized>, data=<uninitialized>], esp=<uninitialized>, mobility=<uninitialized>]]], tcp=<uninitialized>, udp=<uninitialized>, icmp=<uninitialized>]
diff --git a/testing/btest/Traces/ipv6_zero_len_ah.trace b/testing/btest/Traces/ipv6_zero_len_ah.trace
new file mode 100644
index 0000000000000000000000000000000000000000..7c3922525c26f97d870d6c2c3aa0462e82315b4a
GIT binary patch
literal 1320
zcmca|c+)~A1{MYw`2U}Qff2~DHt-7wNoHd31F}JwgF$_t(qt|Mbs)R#ZUT^Gkg)o%
zz#t4_!2lx~pQ(W%X{Sk8#RNw*05ZL?kclA-s1t;ZjX~Bz?0}lCfMGh*e<qkk42&T4
z?F<J!K_Y0jGBQF<0@)H;gs=sRA24lUL^F-4gTZ@#0v21C7$KnnvLyp(3oN7{q9}fm
zuyQ~*6^+l>$dHILq%IdTG28*V0EDslVVN<(c(4NM1c3&IU(bM){NMzjko*MnD-SRM
zf-shlyoQ-7&_j}i@whn9k8BA*f?-&NjZp<6m0?K-6y`@?B-62kJf`VP=pm0Q4Lbni
zb=oRI`S4!@D8j%g{Qo~7jb=JiJHr+^kUY9LBQzg^Y`G4!1#dn?&nZmkwstV6svp2&
F3jp^*vyA`%

literal 0
HcmV?d00001

diff --git a/testing/btest/core/ipv6_zero_len_ah.test b/testing/btest/core/ipv6_zero_len_ah.test
new file mode 100644
index 0000000000..dc3acf8443
--- /dev/null
+++ b/testing/btest/core/ipv6_zero_len_ah.test
@@ -0,0 +1,11 @@
+# @TEST-EXEC: bro -r $TRACES/ipv6_zero_len_ah.trace %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+# Shouldn't crash, but we also won't have seq and data fields set of the ip6_ah
+# record.
+
+event ipv6_ext_headers(c: connection, p: pkt_hdr)
+    {
+    print c$id;
+    print p;
+    }

From 73115dd334824b7293ea51b7222d2d92677a748a Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 24 Sep 2012 11:15:43 -0700
Subject: [PATCH 198/238] Updating CHANGES and VERSION.

---
 CHANGES | 30 ++++++++++++++++++++++++++++++
 VERSION |  2 +-
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index eee6aba604..0ab4fd0960 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,34 @@
 
+2.1-26 | 2012-09-23 08:46:03 -0700
+
+  * Add an item to FAQ page about broctl options. (Daniel Thayer)
+    
+  * Add more language tests. We now have tests of all built-in Bro
+    data types (including different representations of constant
+    values, and max./min. values), keywords, and operators (including
+    special properties of certain operators, such as short-circuit
+    evaluation and associativity). (Daniel Thayer)
+
+  * Fix construction of ip6_ah (Authentication Header) record values.
+
+    Authentication Headers with a Payload Len field set to zero would
+    cause a crash due to invalid memory allocation because the
+    previous code assumed Payload Len would always be great enough to
+    contain all mandatory fields of the header. (Jon Siwek)
+
+  * Update compile/dependency docs for OS X. (Jon Siwek)
+
+  * Adjusting Mac binary packaging script. Setting CMAKE_PREFIX_PATH
+    helps link against standard system libs instead of ones that come
+    from other package manager (e.g. MacPorts). (Jon Siwek)
+
+  * Adjusting some unit tests that do cluster communication. (Jon Siwek)
+
+  * Small change to non-blocking DNS initialization. (Jon Siwek)
+    
+  * Reorder a few statements in scan.l to make 1.5msecs etc work.
+    Adresses #872. (Bernhard Amann)
+
 2.1-6 | 2012-09-06 23:23:14 -0700
 
   * Fixed a bug where "a -= b" (both operands are intervals) was not
diff --git a/VERSION b/VERSION
index d218cbd5c8..e71d828348 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-6
+2.1-26

From 801f8d3de6c6d94083412815f27a8753eadd6c7f Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 24 Sep 2012 11:44:23 -0700
Subject: [PATCH 199/238] Updating submodule(s).

 [nomail]
---
 aux/btest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/btest b/aux/btest
index 44441a6c91..9d4e7c1d7b 160000
--- a/aux/btest
+++ b/aux/btest
@@ -1 +1 @@
-Subproject commit 44441a6c912c7c9f8d4771e042306ec5f44e461d
+Subproject commit 9d4e7c1d7bba8dd53d16ff4b4076690c0af4a2f0

From 8cd85a9013ee157c3bfca29a700c6d73d29f5295 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 24 Sep 2012 11:45:18 -0700
Subject: [PATCH 200/238] Updating submodule(s).

 [nomail]
---
 aux/btest | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/btest b/aux/btest
index 9d4e7c1d7b..e83c5f6e02 160000
--- a/aux/btest
+++ b/aux/btest
@@ -1 +1 @@
-Subproject commit 9d4e7c1d7bba8dd53d16ff4b4076690c0af4a2f0
+Subproject commit e83c5f6e02d6294747941d7a09f2dc327e8ab646

From 45926e6932554e19abb0587255f938c04e776f55 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 24 Sep 2012 16:13:24 -0700
Subject: [PATCH 201/238] Updating submodule(s).

 [nomail]
---
 aux/broctl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/broctl b/aux/broctl
index 2fb9ff62bf..1a7db43a8a 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 2fb9ff62bf08f78071753016863640022fbfe338
+Subproject commit 1a7db43a8a5186fa12b8b19527a971da8cc280ae

From 101ba67203c4b8116ecf6c71b5d6c786c40699d8 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Mon, 24 Sep 2012 18:20:42 -0500
Subject: [PATCH 202/238] Fix race condition in language/when.bro test

---
 testing/btest/Baseline/language.when/out | 1 +
 testing/btest/language/when.bro          | 6 +++++-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/testing/btest/Baseline/language.when/out b/testing/btest/Baseline/language.when/out
index 19f86f493a..3a052217ab 100644
--- a/testing/btest/Baseline/language.when/out
+++ b/testing/btest/Baseline/language.when/out
@@ -1 +1,2 @@
 done
+lookup successful
diff --git a/testing/btest/language/when.bro b/testing/btest/language/when.bro
index d6b08b67e1..19b7f48196 100644
--- a/testing/btest/language/when.bro
+++ b/testing/btest/language/when.bro
@@ -1,6 +1,9 @@
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-bg-run test1 bro %INPUT
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: mv test1/.stdout out
 # @TEST-EXEC: btest-diff out
 
+@load frameworks/communication/listen
 
 event bro_init()
 {
@@ -9,6 +12,7 @@ event bro_init()
 	when ( local h1name = lookup_addr(h1) )
 		{ 
 		print "lookup successful";
+		terminate();
 		}
 	print "done";
 }

From d4b95e2bbfb68982c4f4fe99e40f3e405b82bbfe Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 25 Sep 2012 06:25:15 -0700
Subject: [PATCH 203/238] Updating submodule(s).

 [nomail]
---
 aux/broctl | 2 +-
 aux/btest  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/aux/broctl b/aux/broctl
index 1a7db43a8a..44afce440d 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 1a7db43a8a5186fa12b8b19527a971da8cc280ae
+Subproject commit 44afce440d02e1aac4012d5b0f5a26875ae11c3e
diff --git a/aux/btest b/aux/btest
index e83c5f6e02..44a43e6245 160000
--- a/aux/btest
+++ b/aux/btest
@@ -1 +1 @@
-Subproject commit e83c5f6e02d6294747941d7a09f2dc327e8ab646
+Subproject commit 44a43e62452302277f88e8fac08d1f979dc53f98

From 1044762dfa329b50a42972bb33d319ed3ae3091f Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 25 Sep 2012 14:53:51 -0500
Subject: [PATCH 204/238] Serialize language.when unit test with the "comm"
 group.

Since it now loads the listen script.
---
 testing/btest/language/when.bro | 1 +
 1 file changed, 1 insertion(+)

diff --git a/testing/btest/language/when.bro b/testing/btest/language/when.bro
index 19b7f48196..84c1f06cef 100644
--- a/testing/btest/language/when.bro
+++ b/testing/btest/language/when.bro
@@ -1,3 +1,4 @@
+# @TEST-SERIALIZE: comm
 # @TEST-EXEC: btest-bg-run test1 bro %INPUT
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: mv test1/.stdout out

From 6f45a8f4ef8e009b9fcf71df3ebf5024fd9c8544 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 25 Sep 2012 15:26:44 -0500
Subject: [PATCH 205/238] Fix parsing of integers

This bug was seen on 32-bit systems, where the range of recognized values
was less than the range of hexadecimal values.
---
 src/scan.l | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/scan.l b/src/scan.l
index 377c74cc1a..3f7337ac47 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -437,7 +437,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 	}
 
 {D}		{
-		RET_CONST(new Val(static_cast<bro_uint_t>(strtoul(yytext, (char**) NULL, 10)),
+		RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, (char**) NULL, 10)),
 			  TYPE_COUNT))
 		}
 {FLOAT}		RET_CONST(new Val(atof(yytext), TYPE_DOUBLE))

From f7e55509a447bb11136abe6e7cb21cb3de1037af Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 25 Sep 2012 16:05:23 -0500
Subject: [PATCH 206/238] Uncomment some previously-broken tests

Uncommented some tests that previously would cause Bro to exit
with an error.
---
 testing/btest/Baseline/language.interval/out | 1 +
 testing/btest/language/interval.bro          | 9 +++------
 2 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/testing/btest/Baseline/language.interval/out b/testing/btest/Baseline/language.interval/out
index f42082ef5d..ae9ed5d74e 100644
--- a/testing/btest/Baseline/language.interval/out
+++ b/testing/btest/Baseline/language.interval/out
@@ -1,5 +1,6 @@
 type inference (PASS)
 type inference (PASS)
+type inference (PASS)
 optional space (PASS)
 plural/singular interval are same (PASS)
 different units with same numeric value (PASS)
diff --git a/testing/btest/language/interval.bro b/testing/btest/language/interval.bro
index 6bdbf3a8e8..66d44206d3 100644
--- a/testing/btest/language/interval.bro
+++ b/testing/btest/language/interval.bro
@@ -21,8 +21,7 @@ event bro_init()
 	local in13: interval = 120sec;
 	local in14: interval = 2min;
 	local in15: interval = -2hr;
-	# TODO: this one causes bro to fail
-	#local in16: interval = 2.5day;
+	local in16: interval = 2.5day;
 
 	# Constants with space and no letter "s"
 
@@ -45,15 +44,13 @@ event bro_init()
 	# Type inference
 
 	local in41 = 2 usec;
-	# TODO: this one causes bro to fail
-	#local in42 = 2.1usec;
+	local in42 = 2.1usec;
 	local in43 = 3usecs;
 
 	# Type inference tests
 
 	test_case( "type inference", type_name(in41) == "interval" );
-	# TODO: uncomment when bug is fixed
-	#test_case( "type inference", type_name(in42) == "interval" );
+	test_case( "type inference", type_name(in42) == "interval" );
 	test_case( "type inference", type_name(in43) == "interval" );
 
 	# Test various constant representations

From d6f671494ef2768b45c2eaf39cae00135379a886 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 26 Sep 2012 12:14:11 -0500
Subject: [PATCH 207/238] Reliability adjustments to istate tests with network
 communication.

---
 testing/btest/istate/bro-ipv6-socket.bro      | 4 ++--
 testing/btest/istate/broccoli-ipv6-socket.bro | 3 ++-
 testing/btest/istate/broccoli-ipv6.bro        | 3 ++-
 testing/btest/istate/broccoli-ssl.bro         | 3 ++-
 testing/btest/istate/broccoli.bro             | 3 ++-
 testing/btest/istate/events-ssl.bro           | 4 ++--
 testing/btest/istate/events.bro               | 4 ++--
 testing/btest/istate/sync.bro                 | 3 ++-
 8 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/testing/btest/istate/bro-ipv6-socket.bro b/testing/btest/istate/bro-ipv6-socket.bro
index b339bf4487..305f32caab 100644
--- a/testing/btest/istate/bro-ipv6-socket.bro
+++ b/testing/btest/istate/bro-ipv6-socket.bro
@@ -4,7 +4,7 @@
 #
 # @TEST-EXEC: btest-bg-run recv bro -b ../recv.bro
 # @TEST-EXEC: btest-bg-run send bro -b ../send.bro
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 #
 # @TEST-EXEC: btest-diff recv/.stdout
 # @TEST-EXEC: btest-diff send/.stdout
@@ -14,7 +14,7 @@
 @load base/frameworks/communication
 
 redef Communication::nodes += {
-    ["foo"] = [$host=[::1], $connect=T, $events=/my_event/]
+    ["foo"] = [$host=[::1], $connect=T, $retry=1sec, $events=/my_event/]
 };
 
 global my_event: event(s: string);
diff --git a/testing/btest/istate/broccoli-ipv6-socket.bro b/testing/btest/istate/broccoli-ipv6-socket.bro
index 21067c1b23..be6266fdec 100644
--- a/testing/btest/istate/broccoli-ipv6-socket.bro
+++ b/testing/btest/istate/broccoli-ipv6-socket.bro
@@ -4,7 +4,8 @@
 # @TEST-REQUIRES: ifconfig | grep -q -E "inet6 ::1|inet6 addr: ::1"
 #
 # @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ipv6=T"
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs -6 ::1
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff broccoli/.stdout
diff --git a/testing/btest/istate/broccoli-ipv6.bro b/testing/btest/istate/broccoli-ipv6.bro
index ba181d4987..b4fdfb5fcf 100644
--- a/testing/btest/istate/broccoli-ipv6.bro
+++ b/testing/btest/istate/broccoli-ipv6.bro
@@ -3,7 +3,8 @@
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
 # @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broccoli-v6addrs
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff broccoli/.stdout
diff --git a/testing/btest/istate/broccoli-ssl.bro b/testing/btest/istate/broccoli-ssl.bro
index 4465cd1bb3..dcbea93150 100644
--- a/testing/btest/istate/broccoli-ssl.bro
+++ b/testing/btest/istate/broccoli-ssl.bro
@@ -4,8 +4,9 @@
 #
 # @TEST-EXEC: chmod 600 broccoli.conf
 # @TEST-EXEC: btest-bg-run bro bro $DIST/aux/broccoli/test/broccoli-v6addrs.bro "Communication::listen_ssl=T" "ssl_ca_certificate=../ca_cert.pem" "ssl_private_key=../bro.pem"
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli BROCCOLI_CONFIG_FILE=../broccoli.conf $BUILD/aux/broccoli/test/broccoli-v6addrs
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff bro/.stdout
 # @TEST-EXEC: btest-diff broccoli/.stdout
 
diff --git a/testing/btest/istate/broccoli.bro b/testing/btest/istate/broccoli.bro
index 2bae5dc080..2fdd4cbda4 100644
--- a/testing/btest/istate/broccoli.bro
+++ b/testing/btest/istate/broccoli.bro
@@ -3,8 +3,9 @@
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
 # @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broping-record.bro
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broping -r -c 3 127.0.0.1
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: cat bro/ping.log | sed 's/one-way.*//g' >bro.log
 # @TEST-EXEC: cat broccoli/.stdout | sed 's/time=.*//g' >broccoli.log
 # @TEST-EXEC: btest-diff bro.log
diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro
index e09bf112fd..1d285869b4 100644
--- a/testing/btest/istate/events-ssl.bro
+++ b/testing/btest/istate/events-ssl.bro
@@ -2,7 +2,7 @@
 #
 # @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
@@ -55,7 +55,7 @@ event bro_init()
 redef peer_description = "events-rcv";
 
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T]
+    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T, $retry=1sec]
 };
 
 redef ssl_ca_certificate = "../ca_cert.pem";
diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro
index 70726a9f20..590aabcd23 100644
--- a/testing/btest/istate/events.bro
+++ b/testing/btest/istate/events.bro
@@ -2,7 +2,7 @@
 #
 # @TEST-EXEC: btest-bg-run sender   bro -Bthreading,logging,comm -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
 # @TEST-EXEC: btest-bg-run receiver bro -Bthreading,logging,comm  ../receiver.bro
-# @TEST-EXEC: btest-bg-wait -k 20
+# @TEST-EXEC: btest-bg-wait 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
@@ -50,7 +50,7 @@ event bro_init()
 redef peer_description = "events-rcv";
 
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T]
+    ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $retry=1sec]
 };
 
 event remote_connection_closed(p: event_peer)
diff --git a/testing/btest/istate/sync.bro b/testing/btest/istate/sync.bro
index 776ddfd2fa..e1364a9553 100644
--- a/testing/btest/istate/sync.bro
+++ b/testing/btest/istate/sync.bro
@@ -154,7 +154,8 @@ event bro_init()
     }
 	
 redef Communication::nodes += {
-    ["foo"] = [$host = 127.0.0.1, $events = /.*/, $connect=T, $sync=T]
+    ["foo"] = [$host = 127.0.0.1, $events = /.*/, $connect=T, $sync=T,
+               $retry=1sec]
 };
 
 event remote_connection_closed(p: event_peer)

From 5593f339bdd4dfd9e35c24ededd1b4457350c7e5 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 26 Sep 2012 13:09:54 -0500
Subject: [PATCH 208/238] Remove unused reserved keyword "this"

Removed unused reserved keyword "this" (a script using it would cause Bro to
segfault).
---
 src/parse.y | 9 +--------
 src/scan.l  | 1 -
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/src/parse.y b/src/parse.y
index 75e09dc60f..27af150254 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -14,7 +14,7 @@
 %token TOK_NEXT TOK_OF TOK_PATTERN TOK_PATTERN_TEXT
 %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF
 %token TOK_REMOVE_FROM TOK_RETURN TOK_SCHEDULE TOK_SET
-%token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE TOK_THIS
+%token TOK_STRING TOK_SUBNET TOK_SWITCH TOK_TABLE
 %token TOK_TIME TOK_TIMEOUT TOK_TIMER TOK_TYPE TOK_UNION TOK_VECTOR TOK_WHEN
 
 %token TOK_ATTR_ADD_FUNC TOK_ATTR_ATTR TOK_ATTR_ENCRYPT TOK_ATTR_DEFAULT
@@ -118,7 +118,6 @@ extern const char* g_curr_debug_error;
 
 #define YYLTYPE yyltype
 
-Expr* bro_this = 0;
 int in_init = 0;
 int in_record = 0;
 bool resolving_global_ID = false;
@@ -584,12 +583,6 @@ expr:
 			$$ = new ConstExpr(new PatternVal($1));
 			}
 
-	|	TOK_THIS
-			{
-			set_location(@1);
-			$$ = bro_this->Ref();
-			}
-
 	|       '|' expr '|'
 			{
 			set_location(@1, @3);
diff --git a/src/scan.l b/src/scan.l
index 3f7337ac47..d213b60012 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -306,7 +306,6 @@ string	return TOK_STRING;
 subnet	return TOK_SUBNET;
 switch	return TOK_SWITCH;
 table	return TOK_TABLE;
-this	return TOK_THIS;
 time	return TOK_TIME;
 timeout	return TOK_TIMEOUT;
 timer	return TOK_TIMER;

From f00a7c3ee401405559d13a0597011cf1a1edaa7e Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 26 Sep 2012 14:20:30 -0500
Subject: [PATCH 209/238] Remove deprecated built-in functions

---
 src/bro.bif     | 78 -------------------------------------------------
 src/strings.bif |  9 ------
 2 files changed, 87 deletions(-)

diff --git a/src/bro.bif b/src/bro.bif
index bc791d6858..3cac8c8da5 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -5683,12 +5683,6 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
 #
 # ===========================================================================
 
-## Deprecated. Will be removed.
-function parse_dotted_addr%(s: string%): addr
-	%{
-	IPAddr a(s->CheckString());
-	return new AddrVal(a);
-	%}
 
 
 %%{
@@ -5788,75 +5782,3 @@ function anonymize_addr%(a: addr, cl: IPAddrAnonymizationClass%): addr
 		}
 	%}
 
-## Deprecated. Will be removed.
-function dump_config%(%) : bool
-	%{
-	return new Val(persistence_serializer->WriteConfig(true), TYPE_BOOL);
-	%}
-
-## Deprecated. Will be removed.
-function make_connection_persistent%(c: connection%) : any
-	%{
-	c->MakePersistent();
-	return 0;
-	%}
-
-%%{
-// Experimental code to add support for IDMEF XML output based on
-// notices.  For now, we're implementing it as a builtin you can call on an
-// notices record.
-
-#ifdef USE_IDMEF
-extern "C" {
-#include <libidmef/idmefxml.h>
-}
-#endif
-
-#include <sys/socket.h>
-
-char* port_to_string(PortVal* port)
-	{
-	char buf[256];	// to hold sprintf results on port numbers
-	snprintf(buf, sizeof(buf), "%u", port->Port());
-	return copy_string(buf);
-	}
-
-%%}
-
-## Deprecated. Will be removed.
-function generate_idmef%(src_ip: addr, src_port: port,
-				dst_ip: addr, dst_port: port%) : bool
-	%{
-#ifdef USE_IDMEF
-	xmlNodePtr message =
-		newIDMEF_Message(newAttribute("version","1.0"),
-			newAlert(newCreateTime(NULL),
-			newSource(
-				newNode(newAddress(
-					newAttribute("category","ipv4-addr"),
-					newSimpleElement("address",
-					copy_string(src_ip->AsAddr().AsString().c_str())),
-					NULL), NULL),
-				newService(
-					newSimpleElement("port",
-						port_to_string(src_port)),
-					NULL), NULL),
-			newTarget(
-				newNode(newAddress(
-					newAttribute("category","ipv4-addr"),
-					newSimpleElement("address",
-					copy_string(dst_ip->AsAddr().AsString().c_str())),
-					NULL), NULL),
-				newService(
-					newSimpleElement("port",
-						port_to_string(dst_port)),
-					NULL), NULL), NULL), NULL);
-
-	// if ( validateCurrentDoc() )
-	printCurrentMessage(stderr);
-	return new Val(1, TYPE_BOOL);
-#else
-	builtin_error("Bro was not configured for IDMEF support");
-	return new Val(0, TYPE_BOOL);
-#endif
-	%}
diff --git a/src/strings.bif b/src/strings.bif
index 22e29950ee..43dee25c1b 100644
--- a/src/strings.bif
+++ b/src/strings.bif
@@ -552,15 +552,6 @@ function split_n%(str: string, re: pattern,
 	return do_split(str, re, 0, incl_sep, max_num_sep);
 	%}
 
-## Deprecated. Will be removed.
-# Reason: the parameter ``other`` does nothing.
-function split_complete%(str: string,
-		re: pattern, other: string_set,
-		incl_sep: bool, max_num_sep: count%): string_array
-	%{
-	return do_split(str, re, other->AsTableVal(), incl_sep, max_num_sep);
-	%}
-
 ## Substitutes a given replacement string for the first occurrence of a pattern
 ## in a given string.
 ##

From 72f16f26426ac34b7cf452c1a65f13fd5651491a Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 26 Sep 2012 15:20:54 -0500
Subject: [PATCH 210/238] Remove unused argument of helper function

Removed an unused argument of the "do_split" helper function.  The unused
argument was previously used by a now-removed BIF.
---
 src/strings.bif | 19 +++++--------------
 1 file changed, 5 insertions(+), 14 deletions(-)

diff --git a/src/strings.bif b/src/strings.bif
index 43dee25c1b..dc5e064dc6 100644
--- a/src/strings.bif
+++ b/src/strings.bif
@@ -311,15 +311,9 @@ static int match_prefix(int s_len, const char* s, int t_len, const char* t)
 	return 1;
 	}
 
-Val* do_split(StringVal* str_val, RE_Matcher* re, TableVal* other_sep,
-		int incl_sep, int max_num_sep)
+Val* do_split(StringVal* str_val, RE_Matcher* re, int incl_sep, int max_num_sep)
 	{
 	TableVal* a = new TableVal(string_array);
-	ListVal* other_strings = 0;
-
-	if ( other_sep && other_sep->Size() > 0 )
-		other_strings = other_sep->ConvertToPureList();
-
 	const u_char* s = str_val->Bytes();
 	int n = str_val->Len();
 	const u_char* end_of_s = s + n;
@@ -373,9 +367,6 @@ Val* do_split(StringVal* str_val, RE_Matcher* re, TableVal* other_sep,
 			reporter->InternalError("RegMatch in split goes beyond the string");
 		}
 
-	if ( other_strings )
-		delete other_strings;
-
 	return a;
 	}
 
@@ -483,7 +474,7 @@ Val* do_sub(StringVal* str_val, RE_Matcher* re, StringVal* repl, int do_all)
 ##
 function split%(str: string, re: pattern%): string_array
 	%{
-	return do_split(str, re, 0, 0, 0);
+	return do_split(str, re, 0, 0);
 	%}
 
 ## Splits a string *once* into a two-element array of strings according to a
@@ -503,7 +494,7 @@ function split%(str: string, re: pattern%): string_array
 ## .. bro:see:: split split_all split_n str_split
 function split1%(str: string, re: pattern%): string_array
 	%{
-	return do_split(str, re, 0, 0, 1);
+	return do_split(str, re, 0, 1);
 	%}
 
 ## Splits a string into an array of strings according to a pattern. This
@@ -523,7 +514,7 @@ function split1%(str: string, re: pattern%): string_array
 ## .. bro:see:: split split1 split_n str_split
 function split_all%(str: string, re: pattern%): string_array
 	%{
-	return do_split(str, re, 0, 1, 0);
+	return do_split(str, re, 1, 0);
 	%}
 
 ## Splits a string a given number of times into an array of strings according
@@ -549,7 +540,7 @@ function split_all%(str: string, re: pattern%): string_array
 function split_n%(str: string, re: pattern,
 		incl_sep: bool, max_num_sep: count%): string_array
 	%{
-	return do_split(str, re, 0, incl_sep, max_num_sep);
+	return do_split(str, re, incl_sep, max_num_sep);
 	%}
 
 ## Substitutes a given replacement string for the first occurrence of a pattern

From 254715eaaa30d4888511cbfc1ee81fc2f9c2d2bf Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Wed, 26 Sep 2012 16:47:51 -0500
Subject: [PATCH 211/238] Remove deprecated attribute &disable_print_hook

---
 doc/ext/bro_lexer/bro.py  |   2 +-
 doc/ext/bro_lexer/bro.pyc | Bin 2702 -> 2585 bytes
 doc/scripts/builtins.rst  |   4 ----
 src/Attr.cc               |   7 +------
 src/Attr.h                |   1 -
 src/File.cc               |   3 ---
 src/File.h                |   2 +-
 src/bro.bif               |   4 ++--
 src/parse.y               |   6 ++----
 src/scan.l                |   1 -
 10 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/doc/ext/bro_lexer/bro.py b/doc/ext/bro_lexer/bro.py
index 8cb4475f3b..ae2566a8de 100644
--- a/doc/ext/bro_lexer/bro.py
+++ b/doc/ext/bro_lexer/bro.py
@@ -29,7 +29,7 @@ class BroLexer(RegexLexer):
              r'|vector)\b', Keyword.Type),
             (r'(T|F)\b', Keyword.Constant),
             (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
-             r'|default|disable_print_hook|raw_output|encrypt|group|log'
+             r'|default|raw_output|encrypt|group|log'
              r'|mergeable|optional|persistent|priority|redef'
              r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
                  Keyword)),
diff --git a/doc/ext/bro_lexer/bro.pyc b/doc/ext/bro_lexer/bro.pyc
index 6471e1528d8d02296dbdedc0548e86cd80a3c439..c7b4fde790bb48f424f4a4bedbc75e693112baf0 100644
GIT binary patch
delta 46
ycmeAZohib>{F#?)VM%hpMvjLpj5{a4V)5GS&ib60g^huMVRAC(6$m4TO9=p8_YF({

delta 163
zcmbO!(kIHn{F#^QMC2U5jT{eI7|%?8#o{HDl3ARXl#?1?P?VWh5}%QupS{_b^#!x3
zIRgWOerR!OQL%njab|gHwthfSepYI7NwI!XQNDh8dAWXZa#3ahgdd-iT9I0$KUtph
L3Yj{;b1DG<_P0C1

diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst
index 0501067409..d274de6b7b 100644
--- a/doc/scripts/builtins.rst
+++ b/doc/scripts/builtins.rst
@@ -600,10 +600,6 @@ scripting language supports the following built-in attributes.
 
 .. TODO: needs to be documented.
 
-.. bro:attr:: &disable_print_hook
-
-    Deprecated. Will be removed.
-
 .. bro:attr:: &raw_output
 
     Opens a file in raw mode, i.e., non-ASCII characters are not
diff --git a/src/Attr.cc b/src/Attr.cc
index 2e4e090c0b..bdf247b4f5 100644
--- a/src/Attr.cc
+++ b/src/Attr.cc
@@ -15,7 +15,7 @@ const char* attr_name(attr_tag t)
 		"&add_func", "&delete_func", "&expire_func",
 		"&read_expire", "&write_expire", "&create_expire",
 		"&persistent", "&synchronized", "&postprocessor",
-		"&encrypt", "&match", "&disable_print_hook",
+		"&encrypt", "&match",
 		"&raw_output", "&mergeable", "&priority",
 		"&group", "&log", "&error_handler", "&type_column",
 		"(&tracked)",
@@ -385,11 +385,6 @@ void Attributes::CheckAttr(Attr* a)
 		// FIXME: Check here for global ID?
 		break;
 
-	case ATTR_DISABLE_PRINT_HOOK:
-		if ( type->Tag() != TYPE_FILE )
-			Error("&disable_print_hook only applicable to files");
-		break;
-
 	case ATTR_RAW_OUTPUT:
 		if ( type->Tag() != TYPE_FILE )
 			Error("&raw_output only applicable to files");
diff --git a/src/Attr.h b/src/Attr.h
index e6b09cf96b..c9a0dedb33 100644
--- a/src/Attr.h
+++ b/src/Attr.h
@@ -28,7 +28,6 @@ typedef enum {
 	ATTR_POSTPROCESSOR,
 	ATTR_ENCRYPT,
 	ATTR_MATCH,
-	ATTR_DISABLE_PRINT_HOOK,
 	ATTR_RAW_OUTPUT,
 	ATTR_MERGEABLE,
 	ATTR_PRIORITY,
diff --git a/src/File.cc b/src/File.cc
index 3b9f3be33b..880fd254ef 100644
--- a/src/File.cc
+++ b/src/File.cc
@@ -514,9 +514,6 @@ void BroFile::SetAttrs(Attributes* arg_attrs)
 			InitEncrypt(log_encryption_key->AsString()->CheckString());
 		}
 
-	if ( attrs->FindAttr(ATTR_DISABLE_PRINT_HOOK) )
-		DisablePrintHook();
-
 	if ( attrs->FindAttr(ATTR_RAW_OUTPUT) )
 		EnableRawOutput();
 
diff --git a/src/File.h b/src/File.h
index 37f844867b..8e3d0ca6e7 100644
--- a/src/File.h
+++ b/src/File.h
@@ -57,7 +57,7 @@ public:
 	RecordVal* Rotate();
 
 	// Set &rotate_interval, &rotate_size, &postprocessor,
-	// &disable_print_hook, and &raw_output attributes.
+	// and &raw_output attributes.
 	void SetAttrs(Attributes* attrs);
 
 	// Returns the current size of the file, after fresh stat'ing.
diff --git a/src/bro.bif b/src/bro.bif
index 3cac8c8da5..8ddde6ef86 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -4858,7 +4858,7 @@ function file_size%(f: string%) : double
 	%}
 
 ## Disables sending :bro:id:`print_hook` events to remote peers for a given
-## file. This function is equivalent to :bro:attr:`&disable_print_hook`. In a
+## file. In a
 ## distributed setup, communicating Bro instances generate the event
 ## :bro:id:`print_hook` for each print statement and send it to the remote
 ## side. When disabled for a particular file, these events will not be
@@ -4874,7 +4874,7 @@ function disable_print_hook%(f: file%): any
 	%}
 
 ## Prevents escaping of non-ASCII characters when writing to a file.
-## This function is equivalent to :bro:attr:`&disable_print_hook`.
+## This function is equivalent to :bro:attr:`&raw_output`.
 ##
 ## f: The file to disable raw output for.
 ##
diff --git a/src/parse.y b/src/parse.y
index 27af150254..c1f6ddd96e 100644
--- a/src/parse.y
+++ b/src/parse.y
@@ -2,7 +2,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 90
+%expect 87
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -22,7 +22,7 @@
 %token TOK_ATTR_ROTATE_SIZE TOK_ATTR_DEL_FUNC TOK_ATTR_EXPIRE_FUNC
 %token TOK_ATTR_EXPIRE_CREATE TOK_ATTR_EXPIRE_READ TOK_ATTR_EXPIRE_WRITE
 %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED
-%token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE
+%token TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE
 %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER
 %token TOK_ATTR_TYPE_COLUMN
 
@@ -1290,8 +1290,6 @@ attr:
 			{ $$ = new Attr(ATTR_ENCRYPT); }
 	|	TOK_ATTR_ENCRYPT '=' expr
 			{ $$ = new Attr(ATTR_ENCRYPT, $3); }
-	|	TOK_ATTR_DISABLE_PRINT_HOOK
-			{ $$ = new Attr(ATTR_DISABLE_PRINT_HOOK); }
 	|	TOK_ATTR_RAW_OUTPUT
 			{ $$ = new Attr(ATTR_RAW_OUTPUT); }
 	|	TOK_ATTR_MERGEABLE
diff --git a/src/scan.l b/src/scan.l
index d213b60012..6c87766781 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -319,7 +319,6 @@ when	return TOK_WHEN;
 &create_expire	return TOK_ATTR_EXPIRE_CREATE;
 &default	return TOK_ATTR_DEFAULT;
 &delete_func	return TOK_ATTR_DEL_FUNC;
-&disable_print_hook	return TOK_ATTR_DISABLE_PRINT_HOOK;
 &raw_output return TOK_ATTR_RAW_OUTPUT;
 &encrypt	return TOK_ATTR_ENCRYPT;
 &error_handler	return TOK_ATTR_ERROR_HANDLER;

From b73809d54f960c9e50dd7651ec512f4a16b498eb Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Thu, 27 Sep 2012 12:18:25 -0700
Subject: [PATCH 212/238] Fix compile issues with older versions of libcurl.

Older versions of libcurl do not offer *_MS timeout constants, which causes the
build to fail. For sub-second timeout specification, we now fall back to
hard-coded timeouts in older libcurl version.
---
 src/logging/writers/ElasticSearch.cc | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index cb3248a044..24489314ec 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -48,7 +48,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	last_send = current_time();
 	failing = false;
 
-	transfer_timeout = BifConst::LogElasticSearch::transfer_timeout * 1000;
+    transfer_timeout = static_cast<uint64>(BifConst::LogElasticSearch::transfer_timeout) * 1000;
 
 	curl_handle = HTTPSetup();
 }
@@ -373,8 +373,21 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 
 	// Some timeout options.  These will need more attention later.
 	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
+#if LIBCURL_VERSION_NUM > 0x071002
 	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout);
 	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2);
+#else
+    if ( transfer_timeout > 1000 )
+        {
+        curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout/1000);
+        curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout/2000);
+        }
+    else
+        {
+        curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2);
+        curl_easy_setopt(handle, CURLOPT_TIMEOUT, 1);
+        }
+#endif
 	curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60);
 
 	CURLcode return_code = curl_easy_perform(handle);

From 1ce76da90f4aa032da601e80e339518622272457 Mon Sep 17 00:00:00 2001
From: Matthias Vallentin <vallentin@icir.org>
Date: Thu, 27 Sep 2012 16:25:05 -0700
Subject: [PATCH 213/238] Use second granularity for ElasticSearch timeouts.

Since the millisecond resolution cannot be harnessed universally and is not
supported by older version of libcurl, we will allow only specifications at the
granularity of seconds.

This commit also fixes a typing issue that causes that prevented the
ElasticSearch timeout to work in the first place: curl_easy_setopt requires a
long but was given a uint64_t.
---
 .../logging/writers/elasticsearch.bro         |  5 +++--
 src/logging/writers/ElasticSearch.cc          | 19 +++----------------
 src/logging/writers/ElasticSearch.h           |  2 +-
 3 files changed, 7 insertions(+), 19 deletions(-)

diff --git a/scripts/base/frameworks/logging/writers/elasticsearch.bro b/scripts/base/frameworks/logging/writers/elasticsearch.bro
index b0e8fac40e..1cb1c3f83f 100644
--- a/scripts/base/frameworks/logging/writers/elasticsearch.bro
+++ b/scripts/base/frameworks/logging/writers/elasticsearch.bro
@@ -26,8 +26,9 @@ export {
 	## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
 	const type_prefix = "" &redef;
 
-	## The time before an ElasticSearch transfer will timeout.
-	## This is not working!
+	## The time before an ElasticSearch transfer will timeout. Time
+    ## specifications less than seconds result in a timeout value of 0, which
+    ## means "no timeout."
 	const transfer_timeout = 2secs;
 
 	## The batch size is the number of messages that will be queued up before
diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index 24489314ec..393d52c188 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -48,7 +48,7 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	last_send = current_time();
 	failing = false;
 
-    transfer_timeout = static_cast<uint64>(BifConst::LogElasticSearch::transfer_timeout) * 1000;
+    transfer_timeout = static_cast<long>(BifConst::LogElasticSearch::transfer_timeout);
 
 	curl_handle = HTTPSetup();
 }
@@ -373,21 +373,8 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 
 	// Some timeout options.  These will need more attention later.
 	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
-#if LIBCURL_VERSION_NUM > 0x071002
-	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout);
-	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2);
-#else
-    if ( transfer_timeout > 1000 )
-        {
-        curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout/1000);
-        curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout/2000);
-        }
-    else
-        {
-        curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2);
-        curl_easy_setopt(handle, CURLOPT_TIMEOUT, 1);
-        }
-#endif
+    curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, transfer_timeout);
+    curl_easy_setopt(handle, CURLOPT_TIMEOUT, transfer_timeout);
 	curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60);
 
 	CURLcode return_code = curl_easy_perform(handle);
diff --git a/src/logging/writers/ElasticSearch.h b/src/logging/writers/ElasticSearch.h
index 0e88bf3e88..fef0a00ffd 100644
--- a/src/logging/writers/ElasticSearch.h
+++ b/src/logging/writers/ElasticSearch.h
@@ -68,7 +68,7 @@ private:
 
 	string path;
 	string index_prefix;
-	uint64 transfer_timeout;
+	long transfer_timeout;
 	bool failing;
 
 	uint64 batch_size;

From 474ab86b9c6d6d02850c032d451d2cf6c95c8280 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Sat, 29 Sep 2012 14:44:58 -0700
Subject: [PATCH 214/238] Updating submodule(s).

 [nomail]
---
 aux/broctl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/broctl b/aux/broctl
index 44afce440d..b0e3c0d846 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 44afce440d02e1aac4012d5b0f5a26875ae11c3e
+Subproject commit b0e3c0d84643878c135dcb8a9774ed78147dd648

From 68aead024ab4a93ac83dc83f5ba61427bd1401e4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 1 Oct 2012 12:32:24 -0500
Subject: [PATCH 215/238] Add an example of a GridFTP data channel detection
 script.

It relies on the heuristics of GridFTP data channels commonly default to
SSL mutual authentication with a NULL bulk cipher and that they usually
transfer large datasets (default threshold of script is 1 GB).  The
script also defaults to skip_further_processing() after detection to try
to save cycles analyzing the large, benign connection.

Also added a script in base/protocols/conn/polling that generalizes the
process of polling a connection for interesting features.  The GridFTP
data channel detection script depends on it to monitor bytes
transferred.
---
 scripts/base/protocols/conn/__load__.bro      |   1 +
 scripts/base/protocols/conn/polling.bro       |  51 +++++++++++
 .../protocols/ftp/gridftp-data-detection.bro  |  83 ++++++++++++++++++
 .../scripts.base.protocols.conn.polling/out1  |   7 ++
 .../scripts.base.protocols.conn.polling/out2  |   4 +
 .../notice.log                                |  10 +++
 testing/btest/Traces/globus-url-copy.trace    | Bin 0 -> 21556 bytes
 .../scripts/base/protocols/conn/polling.test  |  20 +++++
 .../protocols/ftp/gridftp-data-dection.test   |   6 ++
 9 files changed, 182 insertions(+)
 create mode 100644 scripts/base/protocols/conn/polling.bro
 create mode 100644 scripts/policy/protocols/ftp/gridftp-data-detection.bro
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.conn.polling/out1
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.conn.polling/out2
 create mode 100644 testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log
 create mode 100644 testing/btest/Traces/globus-url-copy.trace
 create mode 100644 testing/btest/scripts/base/protocols/conn/polling.test
 create mode 100644 testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test

diff --git a/scripts/base/protocols/conn/__load__.bro b/scripts/base/protocols/conn/__load__.bro
index 8c673eca85..719486d885 100644
--- a/scripts/base/protocols/conn/__load__.bro
+++ b/scripts/base/protocols/conn/__load__.bro
@@ -1,3 +1,4 @@
 @load ./main
 @load ./contents
 @load ./inactivity
+@load ./polling
diff --git a/scripts/base/protocols/conn/polling.bro b/scripts/base/protocols/conn/polling.bro
new file mode 100644
index 0000000000..7b9bd8d6af
--- /dev/null
+++ b/scripts/base/protocols/conn/polling.bro
@@ -0,0 +1,51 @@
+##! Implements a generic way to poll connections looking for certain features
+##! (e.g. monitor bytes transferred).  The specific feature of a connection
+##! to look for, the polling interval, and the code to execute if the feature
+##! is found are all controlled by user-defined callback functions.
+
+module ConnPolling;
+
+export {
+	## Starts monitoring a given connection.
+	##
+	## c: The connection to watch.
+	##
+	## callback: A callback function that takes as arguments the monitored
+	##           *connection*, and counter *cnt* that increments each time the
+	##           callback is called.  It returns an interval indicating how long
+	##           in the future to schedule an event which will call the
+	##           callback.  A negative return interval causes polling to stop.
+	##
+	## cnt: The initial value of a counter which gets passed to *callback*.
+	##
+	## i: The initial interval at which to schedule the next callback.
+	##    May be ``0secs`` to poll right away.
+	global watch: function(
+	                c: connection,
+	                callback: function(c: connection, cnt: count): interval,
+	                cnt: count,
+	                i: interval);
+}
+
+event ConnPolling::check(
+  c: connection,
+  callback: function(c: connection, cnt: count): interval,
+  cnt: count)
+	{
+	if ( ! connection_exists(c$id) ) return;
+
+	lookup_connection(c$id); # updates the conn val
+
+	local next_interval = callback(c, cnt);
+	if ( next_interval < 0secs ) return;
+	watch(c, callback, cnt + 1, next_interval);
+	}
+
+function watch(
+  c: connection,
+  callback: function(c: connection, cnt: count): interval,
+  cnt: count,
+  i: interval)
+	{
+	schedule i { ConnPolling::check(c, callback, cnt) };
+	}
diff --git a/scripts/policy/protocols/ftp/gridftp-data-detection.bro b/scripts/policy/protocols/ftp/gridftp-data-detection.bro
new file mode 100644
index 0000000000..15acfba65b
--- /dev/null
+++ b/scripts/policy/protocols/ftp/gridftp-data-detection.bro
@@ -0,0 +1,83 @@
+##! A detection script for GridFTP data channels.  The heuristic used to
+##! identify a GridFTP data channel relies on the fact that default
+##! setting for GridFTP clients typically mutually authenticate the data
+##! channel with SSL and negotiate a NULL bulk cipher (no encryption).
+##! Connections with those attributes are then polled for two minutes
+##! with decreasing frequency to check if the transfer sizes are large
+##! enough to indicate a GridFTP data channel that would be undesireable
+##! to analyze further (e.g. TCP reassembly no longer occurs).  A side
+##! effect is that true connection sizes are not logged, but at the
+##! benefit of saving CPU cycles that otherwise go to analyzing such
+##! large (and hopefully benign) connections.
+
+module GridFTP;
+
+@load base/protocols/conn
+@load base/protocols/ssl
+@load base/frameworks/notice
+
+export {
+	## Number of bytes transferred before guessing a connection is a 
+	## GridFTP data channel.
+	const size_threshold = 1073741824 &redef;
+
+	## Max number of times to check whether a connection's size exceeds the
+	## :bro:see:`GridFTP::size_threshold`.
+	const max_poll_count = 15 &redef;
+
+	## Whether to skip further processing of the GridFTP data channel once
+	## detected, which may help performance.
+	const skip_data = T &redef;
+
+	## Base amount of time between checking whether a GridFTP connection
+	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
+	const poll_interval = 1sec &redef;
+
+	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
+	## increased by each poll interval.  Can be used to make more frequent
+	## checks at the start of a connection and gradually slow down.
+	const poll_interval_increase = 1sec &redef;
+}
+
+redef enum Notice::Type += {
+	Data_Channel
+};
+
+redef record SSL::Info += {
+	## Indicates a client certificate was sent in the SSL handshake.
+	saw_client_cert: bool &optional;
+};
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( is_orig && c?$ssl )
+		c$ssl$saw_client_cert = T;
+	}
+
+function size_callback(c: connection, cnt: count): interval
+	{
+	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+		{
+		local msg = fmt("GridFTP data channel over threshold %d bytes",
+		                size_threshold);
+		NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
+		if ( skip_data )
+			skip_further_processing(c$id);
+		return -1sec;
+		}
+
+	if ( cnt >= max_poll_count ) return -1sec;
+
+	return poll_interval + poll_interval_increase * cnt;
+	}
+
+event ssl_established(c: connection)
+	{
+	# By default GridFTP data channels do mutual authentication and
+	# negotiate a cipher suite with a NULL bulk cipher.
+	if ( c?$ssl && c$ssl?$saw_client_cert && c$ssl?$subject &&
+	     c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher )
+		{
+		ConnPolling::watch(c, size_callback, 0, 0secs);
+		}
+	}
diff --git a/testing/btest/Baseline/scripts.base.protocols.conn.polling/out1 b/testing/btest/Baseline/scripts.base.protocols.conn.polling/out1
new file mode 100644
index 0000000000..9cba678461
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.conn.polling/out1
@@ -0,0 +1,7 @@
+new_connection, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp]
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 0
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 1
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 2
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 3
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 4
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 5
diff --git a/testing/btest/Baseline/scripts.base.protocols.conn.polling/out2 b/testing/btest/Baseline/scripts.base.protocols.conn.polling/out2
new file mode 100644
index 0000000000..8476915d0a
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.conn.polling/out2
@@ -0,0 +1,4 @@
+new_connection, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp]
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 0
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 1
+callback, [orig_h=192.168.3.103, orig_p=54102/tcp, resp_h=128.146.216.51, resp_p=80/tcp], 2
diff --git a/testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log b/testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log
new file mode 100644
index 0000000000..dc007e4e24
--- /dev/null
+++ b/testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	notice
+#open	2012-10-01-17-11-05
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
+1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
+#close	2012-10-01-17-11-05
diff --git a/testing/btest/Traces/globus-url-copy.trace b/testing/btest/Traces/globus-url-copy.trace
new file mode 100644
index 0000000000000000000000000000000000000000..b42ce25bca7414edde499a36af32130444a0ee85
GIT binary patch
literal 21556
zcmeHvd9>Tqxv$SjLJ}ZMVJuT9<pMRt8g02ii6%?3Ez7cGOLi#WQSu}YvMpPo1(Grp
z$~@Cjl0r$#6iO*m8A~BdWoYSb%X~{)=CLhh=56`pgq#C6>#g-(-+h0SwKgY~lTY9N
z?Qj3SY47vIJ=d?9^v(56|M%&qKHmiRuw8R`@<n*AFADzbjRn5_FTCy}@c(Ttx^aPz
z_4y86bHg&<vMnxu<*pxx4KT{DJiO(HFRXuRH&4>)$?$?#<38Wy+<7<5om{Y`FXUe1
zJ)f8}XTB%-AY_K*vCJv?o@AfTlN`C!lj=#|^21jiUS=K)0qI{_y#sOPqiZ*neC?s0
z^l8cW0?8D3lQ%x6Gq&dEW1ZigrSq47bojXKe0K0mpyR|?-)nuLYY+8wJ$v<h-$g*0
zH>T&mQP)#fE%GGZL(S3^_H;dNS6}ynou?(P+Y{({ErLKx@{P8cU&3{(rHGnbqLjOX
za(78^DU2;Sz$g}jSgBwfv;;;m1jk_<Ar2<c1D7o48m&U#UZS>Ije51W1Ye3QMZU45
zTh5n8OBZeFDSG0X`J0>ijpM1m^#xNu*_%2xm{xSfnZD4vJw1uS+W9^SOxqiu1Cv~H
z(^%r4zEI+>KKuNKW=bT}D!+tN6f!AHDZCtr-(;<3S^>6dn=!H%&O-KNfb9N2;Vx!Q
zi(j|5FSO!JPhsVT`Mw@d)*GK&tIDcv#}Yl8+SIP5B!aa%|IUpPzwAjwQEW-tu3E;D
ze7|RwtzNa5?^RpYlHRCYK4b~Wkm{0V-|j7`v>J_;GrbIV&G-Emc)vG32ikAf+&l(N
zng!bazyfat_F^Bta$1KK7l38>^7thYL!?N~rbv>qNGBp|2qYVgnqbic+Ls5TzA-Ep
z?Fd>=w-q@OYz9*S&M-8h>5!ykkT}U0WR;|Ol1Y;$Noph|lC(j(BqNYbgU*p`noN=8
zfQ)8IStcVS3H~RO3Q5+z58x|<qzs@+k^nl8EJ=&t$6<#}E#n>!&iB1C&o>^kL7(^Z
zs_nc9-~Prd&~F9Hcpq3scgvqnPk7w{0Q3t`6dxmd*m%-Rng(wgPER`JHrY`UtuzPO
zw$#B%qSESh#7@9T3;r_Vha$~>E^d%yLMBPN;wj65KLBN^EJMmln083EFOzJ~AZbaZ
zX|k4f41+7TNRcLG0Swk*<ZLw7$_eJ6m;j$6luSAr!zM)1=14fr(x%9!&}ciyu&PM0
zQ_qHCEQJgQSu5jaxQtt3SY4#j)89!&q|qyK!-fp8y{tBL@*F$lR8psmM1~?om1RqI
z0g1xJ5vAre*eP&n1N>cwH;V~T6&wi+Q7w)VH5S*#&%~L}YLp_%kV89JCL_x+)=BZj
zW}_~qM2D5f&*H4p<_lc7TGA8goSuNhR1uic^mn*B_1+94E3jGMsa96~Y<?t|U!DuQ
zB|5y}JDRdwGC>*jj#;mAVF#i-tbn;uHGr3qt<y%9CGC!_%Q2y&z%*CRRh>#WRWc((
z18cB3Lgwm<0r#?0H7u1CvqYtZSVWcmJZ6MD9jLA{klRt~JQ6L2>MGR~_$161^Kx{6
zcEwyF>~HiroK?yZx{!9E5Y)}IoB3K&72}1NTXhU9p+ZtE;`Wsm&D&A8i}eX+s74bF
zC}dYgeMMCgbvsTE&@34RcIE(E$%`408e<mC3ORwy7BJahDKI&PG*m|<A&yo$oFWuZ
zMh4I2G)ZC{Me<b_p~A^tte<GcRkt27`r$z-<4^fh=0HQ}B2&wy%8E*9eq0Hdg58Gt
z2pt{LY#$rIu%L(tqxiLAwQUi2V8ACG!p~HaYOWu^3TZWymK#A;BTGf3Y31A`Zf7zU
z40V$2R1U(NTEG$OTBr}>x&x<cOnU(3x|&@nrR!2olZZCg&vl_#Dq&e7*bGrr6a>2H
z0{btG4AK!O(h#Dg)L|f<6M8Ki*!t(^DMpMTqmuC{n{BBSTN>pxdC0nCYU-J-()A5?
z45u1JR31iLGBf=hV<p-Jow9T31QpjJc9AnpF;#aWQ#Q-6Lu!M4ibi_Fei7INB~qBq
zsO(5&>R_;_s2U@&(ej?5;Mq{}eoj66AKRx!j+o6Gz%~HwW+74(xG32IHWIJp+JPZu
zw&6@NS}v%(tQifW$?J4LAt4&c)71c<GyVEd?<O6)J|Z2_9vWdhYe%w3HyI}@fl)8W
zDV;=~!izoH=!-oOm}|AlxuO`AS^=mm6zka`+bx=IiR?(-B;N|#IYc8cy%CK^L?^~D
zePS4s+)}kQD!B#5E<j<z4@s8H#Re?}qXxM`f%F^>$<<OS#fT)~@mPs<SSsMi5s?Jn
zRS3`BUI3WC45CGE%x3-xZPoT3^Z&9tlljrYnE7}0xvRIB$@~uQu~7<=C@RZFIFSp9
zP%;wMtDSz4<+V5_SBzRouL9jzj@D>SAvw;^0_&A!7FaP!eU6dRV@5J$k}wI27x19!
z6pGDu(n@eO7b+<tkr|Mx(XuTj;qTQOLb+*EL!wc(Ni@k4q?Ri-gE4o|O@|Oc!}zu+
zCx{LiS5vi2sc-ers8GThQVC1eaV-utsb*I+(_N#}1T)B}jU+P6n$W1?CWADdSM-#g
zkxZh&rh=4#YNCl?`cMlcL&<tK5Q`+82%Q^yDl)+U4@Sw}&LlnWy@|W!9-0v~&5X*i
zHMfl8i`Qqx7xyhQu@iv1U4780)1a@o7{nI`c*}UwA7(i484~9`z2;U=;-$=gCoyer
zlz53Jabv_n4Tv~N7D8ZPBnccCAe}e^NGGRFlMb8cQbU%R?MT3~F_if)aA=Z9I@3-$
z0-Rwjt2slqM#|HUm`=9`g(`5uda03j$#TSjhGLDx!5Ai7c_h{%`OluQPC}JY@0rM_
z|L>Edj5!QB?VvLtKuDMchE=>+&<jL>Q;sUla9@Zd+9f2@6;lFmVdKBguudHZ0Rt?$
z0uF4+(LaZ==d2TPfU`i>S+X}6<k~SaZROxdh1QEa;TD=1C0%V52hDmLR|TaV#acu{
zrkS=<$J2vh6o-o{uCqDP0+B`Astim1qU)N0QBy{8AvbHw0tmWDsx<<Ej7yd%!{}0m
z7f=-`1~^qfCf<5DAW{%wPS$g#eB_G*FhHa%87Iq9+6om26RMGVBM|hrJN^QR#uGSc
z8)`pP>UIb5T7W0aatJFoTLV=L_9DY(r5WX0dKsy9`<bdM=V{go1bhBe7*6?zitaYt
zCewzUWG);?CTk91GUf09YosDspz}xyn`I0OTWB^LL1KBOl9Kb`7GztQ5f);I3f)T;
zHNP~%M><<^rJ!yn!QQPKAh3+Gz~-rL9&8DNrM>b0wPSTzM})_Y)!pc9r=B?=82rB;
z05dh<NyY)N4dNmYJR=c%rVTe@1qAbg0W_!C&i|g19kHPr7982_K(mT!sM%MPZXjSe
zQGt)k2|w14GTB_q(L!#LciR3?IEf}nE}0q0L?+Xw;RXXJfzVw!88GT;7ad5hsfW{a
zmu-+y7Ej3)CM3oKTE?n$lbo4sK>3;#Q-dK=E_OmyG8pH`5Kbo5_9)Nbg+bA5qc{k6
zx=AG;pn{30hDr|G93b$tlZ~k4IFa}Qm&OvG+qcs$jh4Fv2*9{O%?RehaH)!9X#xwB
z{Ry#3wFfmZPgVnYqn_fSbgNjj`Cyi9WT6xtC}U|6Z<IaezZfuo6{K$7n4PQ6TfXKO
zW9I+WOy>VDYwSyQ^{qbe4>OrRrX~nzn8DRfE2pTvB5(7o*wv#AAtEOr5TBM|H_WjJ
zO;FKP$cXbbf3jW8M%_@aU1|>%3q<c}!qzQ<ijBHBoQ`JvLX7q&94n($#cDFjChLJ(
z5Nm++t>lHTmI!tkUz|?gXqo94;mEKW&r7QMg<d85FYqckkt$9zvOJvzdaOJScNL0}
z)!Cke#l%{4>`4+X@Gfc{c$cYX|6^gcz*T0m6FHUMBuIv)s&b4nWEw;A6++JDOBr>f
zH=!XEX+<kh6LtoWksy0bqcs>J*-C@R1YIH?6%CPzi`isZi&I*n<Bl9YKy=Gyy;IYS
zvL2UP@mgF3g%qJ>+Ac$@on)A=@=;Dq#ZUpH5V2av9R`yHuGAPv5nNG*sz@77Xdo&z
z8_jk^tB5!uE}X0+drTXms@)*lCku#%I2*hVly+p7OnOYir^cZxOR0d7Wj-KM&^WYZ
z3|XV7RE_Gp31(QyV4bc-QE=7I@|b{TuomN`I1R(B8<aUPy)pp{(sodqD9#T1y>h|W
zTP1}`bU2dAb%H~q9Tj&|Eh5aPYspB#VX;^|>bR9|E~@bDE?KNZ^)8ukhe<Nh)T8~X
z9SXuyxEkqGL6v1HF@uJnEUXvWq0&g<%Td)TGq{wghNDKV;|iG=7Hc;tDwD?5u&oIt
z(IQ;G(bwt?C<CjlN;Jimjcfw038j97$n;&mi^l>ZQza<5DQjNpYl6ZMDF~zyO`+XJ
z5d+N88|4tT-N_VNP;=DFl>M<}Q4C3CAxL+{WStRATqmU<+aU`Y7Rz;2rpe@jl~5`<
zB&nzyFiQg|1d{A78?{O@QY}>3ZhoW$lUZ77l(O9+uj_7J3Tt*qb9;JT%m(|}be(SF
zRWoj4K^5(%ZNHK+nJ$us<Fz_&7L|B3uLi4bV^C)Cafyh{CPAo7gG$dJS!%erfJJIm
zlu<Y-fmE_dI~jK?dEol_U^hlr{Jd!wYr(W`r=>22s^wU4<U}HAwpz*#1jDRGaJ^Py
zLo%6QX_O;D{mV;*SZb;qMK&`8#C3GHg;ib6GEzC9p3pQ$Za7O+<8?ir9T|gA2d~H_
ztYa2LWPrGB#8d-Nnn2wKv7u&mIUwH6ls<Rxi1(gzXJ$^=2`{Yw>v<rEU%St`Y2sby
z2Q{-xJSu<l#reJtsN;KMb}lxzxN5N{@y^m0O5D}A_L^fhN(^`sXI9~QEl?VXYhD%t
ze#&?!9M>C3nU4e%5mkm(afH(iC!$M=6-wJy-b8v#9MMF-)u@Vn3QFo`z8EjoDsbBb
zVOSu}NlGPAYAbZZsD!u<>#~CgELBQ|)Uh>5h{r093=>cr1f@}gL--(rXaN`D{EjOU
z2@M|f!+Im{3`97C<!hye*b2rtx>(E)buHcTyX9!V2Lyo_P1DjTg*Ek-TpyAnTCRlC
zl|UTIPz+l~Oa71>NkgO{_4rN*;+?ukRzir9i<Qw<RPRZNLWK&Hqz=?ASV+8H&?_vS
z7=`tG!7|KV+8&`qfkM-c;|vC!Jl6K(rC!S_h;CtE^OYJ)cZCt>Z!?z2x*8L;2NF`L
zDLiig--HCUc2zX(@}P#s?Lj31Q_Y4jbVv<^n2gZrhYeI8vE?q3q>`ORHXn6{`L^!h
z67Yf{Etu2&Y&VsSrl_HeAi*?VK+;1u4U^%Jm{IUls+H__Ed;Uvid;}4@>-%yTPj_p
zSdePgidw*k%h5uHSAtBV(1!pOv6*;BqYQ?DAqS82BPlwZYS0~j&|h(dLOt9V;Dn&b
zIl3BwVGaPE1M6xC`2;C72T{D6qdL)UD{i(G1dF#@T1~4su}0k-H9I3G9&`!`Wi-&_
zf=z^sJTn}Le!3$ET|}pP6cipPk!qC=C2&hA@+>ye`*})fX&4%-H9FDofNF)@0N#$$
zR*@M*${j3O=9<xg5p(*Ysj0l*VpC!yVPq}6ACU$~nyNS0bdAR3d?BAs)FH!4VsSLd
zW#HI|73v_LO9*(kQdfJWgji1Ha+Qu$(o+5SP^I`3LXL{TxXZXfMJn(p+v#Soifg1x
zxZJB6Hdj-SQUe-=;9i9+=>#RTHB6SAaI4=JvO$#KrI^1sOlLuno$HLc`S!@qNpc+c
zC9xG_U9MI#NLhqwbeQ1jVLwCn5xLGF2}*O~8kTA#!pT<07^dN#sOp2XC~`q_5F#3e
zC<{tDiCLuNSSh&Nbc+3KjU>x@w^>T3Ts_d#6M>f0jt$A8U)0=qGFG5<*NLiGR~+QX
z(JV@Xn@y=#PF7K!5Yde2Afq_kFLkVe(`WOH+AsSv)uuhb%r1t=!>S?{vneen@>nX2
zqE4*7fzn!U%=dj0^gO&VJC^jd*4*Y1(B<o9#*%+MZ<$Hl0Qw#`7C^gXs`IwuGEhYb
zOjQvaz`Zn?*PEK3g9{Re(aE|6N5fFDVqs~M6ATzHjFLUKiq{GvIWVSs61VKO!1pq!
zN_k^88%{j6YDZ6*yI-BjhQtkkUmpX?tbPBPX=T=d=HiOWK4U{qA(c{xbTi2SkLKr5
zG684Uc&i25StFIf<wQoW<*F^AX2u$Br=4qe(`qNvbIY+tq|~Xa6@3_n^1V(Y-Y8XT
zU6l+Jg(?d&oCHPd7|B*DB^I-wsEt<9QV~k#6g^-zBjFrjh+H*GD-|O`Qth_IIB_Xd
z3`bnNE#on)1!Da`fT6`WE{FOxn61Y#M^YLzS}?+5upJ&{otUmmfp9UEEajklyU(#{
zE9h$ZRw}3sO4U+YYcZ-;Z;~NoSc^J5Q^Rwh5298@S}|J&9#MySw%7!FJ;9Kb0tGjr
zF3}*$qaibl<O@b+WV4!D3DJp;n&E8$HrnCTaER1n2{^0|P$aJqT7g3=!Mw>USh)t}
zDt@!=M=_yTLd|FiPoxG(xL}7IGMi;%Xn>9x6`WFog%*c#xdNLXwc@m&396AQ(iyVJ
zE>UP3uGvUuEqkgNwP>FOzQgAD#$&b>&Izo!-COuIUNdTwp7mcJTV{T3Iav5#|7`vA
z!mqdvSmB|cSz(Pb(#beqZ)2LDAC4d)$QG<dyB{ZD*gptUh(=<C7%~c%8!;N^#Ze+?
z2@z7#J2GyVIxh{BEE(yc6*~tNsDzj(HiiQ<<S&xRh7^mLtx(8p#XGK*4;LA%JV>Wn
zxuE5kky1FQFpwFRRM28b=)t0ZBh`K=QS7N0)oqg<q=e+9ez}eJ^`?cfDw}HR0uo`m
zXkOORF|^jiK-*8VAsmTjidrQjMIvSj^|K&&H%DayX|;?h90<1+yvnl4LbDr>(wsnt
zD@3lvH;i-+b?OF#!-7!Dg-NUyF}ec?iPl<tq}CIYnfj0|1Ug8HYQ*JWT*_;Ns05pQ
zU#gLID-r6{comZ>WY)0CjeHg<@^k<cVe9pJy2EGlaG}{Mxk)!}*Sdlq5zy!$mv5zc
zxD+ffh|;5!5r9C(+G^LZD^XOX1A=A}k*Z*q`&1D%!8Yi2B#n<H8D-cL>Zsahg+Vi9
z4u}q#uNq3X%Tx?Ii4trEHmZawa@l^QZkuEg1}!x<Vx}{#GAK{E#-I&X0!+1BES2yY
z$q6{_<l{jzp3NbxL@(Lvle9|fYAl>ZxOOWM(^-iw=LT^)oGr9+TBlZMmDuLcuQ$bz
zZE^WDDpb;iP$8ApJJooz+RlvhAsk0XHAPn;ClhX1eJU=CwPrY_nst+}=h2j<2UT+*
zhuWiv8W-e{3B?0wf7lDwGC5vWnry7qj`vKR8=*pUz%iq)+l@i~20~W{ZOF0Wkw!f?
zj5iowAVwL3SJU}U9Arp^y53243S=S_9qL@3st5#I%@|O<)mBL0y)l2wFE+C>r^wSR
z${e`BR|RzLjoB>fTfS;1Z!_Ifo5`YITmHiOmmdO~>5lC4)0=7CGQgtOA)vE6)&5^X
z&g`e#{bI3f+m(J}Ll6B){{kO4iZLFucd>8fnmfk6<l>pT_~nPdE?(*FVhNl)+2Bil
z26pi>HbF0mc+AZ(CD-lRiI%2zY&k&U)s777!+4QN$Rk%vg*ZtKGi42p`P+3UQjVpt
zP_C@oHY|6cVs9|aZt!*Q9=yPJ8_>@ip9ALFRXdM?*?AT)D_>au%5EMo(r0HvNC3>w
zf+#YjUxev|yCu-a%MX(^Negxz#_knmEKx(l8HMghIZo=;ExIISB`M=%v$4F?D2rjL
z67P2s;ZzUSGH4@B<n2ywkRA#Y-J}eQ81@vTZ-^*9GMI>?>U9j%Z8t9RPXY^kAo?GV
z&tYw?xpR#5H8Zh(WjBEJM;_J>wKigX1HhWxh&4?K`4*?NdjkV8rKU3kSyGefmf{7p
znJtg%fZg*}E(s-@ae1RIfv+v_T@G~d#^-Q7f7LEyTo0aw>qh`r#l!WZN2hgJ#{gV!
z*oZ4j;xaM{B8r<Smt3z7jMNet)<`xnB^w!rkT#9>nRZPL58IVyC6Mn%)nF=3$V|7D
z3Q;9a42`Uz8q$MUj?+pNw?l@}C^hAAE<0qi_}w+e@9J6jDa*`5j`Z*woj<Mfikkp_
z;GDq*Tcv4&$nps!Vz!fV%BY8f49~{me%y9gE=moD7?!m}*;PfO?@Vvl;$aJXSAzu5
z8?*77`-4@xdb-^Cm6`Y*awNd-G!H-YYty={V*!3Q0sIag@0$oM%1kBFp~4}))(I$<
zL5UU8vVy6hqDo;uUcl->Gpj1Sc9G&r$$XG*4O1F!wwrXbluVW7z8JCw2q+W<%5=03
ztBF#fgBFU0lawMUDOjq-K(`3g<_d%HTsFR$V^C+*D#P(ktTAvko*X(zK<xINx^7nV
z4mMQ!1|I?|m24Rqi>?4Kz*j2mpdyH6B5MqbDHkpD*j~NpuXRI1TuaC)3@KF2bSltQ
zXg>&>G0{u}1+{F|$+9E!cqprLjD+O^t)P~cTVTh3rVWMHE5)2nswl_7@m#d)BAH5=
zHV8(jM-xgQSBF|SQfX<hVT7%i9ghY^au!kBd01p?l`xr+1B!&iyG*@A<$1iq3l+DR
zNOA#j80_fMh$6aa!YT1sY?QT)ZV@3Y7em~T9DravTl8Y>&$OYNyij*bm2iz!JA?zl
zXg)4;Nyi1{5KahmGLBG)xE44LRPT_Lej=AHfNp#YO}GQBNGdU^!zIHFe>AF^RX3?-
zEwgGnjec1PQ+~MD$foN3a3!M+p}46dIlaJ-D5q`Y<*eQ?B^u>tVnZTa2x4%m+e>u0
z1Owq}E#}ymm@MK|qLP6_m_qdG9Bjm3EQ3gi8lU2lLQpUF=^#|ghuBfI-m&Y6juGlg
zxvH3tM#6Q%&Qt9uq}59qxlU%n=>UdM`DBeDib4vt!OEi*-Byya7WGOin_$xm4n|Qr
zmE$`-2hOybN<o*QELwK_tQqnHciyv|pc)N9#;D!zCj>!k@Vy~db;HuAVxsV%>gusZ
zF9J>-WHFXvN+f3$2;7F-)w-09DdY%;yNX#J){_~c#fq@gZcD`$T7(nvQ9dfn)TU$!
zC9(;dF4eIM1uB$ck<w6<iLgzT>q=KOK~0r13e8xz3Hke33nVuFK+P(QK-YPsx4L9p
zj|RY@kWsxy+ay8e$x%@4_fuIRS|tjV3MFSUosx+1gv3z|4OF49W?Cy3AOsRI9Bish
zumRn^S|wT=rbRQJ;JYn`P8)hs43lX|L)y8K2GwX#FAQ{2I0_;y!f>)6lPFiCtifs$
z3xm2z)$eK{!!WG69wBngBo^&w#&4MNV>C6W5_u#MZIEadHNyr~>xco6x%RsyD_{mv
z(Hw$i>u76)(i^$!M~82gNADi<=vA|L^t5H>;r9dX`t$eBnC7k(Hv=AJKjTp{IH20a
zA*f98G~Km^L=n@9At`M4s<I{(dPXc(8foPY19vw1p|_ST@ck0>M7%MZN9WzJYBx`p
zI~L94(ZlZtJi6ZF(cdxCx~z);9=&;^RWf9NjODWJ{18;Z&|XVzr}8A<&RI3w9u}FR
zSgMENb|;i?6dEGca=^Y3;CQ)Z%kd6ONk|adbRB6B@Ietf+ZwWgHaHy!V?#fUs;QCz
zldKb?x}(^L;>uNNG<3&@^)~83li{9(C9zyGZlwo=&9tJ>AU6ng`9hj1XM3Y^P_{|F
zYoT3bCLc3Qj%qh69Rl`KWUx0FIW{Y}g#d0+sfZqT1{yhvTLh8p5v*;;L(!2G;d@lS
zPsdb0RE$<<SRy!ohoQA0i<-U6&}GxQQ%@#F=}5(|6lj<b**-0z{z0z^CmZ9n`D_MM
z!mnrAYCg-<EhiNW=-o73>UVpDUQmOEWX1|zu?&YwZU-9|1wQ+iOb(_p!9*|Ks0d`Y
zUrsaibe}2Lw19wQKw6NA^kiL;L$$U(3pH9Eh~4<G1Xyg)rOK@WI>NbX%O*RaQj}Jx
zYOcwTs&t@Mt;~8ST^u%gQc$o$TA_`X0q1gDrfuRCNlt`^^{`(nX9E<6NG3CDo+Q-h
z1jv52mm-}4n=YCmj1ocTLaO%Dtzes1MY37qZMCP@XT6gI8LAo}vz44sOv$)q$!biB
zg41Dktj?q1U_muQM3adbVr%9+V@w{>ki4Kba~V)j)m1&m<Q1zY3S~vqh=P<!mGz)i
zY2_WYswP17-3cNkeF)gSmx*hmZltaVMj1s)?S2JUT$5{d6G}&`CBqhH7o-LgCFoSA
z1x5UTRisR%CK17y0+aP*dNgb!oqj5+=s|zn%~n8>G@7kctI1p$>llNi$#HC?pg<&n
zG|f^F98s<GgTYuT%7*YL-w4-6W!|mi{Ka6&5uz%@O9e5SE44(sz6k@;WWuEMD1=1<
zH9J@_0LP`{pg>VGL9HcdRFbJ;w3~~_8JzXcLY2v&c1BIw6{_Cu+07o7vOz1;-~)7g
zfLbIURl8c+N@G>BHgo6DKvPx~27-wM8al(}%4K0xX6s>#F1EB*xtRkeBzdzHw$i<s
z>p_#vVxeo6>WzkEf`gX5TAYw&Jlg2E3RK8+#Bdn#n~JS;HGZb`FccmrsYC&!Fi|Ih
z1Y%?~?l-i~5S;HUhy}CT7vOxy51Ab^fF`QpT*@>WO|5HbRT8Si+T(oZvz<d0iZ!v?
z#wn<*Vtk%w(-qS$ltE5OCqx7-3>DBg0yW8^9i539Lje=DTdAHf>K5!)QVXi(qCjG)
zaxVviLn9#~-XoJCLm4*Z0o<+)GrX1%!u|lC!O3i@fs0eTH|<?Q1z3)chG4-Fqz4Kz
zGS{(dCI_`EwO*}MY?o{Oh#}=`LSc-g*B-foUf|mf6a&36JIk4vT=UDZckwc|P3l6g
ztY2obJYe(>UfOs_dBrWjyF@m6mndD!xNVro!m(N{R!jxdI#wpbLS4=b3%Po#iuH_K
z45V*iG+42_c4pKw<3=bQE=0jzWHLQG9V<j432-jCY;^40pfoT8T%;`Z4K^qg%#f={
zXr?dWP82C%bUnf7Y`+9^DjH-(TpbWrzE>IPR-zS6j#3O#&6VR)B;b@(E{hqa<JJN=
zhS-YhZ-meQj-ph!R2szmZUA)CL<ToKKKAfAfSuueXP(8kR(;9C=bqF|kH_*atbg@9
zfX@?zywT%vz~kKlY-Yn+(b1mR$k4W(E=IYmo2KMWEyqD3436LoVr;kWmm8rxA&io-
zvCVk8oDi9z%Z&4AK5NZA<F)$9%(Z$IRFYXQ`~M($&a^HoZUt+_ZCopwGPrJ3XthFo
zK}Vo6LyjsDIZ4oh1x_GktwB*{V60xHhP}u}U3%P%O2kZDR<7E8jLTuOZ~>YAtB-oP
zy!-8GUDojcms>aD!cZ6<EY-zsz+cQ4q(rVHBw?&z77KwWDpfT#nkQALQnxY&Lu&)g
zrg+^LV3tA~skFuCi^UD=wnCVp-%PAluKCp%E3c8biCw??D8R}qbbs*lant&(_yxd<
z--s2%RHA;STWl%ec-*mMyFRFtfPXI|WjX0o>adgNp=Pd98tK_l$||Q(wF0NrYAWnz
z5el@)rn>^CMrY_Z8^1lq_#HY6zsFu!|Jt4&ezn)9^;;(Z{C@Ene&8TN9UrlUCTBx-
zJP@GG3YFK}`L@`j6J;dick@x#pWwq*GLo^nQFBzR*Q+ie6LwpOB95^Ezb9jxwY7W4
z_^q9Z-)nm=Gb4L>_<bogt=||wa9nvhd#4B%sYgbwdP{Gq7*j7heZDiQHv9!9AI}JS
zD8MAa!G?M!L~gXTKcqHWRoZim)wVOSitGij(mbqQL#B0E7n^}qgw#g;Af74Zj1Z-V
zs7442K*J>0&1UHWlORK#p^|B4NTeUpl`=)KaZ(HG5Zy-MN+xHZm{Nyo@p#p(nP|**
zYz}ne)S;Q^H2VY=<`T%Dq{~s!FO~A6jeJXI7HsD3*u+wAE^KES*HaFEa&_f-*DLvr
z?~du`ZhFIO#U0-B=^I|h%#=LSq7L=>JjtKk9eZ;1DTkjg&07YfpXJ>hTlCVVlGnun
zQg=*Cz5qzxOx5u|Px9qwPwPB&-|aOG*usULE!+bx9c`#O-U0SfY@;nOqzP^p_!I3>
z92+8hAXyUJfLl^C5*`~^(4bdi@)}p->O_z1h>}!cYE?cxiYKxa4woS%WpA+6y|o3K
z9vYar*RhGf7}Mdjej6}73(TGbhX(B4<1_S&drW)Y(F=Sx0&Kl8+bjBhJOPa*&iO)#
z;I0z%z($F8ff>hlo-^TV0smT`Gm%{0-0pzKuetPq^G?6@j=2vSJ8jjw{0DQ7G~&km
zqO<t#;C)~BO@fPtVISc;*>{o;_kG*9vv09)OCRb(d>4bucC3%_1$^iGNZ&kgJ#i=B
z4!(uH9evvaHMj81@qG$jGuOB8)IG&Vz%2gz!xUF8@@Db<i@+?#x4t%51X+20#XNvZ
zx0joq#kvHrhPymmKB+F)%$24q$2Zf?_pZ`?p~RcA8zst~L=P8y(&vMxaJlH%$sYB?
zi->y<%Fcf;vUJ-U-rU`|?7ef?_1{}!0OR!e7J}J<`GMJif4Okat$Yi2^DW%jH@Q2s
za&i}F<;2~q=S)saPR<RT{lW>*qAeDF^+f)}e=eGsm^aS{_2EU(R$I(T&)stIT<?Y<
zycpWv`?%HOEvIe}s%5(e?*i>K^>7=z+H5z<G<op&QX;$!wDr^%^BS#Uz5(w5ZRdTS
zvv^??+<Q~1PPx9xvoIXOQ6!9E&|bSNf)OZ;K(O~LnA`9XhHxx|z&Ysn@BEMGwW+Qs
zf`(9(fT1Hc#cX1pZ%)hy@sp(Qp_|V-Ve;ymPWfcnE1xVn^ZD9iu|pz%nD@jrUmonr
zv*F|0>r>aix&7h45#mdi8gDM-KD~nEQ)m3@oJW5D*w>%j>cel%p9IT1aoJa(FGEYV
zSm56REXVwbZ6+o@ox44>EqLWt;L+TP?<FTDCKhh00a&#;q%U>c^*jIY)K8XB3qRUp
z%N-v&@5GBvd*Ft`<aZ;_eYy;aTraL2T=P~22`zs7C&xY858labxj1t+{p-J+aT9d^
z8@F_}dDAxu0qZyjSjTxAtRr|`XS-SNf%5<IRoKK5LKp&;ezqn24;koxx9B&1_{YnC
ztX^^nd(3`M@BQ#o7pdo-a#G;p=qUc}pX`6ip}23=j}8o{J`!Hk7XPKQ+vG6^?sd^~
z_dfUiM~Pz}ECr3*{_xYOMc)V78`^W`qJy0TZ8`mDr-@lBG3WT>eFx1yJ@u`p-q~~C
ziP5pIzEXSty3j$-F1vp28?Qh4>TNfFy7&D@zIoCWKm7Z{uRPwl=>6~Pb;7exe)Wya
zL64dH?EmRi-{PK;Kiu(xM~?XGHkY&yyY$#Y-+Sww?Vf%2ll_>?o?C2v`n$W_{K6mq
znLmDT;N_RHTikwGY=3R(A=|G1<<EE7`MuX}d7!iR!*5)0-QRtaJTMssn9Ma(CNnWP
zOG`-qKeLzJpk1ceW4jHvmz6QwETPFw%>;!Z)H4%w%palg$Nv|Wdi<rEP8Se?$NVd6
z3w#T_?uj=x+;*S7?LFs%E5|p8FMNC^^Kbd`*q4AC#Mo6cnSajlr}+T)O@h_F?b~mB
z{c8_Cc<jXo|L3#2pB=sP(@Q7$J#JX^;i=zQbfS5;^|wQhIq<_frKjK8;;<vmd^=s;
zDoZcl`k14zZ9iGK<D)Acd1R-LlVAPSxs%LacHDd0pKbMV+4$Da?)&!py;}G|>Ds%<
z2Y+$$HmB~o&HKK;zx=@J_pN{Kv-Vq=ogaKuSa<hJzsWrP+T4pj{`r)l?gV-MVBygx
z@)IY%M@>x3ojYf8?pvGU;L#CK_sn(cyJr1`Rae~pyFc7`bb$Ey&C`%2)jQBHO$MG^
z$#9QH_vqgJ!O?>L_;z=$IAQC*YNy1X!|pv&x$?wR;`j5f>uh~Zx4cAM^tkoLiX%SW
zRb2Vj4)Pn5>(6-M<?pPt-+1yHYj<2*L!Wr(-WT86;rex#sQVpt@&SR<e%a5QeE%6=
zq5u5J*Ka=GH|YT5w1IIxJ!PB=eXp!~ymRu6r;D?+{eNSmFo7VRk%pkK=i<g6|AlY<
zoNaDmNjLv-?MLK^m!uE9;Cu1j&MS}lk3HLW@{1<_xcY}54f5|Fqh0;cLFX)ikcW@{
z=lKWiblO#iUbpwBes{O8o@VZK#0>{5MSi>W-1~pHJG%8(UOD{nKkaeWcbRMMw|iTj
ze)Zcgy?fta`6IiWd-lEP<f?7ngb#S_nNz-d$*-?_ttB1vA-#54VcnXg6StrL)RZN$
z5Dje$cK4o(Jr_3*ngf0S4g?RjnY;7c9gADM_UmnT`Q4wcd~xoch4;w!_U#W>f6nrJ
z^JlH4XL*O7^uYRke{jy-H~q)u@BjKshyK04@1dEu7Oy;Y|E-_9=Iqp77vI%+N4t8C
z{j0a;UDUNsdE|p9-Yo3=yc=Kk+*f@!Jbs#H{ot7g_IY-{@Uz>VbKa@HIl6N7{pz(_
zUU4*;PAKo*dvfoAj~;z#X-?>A>PyF*%eY%C`cGufn~&P*7l;4s&+Fbf;qVEn<V9<X
z7WsVhK#b+{Z37}>j~8wRys$}(IXm`-$C2`2W~97g-ucp&FM}BKJJxyAys+XP5Gl)^
z#GC4ym57Wb_CGIin{`0q>e|hpmv~EmM$|e($9Ydrpq|9TAO7zo^55Sm@jg#SfWsD_
zZsptJL*Ewf{X4qYeE9qC=;Ghe#mp=T+8l%bI~MqNEHEqa-b|>xIZFbufagno4Vb?O
zyp=a**KsD(6PU;Rv{$_!SAnN0L&sfnz65*;xX1VAQ!|<W-_gatHoAyx=ly=FWf#!y
zK0lrR#{1u%+~cfgp4{Wy!p~p5`kb}@S!B=sPB-}bqaNGpkHqA<M~*#Vp78TSPCV(U
z#?rsLSDkDe{QfoXe(#XngVzVVKqa#6SXA(}W8d3x^(`OoP-+In++nBuCHug=*B@Um
z{n@|p#>dWE!ymcs=eK-x?vg75_cPp^Cr}UW&a^+e{d+qs)_)#-{-)En{o{^a>0-hY
z_>~=fzOCOn?BV&>-FC&9KYs3#yYdI#Z2sfUU;OqL?qz$gtgQd2`|<Jnc_#OJU~-!T
zL^I2<*p=ho`t$wsfyr%BBHLzN<@uG5fbiwg!Yk7zw@w8mvim%VvEvr_lHlmJH$Ino
zQCE(Co5V#YeWAo%eV2ZHW7}t4#*+vpGx_9P-{gOS0M0l0*z<q;<&J;ZzPR=Pt>n8Z
zd&Cj1-1bcRwg<P{_RX^%^Y1|IcjcO!s{2l+cYpBKQ%1?Fo<8S}(_8Z|eDdOh`tz4v
z|Jb_MzA^7uXN{P6;cx#KZhz)^t&a{n)j0U#*H`2B;75M{Eq<Td?vD7o6PNzi*$4jV
zyxYI|R_vJNKfUwD7r*(BfBvx$%G`U|5g*LILVfw%rQhB2!(&>n=WdpNa=?Oz-m)HA
ze^=}(<&=Lsms<Sn>5E$@9OnM^y?^|3^@@qzj(p;8SM9p*>)5_?&%P8|?t2YBX60Z0
zao$nE`|4Z&;DIyWt{PjukUDhjBgUD7A5I?fvGG%Xu)Wh)CLXqaw|2jSn_o?Top|7q
zbD@X7{0DSy_E*1OvhUVwcd&l>?1%e2|Ip$K*M%JO?dbK}y?6ij7ruV_jaPhg|HdKl
z`~LE`{?!ls?Z~(19k}~<q{UaaN1p_1>o2@XzNb{vuRLJO`%Y}XIPdQCvF2G%U$Xbv
z-xXHCr$>@+{Qh0(jL@Oqcy%}PvMc_+aMitmU#7ltsP)b__FwQ(U~>0Ej(zHx5BR4a
z_|5q@UHc~{SMKdiKKA#uJH5AjhfkIte$3KEd%XMWAD%a7zvma?_wnysi{m0Ey;`6D
z8#!1$>Y)36`#o`o6YS~^<kI^eJ&WHD8C;1x_2=u9?T`N2Rd+o6)NUE&KYqk~f9aX%
zilsXfYrlQUj+5W}?n`H$f5feafAH46`BxTxbl2pS===9S^5i|SZ63Sr?GNnG{_caP
zTy)hvnd=I#et6`6z4tD@bonE@?Rmw8)Zu?S;u5-k@{KQApLS7sw-?Tie3L%zwbL@K
z8|X8xIs)DP%yWNSz=jV!vBQ=1f9`eX0rdQ3)w^E0_hs_^yY9UmI&bowQ^-%g+&lX2
zGgoIHfq!B?e)1{ztPTW5+uy&}@?Ecb=;jNy&HU`y?OxiW_MO!8+k6mT_|DrW-Kja;
zGe>{v-Q91$d+sH_x#8_M?+RV^EBu8wOK*L5k6Z6v|882`d#`UE_V90ickgd*8gM^o
zoSRzwLj2<SXWwr9=F-#!@0_tUC0-VNYPVTC_IEe@@#4Rp@Wk@?X&;_@-Va}7jy%*k
z>*MptTOQBC^Ph3v9skn3zkGXv?+CCNyfNEJ_!1NNIN{J|I*FB!oG)#EJ8%*=9&qCH
zj$QE}NDnf*0J7K#di|J}PQ40yZ})P3m(yFXZ#AbE_{s`zFZ)lL(Mp-g4l7sf<-O~k
zuTAgesbBCHxe&Cne&n^X&H-81R4Zkj4)*dLpSAs%$cRj48c<V7gKkwdQq9m(G0l%c
z&2THEWxL5tI+Kl6W3^aZt|!!ZxN660?vR6;Mru^VTiyXGPrsi6{S<(MH)eB<kG*mS
zPrvW4n$~ZsWw8D2fW2M<`dunrF+;xxi{RJr^~$9sphP|{qmN&)?3B$C<l^xw&YAU!
zN5-$%-goIyYd5|k3m|yVQ~IvcHVf+BV^G7>5;uVQ*J)6Hjc$~fn+Yn_&d^XwjzTzU
zT0Nek2zt;^yZM3E9#~k-PX}{!vE3e(IU>Qd>A(P7A*gA5Lj<Q><0B>SozC#|n}7O@
z8PCW+eb_|M({FaqZ{<tCz#ctrd*6*meRhKHAz)y-CvnM4i8IXzyK=`dhF(W*6DPC%
zqX0uM>%R1;%V*5^Ay4A=XKYT#HS0Zzmp=z2ZqhG*?MKT@4g(TDSU^roT=DR<j$dEB
zS&8?LB|iIw61N8uzrIo8BO4|DbcV!PvtG5tlX%aG)3cshXAT4Fd<DSqi3Q82C9XRf
otn(w@tjpKWkT`2));uuQ@tIjVUIAu}k7vDfTH=b|0UeM2U&UvPssI20

literal 0
HcmV?d00001

diff --git a/testing/btest/scripts/base/protocols/conn/polling.test b/testing/btest/scripts/base/protocols/conn/polling.test
new file mode 100644
index 0000000000..a6fbc35f66
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/conn/polling.test
@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT >out1
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT stop_cnt=2 >out2
+# @TEST-EXEC: btest-diff out2
+
+@load base/protocols/conn
+
+const stop_cnt = 10 &redef;
+
+function callback(c: connection, cnt: count): interval
+	{
+	print "callback", c$id, cnt;
+	return cnt >= stop_cnt ? -1 sec : .2 sec;
+	}
+
+event new_connection(c: connection)
+	{
+	print "new_connection", c$id;
+	ConnPolling::watch(c, callback, 0, 0secs);
+	}
diff --git a/testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test b/testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test
new file mode 100644
index 0000000000..bb7b9b510d
--- /dev/null
+++ b/testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test
@@ -0,0 +1,6 @@
+# @TEST-EXEC: bro -r $TRACES/globus-url-copy.trace %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+@load protocols/ftp/gridftp-data-detection
+
+redef GridFTP::size_threshold = 2;

From 4cbf4e3cafb6e4e071970cfeb625f7029354d3d5 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 1 Oct 2012 13:04:40 -0700
Subject: [PATCH 216/238] Small but important fix for the input framework.
 BroStrings were constructed without a final \0 - which means that strings
 read by the input framework are unusable by basically all internal functions
 (like to_count).

the basic test now also checks this.

Thanks at Sheharbano for noticing this.
---
 src/input/Manager.cc                                      | 2 +-
 .../Baseline/scripts.base.frameworks.input.basic/out      | 3 ++-
 testing/btest/scripts/base/frameworks/input/basic.bro     | 8 +++++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 6eadb3aba8..83e9dc9bc5 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -2007,7 +2007,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 
 	case TYPE_STRING:
 		{
-		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 0);
+		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 1);
 		return new StringVal(s);
 		}
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out
index ebac1866b6..c456298062 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out
@@ -1,5 +1,5 @@
 {
-[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={
 2,
 4,
 1,
@@ -12,3 +12,4 @@ BB
 
 }, vc=[10, 20, 30], ve=[]]
 }
+4242
diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro
index df2ab676b8..faab303534 100644
--- a/testing/btest/scripts/base/frameworks/input/basic.bro
+++ b/testing/btest/scripts/base/frameworks/input/basic.bro
@@ -8,9 +8,9 @@
 @TEST-START-FILE input.log
 #separator \x09
 #path	ssh
-#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
-#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
-T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	ns
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	string
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	4242
 @TEST-END-FILE
 
 @load base/protocols/ssh
@@ -37,6 +37,7 @@ type Val: record {
 	t: time;
 	iv: interval;
 	s: string;
+	ns: string;
 	sc: set[count];
 	ss: set[string];
 	se: set[string];
@@ -57,6 +58,7 @@ event bro_init()
 event Input::update_finished(name: string, source:string)
 	{
 	print outfile, servers;
+	print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail.
 	close(outfile);
 	terminate();
 	}

From b4b7a384dcb038060f3e33fc5bbd36708e8ff1f5 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 2 Oct 2012 12:10:13 -0700
Subject: [PATCH 217/238] Updating submodule(s).

 [nomail]

Closes #889
---
 cmake | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmake b/cmake
index 2a72c5e08e..125f9a5fa8 160000
--- a/cmake
+++ b/cmake
@@ -1 +1 @@
-Subproject commit 2a72c5e08e018cf632033af3920432d5f684e130
+Subproject commit 125f9a5fa851381d0350efa41a4d14f27be263a2

From 5f3af9e9ebd474f41d2c20d64cd6ac0a37f75782 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 2 Oct 2012 15:13:38 -0500
Subject: [PATCH 218/238] Add new Tunnel::delay_teredo_confirmation option,
 default to true.

This option indicates that the Teredo analyzer should wait until
it sees both sides of a connection using a valid Teredo encapsulation
before issuing a protocol_confirmation.  Previous behavior confirmed
on the first instance of a valid encapsulation, which could result
in more false positives (and e.g. bogus entries in known-services.log).

Addresses #890.
---
 scripts/base/init-bare.bro                    |  8 ++++++
 src/Teredo.cc                                 | 18 ++++++++++---
 src/Teredo.h                                  | 27 +++++++++++++++----
 src/const.bif                                 |  1 +
 .../core.tunnels.false-teredo/dpd.log         | 15 -----------
 .../known_services.log                        | 10 +++++++
 .../Baseline/core.tunnels.teredo/conn.log     |  2 +-
 .../conn.log                                  |  2 +-
 .../weird.log                                 |  6 ++---
 testing/btest/core/tunnels/false-teredo.bro   | 17 +++++++++++-
 .../core/tunnels/teredo-known-services.test   | 11 ++++++++
 11 files changed, 87 insertions(+), 30 deletions(-)
 delete mode 100644 testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
 create mode 100644 testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log
 create mode 100644 testing/btest/core/tunnels/teredo-known-services.test

diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro
index cc3a40f54b..70026394e9 100644
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@@ -2784,6 +2784,14 @@ export {
 	## to have a valid Teredo encapsulation.
 	const yielding_teredo_decapsulation = T &redef;
 
+	## With this set, the Teredo analyzer waits until it sees both sides
+	## of a connection using a valid Teredo encapsulation before issuing
+	## a :bro:see:`protocol_confirmation`.  If it's false, the first
+	## occurence of a packet with valid Teredo encapsulation causes a
+	## confirmation.  Both cases are still subject to effects of
+	## :bro:see:`Tunnel::yielding_teredo_decapsulation`.
+	const delay_teredo_confirmation = T &redef;
+
 	## How often to cleanup internal state for inactive IP tunnels.
 	const ip_tunnel_timeout = 24hrs &redef;
 } # end export
diff --git a/src/Teredo.cc b/src/Teredo.cc
index 54676c3255..1f01086090 100644
--- a/src/Teredo.cc
+++ b/src/Teredo.cc
@@ -138,6 +138,11 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 	{
 	Analyzer::DeliverPacket(len, data, orig, seq, ip, caplen);
 
+	if ( orig )
+		valid_orig = false;
+	else
+		valid_resp = false;
+
 	TeredoEncapsulation te(this);
 
 	if ( ! te.Parse(data, len) )
@@ -150,7 +155,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 
 	if ( e && e->Depth() >= BifConst::Tunnel::max_depth )
 		{
-		Weird("tunnel_depth");
+		Weird("tunnel_depth", true);
 		return;
 		}
 
@@ -162,7 +167,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 		if ( inner->NextProto() == IPPROTO_NONE && inner->PayloadLen() == 0 )
 			// Teredo bubbles having data after IPv6 header isn't strictly a
 			// violation, but a little weird.
-			Weird("Teredo_bubble_with_payload");
+			Weird("Teredo_bubble_with_payload", true);
 		else
 			{
 			delete inner;
@@ -173,6 +178,11 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 
 	if ( rslt == 0 || rslt > 0 )
 		{
+		if ( orig )
+			valid_orig = true;
+		else
+			valid_resp = true;
+
 		if ( BifConst::Tunnel::yielding_teredo_decapsulation &&
 		     ! ProtocolConfirmed() )
 			{
@@ -193,7 +203,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 				}
 
 			if ( ! sibling_has_confirmed )
-				ProtocolConfirmation();
+				Confirm();
 			else
 				{
 				delete inner;
@@ -203,7 +213,7 @@ void Teredo_Analyzer::DeliverPacket(int len, const u_char* data, bool orig,
 		else
 			{
 			// Aggressively decapsulate anything with valid Teredo encapsulation
-			ProtocolConfirmation();
+			Confirm();
 			}
 		}
 
diff --git a/src/Teredo.h b/src/Teredo.h
index 84ff8ddf38..e4048d4266 100644
--- a/src/Teredo.h
+++ b/src/Teredo.h
@@ -6,7 +6,8 @@
 
 class Teredo_Analyzer : public Analyzer {
 public:
-	Teredo_Analyzer(Connection* conn) : Analyzer(AnalyzerTag::Teredo, conn)
+	Teredo_Analyzer(Connection* conn) : Analyzer(AnalyzerTag::Teredo, conn),
+	                                    valid_orig(false), valid_resp(false)
 		{}
 
 	virtual ~Teredo_Analyzer()
@@ -26,18 +27,34 @@ public:
 
 	/**
 	 * Emits a weird only if the analyzer has previously been able to
-	 * decapsulate a Teredo packet since otherwise the weirds could happen
-	 * frequently enough to be less than helpful.
+	 * decapsulate a Teredo packet in both directions or if *force* param is
+	 * set, since otherwise the weirds could happen frequently enough to be less
+	 * than helpful.  The *force* param is meant for cases where just one side
+	 * has a valid encapsulation and so the weird would be informative.
 	 */
-	void Weird(const char* name) const
+	void Weird(const char* name, bool force = false) const
 		{
-		if ( ProtocolConfirmed() )
+		if ( ProtocolConfirmed() || force )
 			reporter->Weird(Conn(), name);
 		}
 
+	/**
+	 * If the delayed confirmation option is set, then a valid encapsulation
+	 * seen from both end points is required before confirming
+	 */
+	void Confirm()
+		{
+		if ( ! BifConst::Tunnel::delay_teredo_confirmation ||
+		     ( valid_orig && valid_resp ) )
+			ProtocolConfirmation();
+		}
+
 protected:
 	friend class AnalyzerTimer;
 	void ExpireTimer(double t);
+
+	bool valid_orig;
+	bool valid_resp;
 };
 
 class TeredoEncapsulation {
diff --git a/src/const.bif b/src/const.bif
index 499dc63314..7373403c11 100644
--- a/src/const.bif
+++ b/src/const.bif
@@ -16,6 +16,7 @@ const Tunnel::enable_ip: bool;
 const Tunnel::enable_ayiya: bool;
 const Tunnel::enable_teredo: bool;
 const Tunnel::yielding_teredo_decapsulation: bool;
+const Tunnel::delay_teredo_confirmation: bool;
 const Tunnel::ip_tunnel_timeout: interval;
 
 const Threading::heartbeat_interval: interval;
diff --git a/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log b/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
deleted file mode 100644
index 3300a3ef95..0000000000
--- a/testing/btest/Baseline/core.tunnels.false-teredo/dpd.log
+++ /dev/null
@@ -1,15 +0,0 @@
-#separator \x09
-#set_separator	,
-#empty_field	(empty)
-#unset_field	-
-#path	dpd
-#open	2009-11-18-17-59-51
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	analyzer	failure_reason
-#types	time	string	addr	port	addr	port	enum	string	string
-1258567191.486869	UWkUyAuUGXf	192.168.1.105	57696	192.168.1.1	53	udp	TEREDO	Teredo payload length [c\x1d\x81\x80\x00\x01\x00\x02\x00\x02\x00\x00\x04amch\x0equestionmarket\x03com\x00\x00\x01\x00...]
-1258578181.516140	nQcgTWjvg4c	192.168.1.104	64838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xfd\x81\x80\x00\x01\x00\x02\x00\x03\x00\x02\x08football\x02uk\x07reuters\x03com\x00\x00\x01\x00...]
-1258579063.784919	j4u32Pc5bif	192.168.1.104	55778	192.168.1.1	53	udp	TEREDO	Teredo payload length [j\x12\x81\x80\x00\x01\x00\x02\x00\x04\x00\x00\x08fastflip\x0agooglelabs\x03com\x00\x00\x01\x00...]
-1258581768.898165	TEfuqmmG4bh	192.168.1.104	50798	192.168.1.1	53	udp	TEREDO	Teredo payload length [o\xe3\x81\x80\x00\x01\x00\x02\x00\x04\x00\x04\x03www\x0fnashuatelegraph\x03com\x00\x00\x01\x00...]
-1258584478.989528	FrJExwHcSal	192.168.1.104	64963	192.168.1.1	53	udp	TEREDO	Teredo payload length [e\xbd\x81\x80\x00\x01\x00\x08\x00\x06\x00\x06\x08wellness\x05blogs\x04time\x03com\x00\x00\x01\x00...]
-1258600683.934672	5OKnoww6xl4	192.168.1.103	59838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xf0\x81\x80\x00\x01\x00\x01\x00\x02\x00\x00\x06update\x0csanasecurity\x03com\x00\x00\x01\x00...]
-#close	2009-11-19-03-18-03
diff --git a/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log b/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log
new file mode 100644
index 0000000000..705cd0e956
--- /dev/null
+++ b/testing/btest/Baseline/core.tunnels.teredo-known-services/known_services.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	known_services
+#open	2012-10-02-20-10-05
+#fields	ts	host	port_num	port_proto	service
+#types	time	addr	port	enum	table[string]
+1258567191.405770	192.168.1.1	53	udp	TEREDO
+#close	2012-10-02-20-10-05
diff --git a/testing/btest/Baseline/core.tunnels.teredo/conn.log b/testing/btest/Baseline/core.tunnels.teredo/conn.log
index 657e86b8b3..b71e56f073 100644
--- a/testing/btest/Baseline/core.tunnels.teredo/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo/conn.log
@@ -22,7 +22,7 @@
 1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	udp	teredo	8.928880	129	48	SF	-	0	Dd	2	185	1	76	(empty)
 1210953060.829233	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	udp	teredo	13.293994	2359	11243	SF	-	0	Dd	12	2695	13	11607	(empty)
 1210953058.933954	iE6yhOq3SF	0.0.0.0	68	255.255.255.255	67	udp	-	-	-	-	S0	-	0	D	1	328	0	0	(empty)
-1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	udp	teredo	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
+1210953052.324629	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	udp	-	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
 1210953046.591933	UWkUyAuUGXf	192.168.2.16	138	192.168.2.255	138	udp	-	28.448321	416	0	S0	-	0	D	2	472	0	0	(empty)
 1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
 1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
index 757eaf62ca..9d4bf86d57 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log
@@ -9,7 +9,7 @@
 1340127577.354166	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	tcp	http	0.052829	1675	10467	S1	-	0	ShADad	10	2279	12	11191	j4u32Pc5bif
 1340127577.336558	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	udp	teredo	0.010291	129	52	SF	-	0	Dd	2	185	1	80	(empty)
 1340127577.341510	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	udp	teredo	0.065485	2367	11243	SF	-	0	Dd	12	2703	13	11607	(empty)
-1340127577.339015	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	udp	teredo	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
+1340127577.339015	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	udp	-	-	-	-	SHR	-	0	d	0	0	1	137	(empty)
 1340127577.339015	nQcgTWjvg4c	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	k6kgXLOoSKl
 1340127577.343969	TEfuqmmG4bh	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.007778	4	4	OTH	-	0	-	1	52	1	52	UWkUyAuUGXf,j4u32Pc5bif
 1340127577.336558	arKYeMETxOg	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	UWkUyAuUGXf
diff --git a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
index 4ead29302f..764b78656a 100644
--- a/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
+++ b/testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
-#open	2012-06-19-17-39-37
+#open	2012-10-02-16-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
+1340127577.341510	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
 1340127577.346849	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Teredo_bubble_with_payload	-	F	bro
-1340127577.349292	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
-#close	2012-06-19-17-39-37
+#close	2012-10-02-16-53-03
diff --git a/testing/btest/core/tunnels/false-teredo.bro b/testing/btest/core/tunnels/false-teredo.bro
index 37088e9535..381478bd54 100644
--- a/testing/btest/core/tunnels/false-teredo.bro
+++ b/testing/btest/core/tunnels/false-teredo.bro
@@ -1,8 +1,23 @@
 # @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap %INPUT >output
 # @TEST-EXEC: test ! -e weird.log
+# @TEST-EXEC: test ! -e dpd.log
 # @TEST-EXEC: bro -r $TRACES/tunnels/false-teredo.pcap %INPUT Tunnel::yielding_teredo_decapsulation=F >output
 # @TEST-EXEC: btest-diff weird.log
-# @TEST-EXEC: btest-diff dpd.log
+# @TEST-EXEC: test ! -e dpd.log
+
+# In the first case, there isn't any weird or protocol violation logged
+# since the teredo analyzer recognizes that the DNS analyzer has confirmed
+# the protocol and yields.
+
+# In the second case, there are weirds since the teredo analyzer decapsulates
+# despite the presence of the confirmed DNS analyzer and the resulting
+# inner packets are malformed (no surprise there).  There's also no dpd.log
+# since the teredo analyzer doesn't confirm until it's seen a valid teredo
+# encapsulation in both directions and protocol violations aren't logged
+# until there's been a confirmation.
+
+# In either case, the analyzer doesn't, by default, get disabled as a result
+# of the protocol violations.
 
 function print_teredo(name: string, outer: connection, inner: teredo_hdr)
 	{
diff --git a/testing/btest/core/tunnels/teredo-known-services.test b/testing/btest/core/tunnels/teredo-known-services.test
new file mode 100644
index 0000000000..862930758f
--- /dev/null
+++ b/testing/btest/core/tunnels/teredo-known-services.test
@@ -0,0 +1,11 @@
+# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=T "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: test ! -e known_services.log
+# @TEST-EXEC: bro -b -r $TRACES/tunnels/false-teredo.pcap base/frameworks/dpd protocols/conn/known-services Tunnel::delay_teredo_confirmation=F "Site::local_nets+={192.168.1.0/24}"
+# @TEST-EXEC: btest-diff known_services.log
+
+# The first case using Tunnel::delay_teredo_confirmation=T doesn't produce
+# a known services.log since valid Teredo encapsulations from both endpoints
+# of a connection is never witnessed and a protocol_confirmation never issued.
+
+# The second case issues protocol_confirmations more hastily and so bogus
+# entries in known-services.log are more likely to appear.

From e93748d28b5d1915bda94dc951c42406e0eb2f9e Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 2 Oct 2012 15:36:12 -0500
Subject: [PATCH 219/238] Add general FAQ entry about upgrading Bro.

---
 doc/faq.rst | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/doc/faq.rst b/doc/faq.rst
index f265505def..1836e5a5e9 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -12,6 +12,14 @@ Frequently Asked Questions
 Installation and Configuration
 ==============================
 
+What files will get overwritten when upgrading/installing a new Bro version?
+----------------------------------------------------------------------------
+
+Expect everything except things in ``$prefix/share/bro`` and
+``$prefix/etc`` to be overwritten, but backing up the entire ``$prefix``
+before upgrading is good practice (``$prefix`` indicating the root of
+where Bro was installed).
+
 How can I tune my operating system for best capture performance?
 ----------------------------------------------------------------
 

From 06d6277f0aa97836b9c25f7aa97fdf8549fd7da9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 3 Oct 2012 16:14:52 -0500
Subject: [PATCH 220/238] Redo the "how to upgrade" FAQ.

---
 doc/faq.rst | 41 +++++++++++++++++++++++++++++++++++------
 1 file changed, 35 insertions(+), 6 deletions(-)

diff --git a/doc/faq.rst b/doc/faq.rst
index 1836e5a5e9..2342af1546 100644
--- a/doc/faq.rst
+++ b/doc/faq.rst
@@ -12,13 +12,42 @@ Frequently Asked Questions
 Installation and Configuration
 ==============================
 
-What files will get overwritten when upgrading/installing a new Bro version?
-----------------------------------------------------------------------------
+How do I upgrade to a new version of Bro?
+-----------------------------------------
 
-Expect everything except things in ``$prefix/share/bro`` and
-``$prefix/etc`` to be overwritten, but backing up the entire ``$prefix``
-before upgrading is good practice (``$prefix`` indicating the root of
-where Bro was installed).
+There's two suggested approaches, either install Bro using the same
+installation prefix directory as before, or pick a new prefix and copy
+local customizations over.
+
+Re-Use Previous Install Prefix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you choose to configure and install Bro with the same prefix
+directory as before, local customization and configuration to files in
+``$prefix/share/bro/site`` and ``$prefix/etc`` won't be overwritten
+(``$prefix`` indicating the root of where Bro was installed), but making
+a backup of local changes before proceeding is recommended.  Also, logs
+generated at run-time won't be touched by the upgrade.
+
+After upgrading, remember to check ``$prefix/share/bro/site`` and
+``$prefix/etc`` for ``.example`` files, which indicate the
+distribution's version of the file differs from the local one, which may
+include local changes.  Review the differences, and make adjustments
+as necessary (for differences that aren't the result of a local change,
+use the new version's).
+
+Pick a New Install prefix
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+If you want the install the newer version in a different prefix
+directory than before, you can just copy local customization and
+configuration files from ``$prefix/share/bro/site`` and ``$prefix/etc``
+to the new location (``$prefix`` indicating the root of where Bro was
+originally installed).  Make sure to review the files for difference
+before copying and make adjustments as necessary (for differences that
+aren't the result of a local change, use the new version's).  Of
+particular note, the copied version of ``$prefix/etc/broctl.cfg`` is
+likely to need changes to the ``SpoolDir`` and ``LogDir`` settings.
 
 How can I tune my operating system for best capture performance?
 ----------------------------------------------------------------

From 53d9832d5adb526bf80f1d225a13941c7d05bdb2 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 4 Oct 2012 16:32:34 -0500
Subject: [PATCH 221/238] Fix a problem with non-manager cluster nodes applying
 Notice::policy.

This could, for example, result in duplicate emails being sent (one from
manager and one from worker) if Notice::emailed_types is redef'd in
local.bro (or any script that gets loaded on all cluster nodes).

The problem was that Notice::policy is used to populate the internal
Notice::ordered_policy vector in a priority 10 bro_init handler (in
scripts/base/frameworks/notice/main.bro) and then that is what is used
when applying policy to notices.  In order for
scripts/base/frameworks/notice/cluster.bro to prevent Notice::policy
from being used on non-manager nodes, it needs to clear it in a
bro_init hander of higher priority than 10.
---
 scripts/base/frameworks/notice/cluster.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro
index 087c3ead51..3ee113acf3 100644
--- a/scripts/base/frameworks/notice/cluster.bro
+++ b/scripts/base/frameworks/notice/cluster.bro
@@ -23,7 +23,7 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # The notice policy is completely handled by the manager and shouldn't be 
 # done by workers or proxies to save time for packet processing.
-event bro_init() &priority=-11
+event bro_init() &priority=11
 	{
 	Notice::policy = table();
 	}

From 49b8c7e3909ba0b57019285eaa07022c44f45270 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 5 Oct 2012 10:43:23 -0500
Subject: [PATCH 222/238] Add analyzer for GSI mechanism of GSSAPI FTP AUTH
 method.

GSI authentication involves an encoded TLS/SSL handshake over the FTP
control session.  Decoding the exchanged tokens and passing them to an
SSL analyzer instance allows use of all the familiar script-layer events
in inspecting the handshake (e.g. client/server certificats are
available).  For FTP sessions that attempt GSI authentication, the
service field of the connection record will have both "ftp" and "ssl".

One additional change is an FTP server's acceptance of an AUTH request
no longer causes analysis of the connection to cease (because further
analysis likely wasn't possible).  This decision can be made more
dynamically at the script-layer (plus there's now the fact that further
analysis can be done at least on the GSSAPI AUTH method).
---
 doc/scripts/DocSourcesList.cmake              |   2 +
 scripts/base/protocols/ftp/main.bro           |   6 +-
 .../protocols/ftp/gridftp-data-detection.bro  |   4 +-
 scripts/test-all-policy.bro                   |   1 +
 src/Analyzer.cc                               |   1 +
 src/AnalyzerTags.h                            |   1 +
 src/FTP.cc                                    | 173 ++++++++++++++++--
 src/FTP.h                                     |  22 +++
 .../Baseline/core.print-bpf-filters/output    |   2 +-
 .../canonified_loaded_scripts.log             |   1 +
 10 files changed, 188 insertions(+), 25 deletions(-)

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index 1abe6b9305..077e103dca 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -65,6 +65,7 @@ rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
+rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/file-extract.bro)
@@ -122,6 +123,7 @@ rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
+rest_target(${psd} policy/protocols/ftp/gridftp-data-detection.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)
 rest_target(${psd} policy/protocols/http/detect-intel.bro)
diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro
index d20bc92d8a..0a4bfc07cc 100644
--- a/scripts/base/protocols/ftp/main.bro
+++ b/scripts/base/protocols/ftp/main.bro
@@ -96,11 +96,11 @@ redef record connection += {
 };
 
 # Configure DPD
-const ports = { 21/tcp } &redef;
-redef capture_filters += { ["ftp"] = "port 21" };
+const ports = { 21/tcp, 2811/tcp } &redef;
+redef capture_filters += { ["ftp"] = "port 21 and port 2811" };
 redef dpd_config += { [ANALYZER_FTP] = [$ports = ports] };
 
-redef likely_server_ports += { 21/tcp };
+redef likely_server_ports += { 21/tcp, 2811/tcp };
 
 # Establish the variable for tracking expected connections.
 global ftp_data_expected: table[addr, port] of Info &create_expire=5mins;
diff --git a/scripts/policy/protocols/ftp/gridftp-data-detection.bro b/scripts/policy/protocols/ftp/gridftp-data-detection.bro
index 15acfba65b..ffa2fa5816 100644
--- a/scripts/policy/protocols/ftp/gridftp-data-detection.bro
+++ b/scripts/policy/protocols/ftp/gridftp-data-detection.bro
@@ -10,12 +10,12 @@
 ##! benefit of saving CPU cycles that otherwise go to analyzing such
 ##! large (and hopefully benign) connections.
 
-module GridFTP;
-
 @load base/protocols/conn
 @load base/protocols/ssl
 @load base/frameworks/notice
 
+module GridFTP;
+
 export {
 	## Number of bytes transferred before guessing a connection is a 
 	## GridFTP data channel.
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index a7c43b14b3..f535d88cd5 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -34,6 +34,7 @@
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
 @load protocols/ftp/detect.bro
+@load protocols/ftp/gridftp-data-detection.bro
 @load protocols/ftp/software.bro
 @load protocols/http/detect-intel.bro
 @load protocols/http/detect-MHR.bro
diff --git a/src/Analyzer.cc b/src/Analyzer.cc
index 9e30da0066..8c5573f96b 100644
--- a/src/Analyzer.cc
+++ b/src/Analyzer.cc
@@ -171,6 +171,7 @@ const Analyzer::Config Analyzer::analyzer_configs[] = {
 	{ AnalyzerTag::Contents_SMB, "CONTENTS_SMB", 0, 0, 0, false },
 	{ AnalyzerTag::Contents_RPC, "CONTENTS_RPC", 0, 0, 0, false },
 	{ AnalyzerTag::Contents_NFS, "CONTENTS_NFS", 0, 0, 0, false },
+	{ AnalyzerTag::FTP_ADAT, "FTP_ADAT", 0, 0, 0, false },
 };
 
 AnalyzerTimer::~AnalyzerTimer()
diff --git a/src/AnalyzerTags.h b/src/AnalyzerTags.h
index 7fad4d35bb..4301de8f71 100644
--- a/src/AnalyzerTags.h
+++ b/src/AnalyzerTags.h
@@ -46,6 +46,7 @@ namespace AnalyzerTag {
 		Contents, ContentLine, NVT, Zip, Contents_DNS, Contents_NCP,
 		Contents_NetbiosSSN, Contents_Rlogin, Contents_Rsh,
 		Contents_DCE_RPC, Contents_SMB, Contents_RPC, Contents_NFS,
+		FTP_ADAT,
 		// End-marker.
 		LastAnalyzer
 	};
diff --git a/src/FTP.cc b/src/FTP.cc
index 588348ea8d..fba6b3eea6 100644
--- a/src/FTP.cc
+++ b/src/FTP.cc
@@ -8,6 +8,8 @@
 #include "FTP.h"
 #include "NVT.h"
 #include "Event.h"
+#include "SSL.h"
+#include "Base64.h"
 
 FTP_Analyzer::FTP_Analyzer(Connection* conn)
 : TCP_ApplicationAnalyzer(AnalyzerTag::FTP, conn)
@@ -44,6 +46,14 @@ void FTP_Analyzer::Done()
 		Weird("partial_ftp_request");
 	}
 
+static uint32 get_reply_code(int len, const char* line)
+	{
+	if ( len >= 3 && isdigit(line[0]) && isdigit(line[1]) && isdigit(line[2]) )
+		return (line[0] - '0') * 100 + (line[1] - '0') * 10 + (line[2] - '0');
+	else
+		return 0;
+	}
+
 void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 	{
 	TCP_ApplicationAnalyzer::DeliverStream(length, data, orig);
@@ -93,16 +103,7 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 		}
 	else
 		{
-		uint32 reply_code;
-		if ( length >= 3 &&
-		     isdigit(line[0]) && isdigit(line[1]) && isdigit(line[2]) )
-			{
-			reply_code = (line[0] - '0') * 100 +
-					(line[1] - '0') * 10 +
-					(line[2] - '0');
-			}
-		else
-			reply_code = 0;
+		uint32 reply_code = get_reply_code(length, line);
 
 		int cont_resp;
 
@@ -143,19 +144,22 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 				else
 					line = end_of_line;
 
-				if ( auth_requested.size() > 0 &&
-				     (reply_code == 234 || reply_code == 335) )
-					// Server accepted AUTH requested,
-					// which means that very likely we
-					// won't be able to parse the rest
-					// of the session, and thus we stop
-					// here.
-					SetSkip(true);
-
 				cont_resp = 0;
 				}
 			}
 
+		if ( reply_code == 334 && auth_requested.size() > 0 &&
+		     auth_requested == "GSSAPI" )
+			{
+			// Server wants to proceed with an ADAT exchange and we
+			// know how to analyze the GSI mechanism, so attach analyzer
+			// to look for that.
+			SSL_Analyzer* ssl = new SSL_Analyzer(Conn());
+			ssl->AddSupportAnalyzer(new FTP_ADAT_Analyzer(Conn(), true));
+			ssl->AddSupportAnalyzer(new FTP_ADAT_Analyzer(Conn(), false));
+			AddChildAnalyzer(ssl);
+			}
+
 		vl->append(new Val(reply_code, TYPE_COUNT));
 		vl->append(new StringVal(end_of_line - line, line));
 		vl->append(new Val(cont_resp, TYPE_BOOL));
@@ -164,5 +168,136 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig)
 		}
 
 	ConnectionEvent(f, vl);
+
+	ForwardStream(length, data, orig);
 	}
 
+void FTP_ADAT_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	// Don't know how to parse anything but the ADAT exchanges of GSI GSSAPI,
+	// which is basically just TLS/SSL.
+	if ( ! Parent()->GetTag() == AnalyzerTag::SSL )
+		{
+		Parent()->Remove();
+		return;
+		}
+
+	bool done = false;
+	const char* line = (const char*) data;
+	const char* end_of_line = line + len;
+
+	BroString* decoded_adat = 0;
+
+	if ( orig )
+		{
+		int cmd_len;
+		const char* cmd;
+		line = skip_whitespace(line, end_of_line);
+		get_word(len, line, cmd_len, cmd);
+
+		if ( strncmp(cmd, "ADAT", cmd_len) == 0 )
+			{
+			line = skip_whitespace(line + cmd_len, end_of_line);
+			StringVal* encoded = new StringVal(end_of_line - line, line);
+			decoded_adat = decode_base64(encoded->AsString());
+			delete encoded;
+
+			if ( first_token )
+				{
+				// RFC 2743 section 3.1 specifies a framing format for tokens
+				// that includes an identifier for the mechanism type.  The
+				// framing is supposed to be required for the initial context
+				// token, but GSI doesn't do that and starts right in on a
+				// TLS/SSL handshake, so look for that to identify it.
+				const u_char* msg = decoded_adat->Bytes();
+				int msg_len = decoded_adat->Len();
+
+				// Just check that it looks like a viable TLS/SSL handshake
+				// record from the first byte (content type of 0x16) and
+				// that the fourth and fifth bytes indicating the length of
+				// the record match the length of the decoded data.
+				if ( msg_len < 5 || msg[0] != 0x16 ||
+				     msg_len - 5 != ntohs(*((uint16*)(msg + 3))) )
+					{
+					// Doesn't look like TLS/SSL, so done analyzing.
+					done = true;
+					delete decoded_adat;
+					decoded_adat = 0;
+					}
+				}
+
+			first_token = false;
+			}
+		else if ( strncmp(cmd, "AUTH", cmd_len) == 0 )
+			{
+			// Security state will be reset by a reissued AUTH
+			done = true;
+			}
+		}
+	else
+		{
+		uint32 reply_code = get_reply_code(len, line);
+
+		switch ( reply_code ) {
+		case 232:
+		case 234:
+			// Indicates security data exchange is complete, but nothing
+			// more to decode in replies.
+			done = true;
+			break;
+
+		case 235:
+			// Security data exchange complete, but may have more to decode
+			// in the reply (same format at 334 and 335).
+			done = true;
+		case 334:
+		case 335:
+			// Security data exchange still in progress, and there could be data
+			// to decode in the reply.
+			line += 3;
+			if ( len > 3 && line[0] == '-' ) line++;
+			line = skip_whitespace(line, end_of_line);
+
+			if ( end_of_line - line >= 5 && strncmp(line, "ADAT=", 5) == 0 )
+				{
+				line += 5;
+				StringVal* encoded = new StringVal(end_of_line - line, line);
+				decoded_adat = decode_base64(encoded->AsString());
+				delete encoded;
+				}
+			break;
+
+		case 421:
+		case 431:
+		case 500:
+		case 501:
+		case 503:
+		case 535:
+			// Server isn't going to accept named security mechanism.
+			// Client has to restart back at the AUTH.
+			done = true;
+			break;
+
+		case 631:
+		case 632:
+		case 633:
+			// If the server is sending protected replies, the security
+			// data exchange must have already succeeded.  It does have
+			// encoded data in the reply, but 632 and 633 are also encrypted.
+			done = true;
+			break;
+
+		default:
+			break;
+		}
+		}
+
+	if ( decoded_adat )
+		{
+		ForwardStream(decoded_adat->Len(), decoded_adat->Bytes(), orig);
+		delete decoded_adat;
+		}
+
+	if ( done )
+		Parent()->Remove();
+	}
diff --git a/src/FTP.h b/src/FTP.h
index 4ef6c44d83..f8d7644808 100644
--- a/src/FTP.h
+++ b/src/FTP.h
@@ -30,4 +30,26 @@ protected:
 	string auth_requested;	// AUTH method requested
 };
 
+/**
+ * Analyzes security data of ADAT exchanges over FTP control session (RFC 2228).
+ * Currently only the GSI mechanism of GSSAPI AUTH method is understood.
+ * The ADAT exchange for GSI is base64 encoded TLS/SSL handshake tokens.  This
+ * analyzer just decodes the tokens and passes them on to the parent, which must
+ * be an SSL analyzer instance.
+ */
+class FTP_ADAT_Analyzer : public SupportAnalyzer {
+public:
+	FTP_ADAT_Analyzer(Connection* conn, bool arg_orig)
+	    : SupportAnalyzer(AnalyzerTag::FTP_ADAT, conn, arg_orig),
+	      first_token(true) { }
+
+	void DeliverStream(int len, const u_char* data, bool orig);
+
+protected:
+	// Used by the client-side analyzer to tell if it needs to peek at the
+	// initial context token and do sanity checking (i.e. does it look like
+	// a TLS/SSL handshake token).
+	bool first_token;
+};
+
 #endif
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output
index c55952ffed..55473b8991 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@@ -16,7 +16,7 @@
 #open	2012-07-27-19-14-29
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416469.888870	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+1343416469.888870	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
 #close	2012-07-27-19-14-29
 #separator \x09
 #set_separator	,
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index b2afadc0fe..755260351b 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -77,6 +77,7 @@ scripts/base/init-default.bro
     scripts/base/protocols/conn/./main.bro
     scripts/base/protocols/conn/./contents.bro
     scripts/base/protocols/conn/./inactivity.bro
+    scripts/base/protocols/conn/./polling.bro
   scripts/base/protocols/dns/__load__.bro
     scripts/base/protocols/dns/./consts.bro
     scripts/base/protocols/dns/./main.bro

From db62369508033a68e8e636ac19a3466775169a88 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 5 Oct 2012 13:48:49 -0400
Subject: [PATCH 223/238] Fix for DNS log problem when a DNS response is seen
 with 0 RRs.

---
 scripts/base/protocols/dns/main.bro           |  22 ++++++++++--------
 testing/btest/Baseline/core.ipv6-frag/dns.log |  12 +++++-----
 .../dns.log                                   |  10 ++++++++
 .../dns.log                                   |  10 ++++----
 testing/btest/Traces/dns-zero-RRs.trace       | Bin 0 -> 242 bytes
 .../base/protocols/dns/zero-responses.bro     |   4 ++++
 6 files changed, 37 insertions(+), 21 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.dns.zero-responses/dns.log
 create mode 100644 testing/btest/Traces/dns-zero-RRs.trace
 create mode 100644 testing/btest/scripts/base/protocols/dns/zero-responses.bro

diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro
index c951ff4fd2..8ae3806ab6 100644
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@@ -59,13 +59,15 @@ export {
 		## The caching intervals of the associated RRs described by the
 		## ``answers`` field.
 		TTLs:          vector of interval &log &optional;
+		## The DNS query was rejected by the server.
+		rejected:      bool               &log &default=F;
 
 		## This value indicates if this request/response pair is ready to be
 		## logged.
 		ready:         bool            &default=F;
 		## The total number of resource records in a reply message's answer
 		## section.
-		total_answers: count           &optional;
+		total_answers: count           &default=0;
 		## The total number of resource records in a reply message's answer,
 		## authority, and additional sections.
 		total_replies: count           &optional;
@@ -186,10 +188,13 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
 		}
 	}
 
+event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count) &priority=5
+	{
+	set_session(c, msg, is_orig);
+	}
+
 event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
 	{
-	set_session(c, msg, F);
-
 	if ( ans$answer_type == DNS_ANS )
 		{
 		c$dns$AA    = msg$AA;
@@ -209,7 +214,8 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 			c$dns$TTLs[|c$dns$TTLs|] = ans$TTL;
 			}
 
-		if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers )
+		if ( c$dns?$answers && c$dns?$total_answers &&
+		     |c$dns$answers| == c$dns$total_answers )
 			{
 			add c$dns_state$finished_answers[c$dns$trans_id];
 			# Indicate this request/reply pair is ready to be logged.
@@ -230,8 +236,6 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 
 event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
 	{
-	set_session(c, msg, T);
-
 	c$dns$RD          = msg$RD;
 	c$dns$TC          = msg$TC;
 	c$dns$qclass      = qclass;
@@ -321,11 +325,9 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
 #
 #	}
 
-
-event dns_rejected(c: connection, msg: dns_msg,
-                   query: string, qtype: count, qclass: count) &priority=5
+event dns_rejected(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
 	{
-	set_session(c, msg, F);
+	c$dns$rejected = T;
 	}
 
 event connection_state_remove(c: connection) &priority=-5
diff --git a/testing/btest/Baseline/core.ipv6-frag/dns.log b/testing/btest/Baseline/core.ipv6-frag/dns.log
index d763fc4fee..de027644e8 100644
--- a/testing/btest/Baseline/core.ipv6-frag/dns.log
+++ b/testing/btest/Baseline/core.ipv6-frag/dns.log
@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
-#open	2012-03-07-01-37-58
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs
-#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]
-1331084278.438444	UWkUyAuUGXf	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51850	2607:f740:b::f93	53	udp	3903	txtpadding_323.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
-1331084293.592245	arKYeMETxOg	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51851	2607:f740:b::f93	53	udp	40849	txtpadding_3230.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
-#close	2012-03-07-01-38-18
+#open	2012-10-05-17-47-27
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	rejected
+#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	bool
+1331084278.438444	UWkUyAuUGXf	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51850	2607:f740:b::f93	53	udp	3903	txtpadding_323.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000	F
+1331084293.592245	arKYeMETxOg	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51851	2607:f740:b::f93	53	udp	40849	txtpadding_3230.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000	F
+#close	2012-10-05-17-47-27
diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.zero-responses/dns.log b/testing/btest/Baseline/scripts.base.protocols.dns.zero-responses/dns.log
new file mode 100644
index 0000000000..14ad7b77bc
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.dns.zero-responses/dns.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dns
+#open	2012-10-05-15-59-39
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	rejected
+#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	bool
+1349445121.080922	UWkUyAuUGXf	10.0.0.64	49204	146.186.163.66	53	udp	17323	psu.edu	1	C_INTERNET	28	AAAA	0	NOERROR	F	F	T	F	0	-	-	F
+#close	2012-10-05-15-59-39
diff --git a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
index f4b77edde7..74de757007 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
+++ b/testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
-#open	1999-06-28-23-40-27
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	auth	addl
-#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	table[string]	table[string]
-930613226.529070	UWkUyAuUGXf	212.180.42.100	25000	131.243.64.3	53	tcp	34798	-	-	-	-	-	0	NOERROR	F	F	F	T	0	4.3.2.1	31337.000000	-	-
-#close	1999-06-28-23-40-27
+#open	2012-10-05-17-47-40
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	rejected	auth	addl
+#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	bool	table[string]	table[string]
+930613226.518174	UWkUyAuUGXf	212.180.42.100	25000	131.243.64.3	53	tcp	34798	-	-	-	-	-	0	NOERROR	F	F	F	T	0	4.3.2.1	31337.000000	F	-	-
+#close	2012-10-05-17-47-40
diff --git a/testing/btest/Traces/dns-zero-RRs.trace b/testing/btest/Traces/dns-zero-RRs.trace
new file mode 100644
index 0000000000000000000000000000000000000000..0f4785b3f0ba8be9e547bc7f289a11eb651e20b5
GIT binary patch
literal 242
zcmca|c+)~A1{MYw`2U}Qff2}Ie3lm=Wy8qe3}l1w4TB_)qgx$v`mJV#b1=9vFqpbs
zWngd+{Je|{NKM+c*y(@?gDHdJzM0Oef%-v400DDBaVc|ZN+|<_3<D#`$O*Dc4CO!x
z5QZ2AF&J!Q=3-k022H_rTp;5>#xj@!4Gmqo!+CWp)KzFk9^heM1KG9W0D}fgeo3;?
l0UoZ5{Nj?_#Nv|Fq60kT@64`&Wf^1?7^K%XFfax-003r}J}dwL

literal 0
HcmV?d00001

diff --git a/testing/btest/scripts/base/protocols/dns/zero-responses.bro b/testing/btest/scripts/base/protocols/dns/zero-responses.bro
new file mode 100644
index 0000000000..54f7d7b7d3
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/dns/zero-responses.bro
@@ -0,0 +1,4 @@
+# This tests the case where the DNS server responded with zero RRs.
+#
+# @TEST-EXEC: bro -r $TRACES/dns-zero-RRs.trace
+# @TEST-EXEC: btest-diff dns.log
\ No newline at end of file

From e34f6d9e3b1475828e11b590211311581dd05955 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 8 Oct 2012 11:38:29 -0500
Subject: [PATCH 224/238] Enable GridFTP detection by default.  Track/log SSL
 client certs.

In the *service* field of connection records, GridFTP control channels
are labeled as "gridftp" and data channels as "gridftp-data".

Added *client_subject* and *client_issuer_subject* as &log'd fields to
SSL::Info record.  Also added *client_cert* and *client_cert_chain*
fields to track client cert chain.
---
 doc/scripts/DocSourcesList.cmake              |   2 +-
 scripts/base/protocols/ftp/__load__.bro       |   3 +-
 scripts/base/protocols/ftp/gridftp.bro        | 106 ++++++++++++++++++
 scripts/base/protocols/ssl/main.bro           |  61 +++++++---
 .../protocols/ftp/gridftp-data-detection.bro  |  83 --------------
 scripts/test-all-policy.bro                   |   1 -
 .../Baseline/core.print-bpf-filters/output    |  24 ++--
 .../canonified_loaded_scripts.log             |   9 +-
 .../conn.log                                  |  11 ++
 .../notice.log                                |   4 +-
 .../ssl.log                                   |  11 ++
 .../scripts.base.protocols.ssl.basic/ssl.log  |  10 +-
 .../scripts/base/protocols/ftp/gridftp.test   |  21 ++++
 .../protocols/ftp/gridftp-data-dection.test   |   6 -
 testing/scripts/diff-remove-x509-names        |  18 ++-
 15 files changed, 238 insertions(+), 132 deletions(-)
 create mode 100644 scripts/base/protocols/ftp/gridftp.bro
 delete mode 100644 scripts/policy/protocols/ftp/gridftp-data-detection.bro
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/conn.log
 rename testing/btest/Baseline/{scripts.policy.protocols.ftp.gridftp-data-dection => scripts.base.protocols.ftp.gridftp}/notice.log (93%)
 create mode 100644 testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log
 create mode 100644 testing/btest/scripts/base/protocols/ftp/gridftp.test
 delete mode 100644 testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test

diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake
index 077e103dca..b127e1526d 100644
--- a/doc/scripts/DocSourcesList.cmake
+++ b/doc/scripts/DocSourcesList.cmake
@@ -69,6 +69,7 @@ rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/gridftp.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
 rest_target(${psd} base/protocols/http/file-extract.bro)
@@ -123,7 +124,6 @@ rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/gridftp-data-detection.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)
 rest_target(${psd} policy/protocols/http/detect-intel.bro)
diff --git a/scripts/base/protocols/ftp/__load__.bro b/scripts/base/protocols/ftp/__load__.bro
index 0a399aef36..15c61be614 100644
--- a/scripts/base/protocols/ftp/__load__.bro
+++ b/scripts/base/protocols/ftp/__load__.bro
@@ -1,3 +1,4 @@
 @load ./utils-commands
 @load ./main
-@load ./file-extract
\ No newline at end of file
+@load ./file-extract
+@load ./gridftp
diff --git a/scripts/base/protocols/ftp/gridftp.bro b/scripts/base/protocols/ftp/gridftp.bro
new file mode 100644
index 0000000000..e94836cceb
--- /dev/null
+++ b/scripts/base/protocols/ftp/gridftp.bro
@@ -0,0 +1,106 @@
+##! A detection script for GridFTP data and control channels.
+##!
+##! GridFTP control channels are identified by FTP control channels
+##! that successfully negotiate the GSSAPI method of an AUTH request
+##! and for which the exchange involved an encoded TLS/SSL handshake,
+##! indicating the GSI mechanism for GSSAPI was used.  This analysis
+##! is all supported internally, this script simple adds the "gridftp"
+##! label to the *service* field of the control channel's
+##! :bro:type:`connection` record.
+##!
+##! GridFTP data channels are identified by a heuristic that relies on
+##! the fact that default settings for GridFTP clients typically
+##! mutally authenticate the data channel with TLS/SSL and negotiate a
+##! NULL bulk cipher (no encryption).  Connections with those
+##! attributes are then polled for two minutes with decreasing frequency
+##! to check if the transfer sizes are large enough to indicate a
+##! GridFTP ata channel that would be undesireable to analyze further
+##! (e.g. stop TCP reassembly).  A side effect is that true connection
+##! sizes are not logged, but at the benefit of saving CPU cycles that
+##! otherwise go to analyzing the large (and likely benign) connections.
+
+@load ./main
+@load base/protocols/conn
+@load base/protocols/ssl
+@load base/frameworks/notice
+
+module GridFTP;
+
+export {
+	## Number of bytes transferred before guessing a connection is a
+	## GridFTP data channel.
+	const size_threshold = 1073741824 &redef;
+
+	## Max number of times to check whether a connection's size exceeds the
+	## :bro:see:`GridFTP::size_threshold`.
+	const max_poll_count = 15 &redef;
+
+	## Whether to skip further processing of the GridFTP data channel once
+	## detected, which may help performance.
+	const skip_data = T &redef;
+
+	## Base amount of time between checking whether a GridFTP data connection
+	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
+	const poll_interval = 1sec &redef;
+
+	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
+	## increased by each poll interval.  Can be used to make more frequent
+	## checks at the start of a connection and gradually slow down.
+	const poll_interval_increase = 1sec &redef;
+
+	## Raised when a GridFTP data channel is detected.
+	##
+	## c: The connection pertaining to the GridFTP data channel.
+	global data_channel_detected: event(c: connection);
+
+	## The initial criteria used to determine whether to start polling
+	## the connection for the :bro:see:`GridFTP::size_threshold` to have
+	## been exceeded.  This is called in a :bro:see:`ssl_established` event
+	## handler and by default looks for both a client and server certificate
+	## and for a NULL bulk cipher.  One way in which this function could be
+	## redefined is to make it also consider client/server certificate issuer
+	## subjects.
+	##
+	## c: The connection which may possibly be a GridFTP data channel.
+	##
+	## Returns: true if the connection should be further polled for an
+	##          exceeded :bro:see:`GridFTP::size_threshold`, else false.
+	const data_channel_initial_criteria: function(c: connection): bool &redef;
+}
+
+function size_callback(c: connection, cnt: count): interval
+	{
+	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+		{
+		add c$service["gridftp-data"];
+		event GridFTP::data_channel_detected(c);
+		if ( skip_data )
+			skip_further_processing(c$id);
+		return -1sec;
+		}
+
+	if ( cnt >= max_poll_count ) return -1sec;
+
+	return poll_interval + poll_interval_increase * cnt;
+	}
+
+event ssl_established(c: connection) &priority=5
+	{
+	# Add service label to control channels.
+	if ( "FTP" in c$service )
+		add c$service["gridftp"];
+	}
+
+function data_channel_initial_criteria(c: connection): bool
+	{
+	return ( c?$ssl && c$ssl?$client_subject && c$ssl?$subject &&
+	         c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher );
+	}
+
+event ssl_established(c: connection) &priority=-3
+	{
+	# By default GridFTP data channels do mutual authentication and
+	# negotiate a cipher suite with a NULL bulk cipher.
+	if ( data_channel_initial_criteria(c) )
+		ConnPolling::watch(c, size_callback, 0, 0secs);
+	}
diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro
index f61e0d68ab..788336e0a6 100644
--- a/scripts/base/protocols/ssl/main.bro
+++ b/scripts/base/protocols/ssl/main.bro
@@ -30,17 +30,28 @@ export {
 		issuer_subject:   string           &log &optional;
 		## NotValidBefore field value from the server certificate.
 		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the serve certificate.
+		## NotValidAfter field value from the server certificate.
 		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
-		
+
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the client.
+		client_issuer_subject:   string           &log &optional;
+
 		## Full binary server certificate stored in DER format.
 		cert:             string           &optional;
-		## Chain of certificates offered by the server to validate its 
+		## Chain of certificates offered by the server to validate its
 		## complete signing chain.
 		cert_chain:       vector of string &optional;
 
+		## Full binary client certificate stored in DER format.
+		client_cert:             string           &optional;
+		## Chain of certificates offered by the client to validate its 
+		## complete signing chain.
+		client_cert_chain:       vector of string &optional;
+
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -107,7 +118,8 @@ redef likely_server_ports += {
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
+		         $client_cert_chain=vector()];
 	}
 
 function finish(c: connection)
@@ -141,23 +153,40 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 
 	# We aren't doing anything with client certificates yet.
 	if ( is_orig )
-		return;
-
-	if ( chain_idx == 0 )
 		{
-		# Save the primary cert.
-		c$ssl$cert = der_cert;
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$client_cert = der_cert;
 
-		# Also save other certificate information about the primary cert.
-		c$ssl$subject = cert$subject;
-		c$ssl$issuer_subject = cert$issuer;
-		c$ssl$not_valid_before = cert$not_valid_before;
-		c$ssl$not_valid_after = cert$not_valid_after;
+			# Also save other certificate information about the primary cert.
+			c$ssl$client_subject = cert$subject;
+			c$ssl$client_issuer_subject = cert$issuer;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
+			}
 		}
 	else
 		{
-		# Otherwise, add it to the cert validation chain.
-		c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$cert = der_cert;
+
+			# Also save other certificate information about the primary cert.
+			c$ssl$subject = cert$subject;
+			c$ssl$issuer_subject = cert$issuer;
+			c$ssl$not_valid_before = cert$not_valid_before;
+			c$ssl$not_valid_after = cert$not_valid_after;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
+			}
 		}
 	}
 
diff --git a/scripts/policy/protocols/ftp/gridftp-data-detection.bro b/scripts/policy/protocols/ftp/gridftp-data-detection.bro
deleted file mode 100644
index ffa2fa5816..0000000000
--- a/scripts/policy/protocols/ftp/gridftp-data-detection.bro
+++ /dev/null
@@ -1,83 +0,0 @@
-##! A detection script for GridFTP data channels.  The heuristic used to
-##! identify a GridFTP data channel relies on the fact that default
-##! setting for GridFTP clients typically mutually authenticate the data
-##! channel with SSL and negotiate a NULL bulk cipher (no encryption).
-##! Connections with those attributes are then polled for two minutes
-##! with decreasing frequency to check if the transfer sizes are large
-##! enough to indicate a GridFTP data channel that would be undesireable
-##! to analyze further (e.g. TCP reassembly no longer occurs).  A side
-##! effect is that true connection sizes are not logged, but at the
-##! benefit of saving CPU cycles that otherwise go to analyzing such
-##! large (and hopefully benign) connections.
-
-@load base/protocols/conn
-@load base/protocols/ssl
-@load base/frameworks/notice
-
-module GridFTP;
-
-export {
-	## Number of bytes transferred before guessing a connection is a 
-	## GridFTP data channel.
-	const size_threshold = 1073741824 &redef;
-
-	## Max number of times to check whether a connection's size exceeds the
-	## :bro:see:`GridFTP::size_threshold`.
-	const max_poll_count = 15 &redef;
-
-	## Whether to skip further processing of the GridFTP data channel once
-	## detected, which may help performance.
-	const skip_data = T &redef;
-
-	## Base amount of time between checking whether a GridFTP connection
-	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
-	const poll_interval = 1sec &redef;
-
-	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
-	## increased by each poll interval.  Can be used to make more frequent
-	## checks at the start of a connection and gradually slow down.
-	const poll_interval_increase = 1sec &redef;
-}
-
-redef enum Notice::Type += {
-	Data_Channel
-};
-
-redef record SSL::Info += {
-	## Indicates a client certificate was sent in the SSL handshake.
-	saw_client_cert: bool &optional;
-};
-
-event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
-	{
-	if ( is_orig && c?$ssl )
-		c$ssl$saw_client_cert = T;
-	}
-
-function size_callback(c: connection, cnt: count): interval
-	{
-	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
-		{
-		local msg = fmt("GridFTP data channel over threshold %d bytes",
-		                size_threshold);
-		NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
-		if ( skip_data )
-			skip_further_processing(c$id);
-		return -1sec;
-		}
-
-	if ( cnt >= max_poll_count ) return -1sec;
-
-	return poll_interval + poll_interval_increase * cnt;
-	}
-
-event ssl_established(c: connection)
-	{
-	# By default GridFTP data channels do mutual authentication and
-	# negotiate a cipher suite with a NULL bulk cipher.
-	if ( c?$ssl && c$ssl?$saw_client_cert && c$ssl?$subject &&
-	     c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher )
-		{
-		ConnPolling::watch(c, size_callback, 0, 0secs);
-		}
-	}
diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro
index f535d88cd5..a7c43b14b3 100644
--- a/scripts/test-all-policy.bro
+++ b/scripts/test-all-policy.bro
@@ -34,7 +34,6 @@
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
 @load protocols/ftp/detect.bro
-@load protocols/ftp/gridftp-data-detection.bro
 @load protocols/ftp/software.bro
 @load protocols/http/detect-intel.bro
 @load protocols/http/detect-MHR.bro
diff --git a/testing/btest/Baseline/core.print-bpf-filters/output b/testing/btest/Baseline/core.print-bpf-filters/output
index 55473b8991..cd6e77dfcc 100644
--- a/testing/btest/Baseline/core.print-bpf-filters/output
+++ b/testing/btest/Baseline/core.print-bpf-filters/output
@@ -3,38 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-29
+#open	2012-10-08-16-16-08
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416469.508262	-	ip or not ip	T	T
-#close	2012-07-27-19-14-29
+1349712968.812610	-	ip or not ip	T	T
+#close	2012-10-08-16-16-08
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-29
+#open	2012-10-08-16-16-09
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416469.888870	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
-#close	2012-07-27-19-14-29
+1349712969.042094	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 995)) or (tcp port 22)) or (port 21 and port 2811)) or (tcp port 25 or tcp port 587)) or (tcp port 614)) or (tcp port 990)) or (port 6667)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+#close	2012-10-08-16-16-09
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-30
+#open	2012-10-08-16-16-09
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416470.252918	-	port 42	T	T
-#close	2012-07-27-19-14-30
+1349712969.270826	-	port 42	T	T
+#close	2012-10-08-16-16-09
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
-#open	2012-07-27-19-14-30
+#open	2012-10-08-16-16-09
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1343416470.614962	-	port 56730	T	T
-#close	2012-07-27-19-14-30
+1349712969.499878	-	port 56730	T	T
+#close	2012-10-08-16-16-09
diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
index 755260351b..c3ee64cffe 100644
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@@ -85,6 +85,11 @@ scripts/base/init-default.bro
     scripts/base/protocols/ftp/./utils-commands.bro
     scripts/base/protocols/ftp/./main.bro
     scripts/base/protocols/ftp/./file-extract.bro
+    scripts/base/protocols/ftp/./gridftp.bro
+      scripts/base/protocols/ssl/__load__.bro
+        scripts/base/protocols/ssl/./consts.bro
+        scripts/base/protocols/ssl/./main.bro
+        scripts/base/protocols/ssl/./mozilla-ca-list.bro
   scripts/base/protocols/http/__load__.bro
     scripts/base/protocols/http/./main.bro
     scripts/base/protocols/http/./utils.bro
@@ -103,10 +108,6 @@ scripts/base/init-default.bro
     scripts/base/protocols/socks/./main.bro
   scripts/base/protocols/ssh/__load__.bro
     scripts/base/protocols/ssh/./main.bro
-  scripts/base/protocols/ssl/__load__.bro
-    scripts/base/protocols/ssl/./consts.bro
-    scripts/base/protocols/ssl/./main.bro
-    scripts/base/protocols/ssl/./mozilla-ca-list.bro
   scripts/base/protocols/syslog/__load__.bro
     scripts/base/protocols/syslog/./consts.bro
     scripts/base/protocols/syslog/./main.bro
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/conn.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/conn.log
new file mode 100644
index 0000000000..f3ac10b5b0
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/conn.log
@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2012-10-05-21-45-15
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1348168976.274919	UWkUyAuUGXf	192.168.57.103	60108	192.168.57.101	2811	tcp	ssl,ftp,gridftp	0.294743	4491	6659	SF	-	0	ShAdDaFf	22	5643	21	7759	(empty)
+1348168976.546371	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	ssl,gridftp-data	0.011938	2135	3196	S1	-	0	ShADad	8	2559	6	3516	(empty)
+#close	2012-10-05-21-45-15
diff --git a/testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
similarity index 93%
rename from testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log
rename to testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
index dc007e4e24..f9292344a8 100644
--- a/testing/btest/Baseline/scripts.policy.protocols.ftp.gridftp-data-dection/notice.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	notice
-#open	2012-10-01-17-11-05
+#open	2012-10-05-21-45-15
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	enum	string	string	addr	addr	port	count	string	table[enum]	table[count]	interval	bool	string	string	string	double	double	addr	string	subnet
 1348168976.558309	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	tcp	GridFTP::Data_Channel	GridFTP data channel over threshold 2 bytes	-	192.168.57.103	192.168.57.101	55968	-	bro	Notice::ACTION_LOG	6	3600.000000	F	-	-	-	-	-	-	-	-
-#close	2012-10-01-17-11-05
+#close	2012-10-05-21-45-15
diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log
new file mode 100644
index 0000000000..512676bbb6
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log
@@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#open	2012-10-05-21-45-15
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+1348168976.508038	UWkUyAuUGXf	192.168.57.103	60108	192.168.57.101	2811	TLSv10	TLS_RSA_WITH_AES_256_CBC_SHA	-	-	CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348161979.000000	1379697979.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
+1348168976.551422	arKYeMETxOg	192.168.57.103	35391	192.168.57.101	55968	TLSv10	TLS_RSA_WITH_NULL_SHA	-	-	CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	1348168676.000000	1348206441.000000	-	CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid	CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid
+#close	2012-10-05-21-45-15
diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
index 5bf3feddc5..872da052ea 100644
--- a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
+++ b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log
@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssl
-#open	2012-04-27-14-53-12
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert
-#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string
-1335538392.319381	UWkUyAuUGXf	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-
-#close	2012-04-27-14-53-16
+#open	2012-10-08-16-18-56
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	server_name	session_id	subject	issuer_subject	not_valid_before	not_valid_after	last_alert	client_subject	client_issuer_subject
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	time	time	string	string	string
+1335538392.319381	UWkUyAuUGXf	192.168.1.105	62045	74.125.224.79	443	TLSv10	TLS_ECDHE_RSA_WITH_RC4_128_SHA	ssl.gstatic.com	-	CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US	CN=Google Internet Authority,O=Google Inc,C=US	1334102677.000000	1365639277.000000	-	-	-
+#close	2012-10-08-16-18-56
diff --git a/testing/btest/scripts/base/protocols/ftp/gridftp.test b/testing/btest/scripts/base/protocols/ftp/gridftp.test
new file mode 100644
index 0000000000..494729cf5f
--- /dev/null
+++ b/testing/btest/scripts/base/protocols/ftp/gridftp.test
@@ -0,0 +1,21 @@
+# @TEST-EXEC: bro -r $TRACES/globus-url-copy.trace %INPUT
+# @TEST-EXEC: btest-diff notice.log
+# @TEST-EXEC: btest-diff conn.log
+# @TEST-EXEC: btest-diff ssl.log
+
+@load base/protocols/ftp/gridftp
+
+module GridFTP;
+
+redef size_threshold = 2;
+
+redef enum Notice::Type += {
+    Data_Channel
+};
+
+event GridFTP::data_channel_detected(c: connection)
+	{
+	local msg = fmt("GridFTP data channel over threshold %d bytes",
+	                size_threshold);
+	NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
+	}
diff --git a/testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test b/testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test
deleted file mode 100644
index bb7b9b510d..0000000000
--- a/testing/btest/scripts/policy/protocols/ftp/gridftp-data-dection.test
+++ /dev/null
@@ -1,6 +0,0 @@
-# @TEST-EXEC: bro -r $TRACES/globus-url-copy.trace %INPUT
-# @TEST-EXEC: btest-diff notice.log
-
-@load protocols/ftp/gridftp-data-detection
-
-redef GridFTP::size_threshold = 2;
diff --git a/testing/scripts/diff-remove-x509-names b/testing/scripts/diff-remove-x509-names
index 6209edfc65..4863efc990 100755
--- a/testing/scripts/diff-remove-x509-names
+++ b/testing/scripts/diff-remove-x509-names
@@ -3,7 +3,7 @@
 # A diff canonifier that removes all X.509 Distinguished Name subject fields
 # because that output can differ depending on installed OpenSSL version.
 
-BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1 }
+BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1; cs_col = -1; ci_col = -1 }
 
 /^#fields/ {
     for ( i = 2; i < NF; ++i )
@@ -12,6 +12,10 @@ BEGIN { FS="\t"; OFS="\t"; s_col = -1; i_col = -1 }
             s_col = i-1;
         if ( $i == "issuer_subject" )
             i_col = i-1;
+        if ( $i == "client_subject" )
+            cs_col = i-1;
+        if ( $i == "client_issuer_subject" )
+            ci_col = i-1;
         }
 }
 
@@ -27,6 +31,18 @@ i_col >= 0 {
         $i_col = "+";
 }
 
+cs_col >= 0 {
+    if ( $cs_col != "-" )
+        # Mark that it's set, but ignore content.
+        $cs_col = "+";
+}
+
+ci_col >= 0 {
+    if ( $ci_col != "-" )
+        # Mark that it's set, but ignore content.
+        $ci_col = "+";
+}
+
 {
     print;
 }

From dedfdf7e11e5f653a5d54c582353214f8ac65f4c Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 8 Oct 2012 13:15:47 -0500
Subject: [PATCH 225/238] Add memory leak unit test for GridFTP.

---
 testing/btest/core/leaks/gridftp.test | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)
 create mode 100644 testing/btest/core/leaks/gridftp.test

diff --git a/testing/btest/core/leaks/gridftp.test b/testing/btest/core/leaks/gridftp.test
new file mode 100644
index 0000000000..6364000b0d
--- /dev/null
+++ b/testing/btest/core/leaks/gridftp.test
@@ -0,0 +1,24 @@
+# Needs perftools support.
+#
+# @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
+#
+# @TEST-GROUP: leaks
+#
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/globus-url-copy.trace %INPUT
+
+@load base/protocols/ftp/gridftp
+
+module GridFTP;
+
+redef size_threshold = 2;
+
+redef enum Notice::Type += {
+    Data_Channel
+};
+
+event GridFTP::data_channel_detected(c: connection)
+	{
+	local msg = fmt("GridFTP data channel over threshold %d bytes",
+	                size_threshold);
+	NOTICE([$note=Data_Channel, $msg=msg, $conn=c]);
+	}

From 6caeb7d7cfe51b672ed2784877cdc050c41b6d0b Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 8 Oct 2012 16:27:15 -0500
Subject: [PATCH 226/238] Add --with-curl option to ./configure, addresses
 #877.

---
 CMakeLists.txt                               | 8 ++++----
 configure                                    | 4 ++++
 testing/btest/coverage/bare-mode-errors.test | 4 ++--
 3 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2c8a726a1a..e8f9dcd6cd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -132,13 +132,13 @@ endif()
 
 set(USE_ELASTICSEARCH false)
 set(USE_CURL false)
-find_package(CURL)
+find_package(LibCURL)
 
-if (CURL_FOUND)
+if (LIBCURL_FOUND)
   set(USE_ELASTICSEARCH true)
   set(USE_CURL true)
-  include_directories(BEFORE ${CURL_INCLUDE_DIR})
-  list(APPEND OPTLIBS ${CURL_LIBRARIES})
+  include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
+  list(APPEND OPTLIBS ${LibCURL_LIBRARIES})
 endif()
 
 if (ENABLE_PERFTOOLS_DEBUG)
diff --git a/configure b/configure
index 8e4aaa8425..c67c02f76d 100755
--- a/configure
+++ b/configure
@@ -61,6 +61,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-swig=PATH       path to SWIG executable
     --with-dataseries=PATH path to DataSeries and Lintel libraries
     --with-xml2=PATH       path to libxml2 installation (for DataSeries)
+    --with-curl=PATH       path to libcurl install root (for ElasticSearch)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -234,6 +235,9 @@ while [ $# -ne 0 ]; do
         --with-xml2=*)
             append_cache_entry LibXML2_ROOT_DIR PATH $optarg
             ;;
+        --with-curl=*)
+            append_cache_entry LibCURL_ROOT_DIR PATH $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;
diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test
index 635726841b..894c9e67f4 100644
--- a/testing/btest/coverage/bare-mode-errors.test
+++ b/testing/btest/coverage/bare-mode-errors.test
@@ -10,5 +10,5 @@
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
-# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
-# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi
+# @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
+# @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

From c69431beac1c636439929e059cda47a4b6bd2f4f Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 8 Oct 2012 16:45:04 -0500
Subject: [PATCH 227/238] Teach --disable-dataseries/--disable-elasticsearch to
 ./configure.

Addresses #877.
---
 CMakeLists.txt | 5 +++--
 configure      | 8 ++++++++
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index e8f9dcd6cd..17ba34ab3b 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -120,7 +120,8 @@ find_package(Lintel)
 find_package(DataSeries)
 find_package(LibXML2)
 
-if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
+if (NOT DISABLE_DATASERIES AND
+    LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
     set(USE_DATASERIES true)
 	include_directories(BEFORE ${Lintel_INCLUDE_DIR})
 	include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
@@ -134,7 +135,7 @@ set(USE_ELASTICSEARCH false)
 set(USE_CURL false)
 find_package(LibCURL)
 
-if (LIBCURL_FOUND)
+if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
   set(USE_ELASTICSEARCH true)
   set(USE_CURL true)
   include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
diff --git a/configure b/configure
index c67c02f76d..6c557a22d0 100755
--- a/configure
+++ b/configure
@@ -38,6 +38,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
+    --disable-dataseries   don't use the optional DataSeries log writer
+    --disable-elasticsearch don't use the optional ElasticSearch log writer
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -175,6 +177,12 @@ while [ $# -ne 0 ]; do
         --disable-ruby)
             append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   true
             ;;
+        --disable-dataseries)
+            append_cache_entry DISABLE_DATASERIES     BOOL   true
+            ;;
+        --disable-elasticsearch)
+            append_cache_entry DISABLE_ELASTICSEARCH     BOOL   true
+            ;;
         --with-openssl=*)
             append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
             ;;

From 82eaddb160727d4489bd45cf787d2d8a8031db96 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 9 Oct 2012 14:00:32 -0700
Subject: [PATCH 228/238] and another bug in the input framework: config table
 does not work (is not transmitted to the readers) because the initialization
 was done the wrong way round.

Just re-ordered some lines, no other changes.
---
 src/input/Manager.cc | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 83e9dc9bc5..df06af7454 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -320,22 +320,12 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 		}
 
 	Unref(mode);
-
+	
 	Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
-
-	ReaderFrontend* reader_obj = new ReaderFrontend(*rinfo, reader);
-	assert(reader_obj);
-
-	info->reader = reader_obj;
-	info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault
-	info->name = name;
 	info->config = config->AsTableVal(); // ref'd by LookupWithDefault
-	info->info = rinfo;
-
-	Ref(description);
-	info->description = description;
-
+		
 		{
+		// create config mapping in ReaderInfo. Has to be done before the construction of reader_obj.
 		HashKey* k;
 		IterCookie* c = info->config->AsTable()->InitForIteration();
 
@@ -345,13 +335,27 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 			ListVal* index = info->config->RecoverIndex(k);
 			string key = index->Index(0)->AsString()->CheckString();
 			string value = v->Value()->AsString()->CheckString();
-			info->info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
+			printf("Inserting %s:%s\n", key.c_str(), value.c_str());
+			rinfo->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
 			Unref(index);
 			delete k;
 			}
 
 		}
 
+
+	ReaderFrontend* reader_obj = new ReaderFrontend(*rinfo, reader);
+	assert(reader_obj);
+
+	info->reader = reader_obj;
+	info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault
+	info->name = name;
+	info->info = rinfo;
+
+	Ref(description);
+	info->description = description;
+
+
 	DBG_LOG(DBG_INPUT, "Successfully created new input stream %s",
 		name.c_str());
 

From 2efb976aaf91c99b596f52c220e6d9e3c783573f Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 9 Oct 2012 14:03:55 -0700
Subject: [PATCH 229/238] ...and forgotten debug-output, sorry (was already
 merged in some other internal repositories before I noticed)

---
 src/input/Manager.cc | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index df06af7454..b5dfdcb2cd 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -335,7 +335,6 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 			ListVal* index = info->config->RecoverIndex(k);
 			string key = index->Index(0)->AsString()->CheckString();
 			string value = v->Value()->AsString()->CheckString();
-			printf("Inserting %s:%s\n", key.c_str(), value.c_str());
 			rinfo->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
 			Unref(index);
 			delete k;

From a6d87fcab7d25fc31fab43e0d09fce0693490a20 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Wed, 10 Oct 2012 11:51:20 -0700
Subject: [PATCH 230/238] rename the update_finished event to end_of_data and
 make it fire in more cases.

It will now not only fire after table-reads have been completed,
but also after the last event of a whole-file-read (or whole-db-read, etc.).

The interface also has been extended a bit to allow readers to
directly fire the event should they so choose. This allows the
event to be fired in direct table-setting/event-sending modes,
which was previously not possible.
---
 doc/input.rst                                 | 10 ++--
 scripts/base/frameworks/input/main.bro        |  4 +-
 src/input/Manager.cc                          | 29 +++++++++--
 src/input/Manager.h                           | 16 ++++--
 src/input/ReaderBackend.cc                    | 19 +++++++
 src/input/ReaderBackend.h                     | 10 ++++
 .../scripts.base.frameworks.input.event/out   | 50 +------------------
 .../scripts/base/frameworks/input/basic.bro   |  2 +-
 .../base/frameworks/input/bignumber.bro       |  2 +-
 .../frameworks/input/empty-values-hashing.bro |  2 +-
 .../base/frameworks/input/emptyvals.bro       |  2 +-
 .../scripts/base/frameworks/input/event.bro   | 15 +++---
 .../base/frameworks/input/invalidnumbers.bro  |  2 +-
 .../frameworks/input/onecolumn-norecord.bro   |  2 +-
 .../frameworks/input/onecolumn-record.bro     |  2 +-
 .../base/frameworks/input/optional.bro        |  2 +-
 .../scripts/base/frameworks/input/port.bro    |  2 +-
 .../base/frameworks/input/predicate.bro       |  2 +-
 .../base/frameworks/input/predicatemodify.bro |  2 +-
 .../input/predicatemodifyandreread.bro        |  2 +-
 .../input/predicaterefusesecondsamerecord.bro |  2 +-
 .../scripts/base/frameworks/input/repeat.bro  |  2 +-
 .../scripts/base/frameworks/input/reread.bro  |  2 +-
 .../scripts/base/frameworks/input/set.bro     |  2 +-
 .../base/frameworks/input/setseparator.bro    |  2 +-
 .../base/frameworks/input/setspecialcases.bro |  2 +-
 .../base/frameworks/input/subrecord.bro       |  2 +-
 .../base/frameworks/input/twotables.bro       |  2 +-
 .../frameworks/input/unsupported_types.bro    |  2 +-
 29 files changed, 101 insertions(+), 94 deletions(-)

diff --git a/doc/input.rst b/doc/input.rst
index 6a089c0635..2945918733 100644
--- a/doc/input.rst
+++ b/doc/input.rst
@@ -98,12 +98,12 @@ been completed. Because of this, it is, for example, possible to call
 will remain queued until the first read has been completed.
 
 Once the input framework finishes reading from a data source, it fires
-the ``update_finished`` event. Once this event has been received all data
+the ``end_of_data`` event. Once this event has been received all data
 from the input file is available in the table.
 
 .. code:: bro
 
-        event Input::update_finished(name: string, source: string) {
+        event Input::end_of_data(name: string, source: string) {
                 # now all data is in the table
                 print blacklist;
         }
@@ -129,7 +129,7 @@ deal with changing data files.
 The first, very basic method is an explicit refresh of an input stream. When
 an input stream is open, the function ``force_update`` can be called. This
 will trigger a complete refresh of the table; any changed elements from the
-file will be updated.  After the update is finished the ``update_finished``
+file will be updated.  After the update is finished the ``end_of_data``
 event will be raised.
 
 In our example the call would look like:
@@ -142,7 +142,7 @@ The input framework also supports two automatic refresh modes. The first mode
 continually checks if a file has been changed. If the file has been changed, it
 is re-read and the data in the Bro table is updated to reflect the current
 state.  Each time a change has been detected and all the new data has been
-read into the table, the ``update_finished`` event is raised.
+read into the table, the ``end_of_data`` event is raised.
 
 The second mode is a streaming mode. This mode assumes that the source data
 file is an append-only file to which new data is continually appended. Bro
@@ -150,7 +150,7 @@ continually checks for new data at the end of the file and will add the new
 data to the table.  If newer lines in the file have the same index as previous
 lines, they will overwrite the values in the output table.  Because of the
 nature of streaming reads (data is continually added to the table),
-the ``update_finished`` event is never raised when using streaming reads.
+the ``end_of_data`` event is never raised when using streaming reads.
 
 The reading mode can be selected by setting the ``mode`` option of the
 add_table call.  Valid values are ``MANUAL`` (the default), ``REREAD``
diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index b5f44669c2..08ab0defb0 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -125,8 +125,8 @@ export {
 	## id: string value identifying the stream
 	global force_update: function(id: string) : bool;
 
-	## Event that is called, when the update of a specific source is finished
-	global update_finished: event(name: string, source:string);
+	## Event that is called, when the end of a data source has been reached, usually after an update
+	global end_of_data: event(name: string, source:string);
 }
 
 @load base/input.bif
diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 83e9dc9bc5..b3906ff73c 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -196,7 +196,7 @@ Manager::TableStream::~TableStream()
 
 Manager::Manager()
 	{
-	update_finished = internal_handler("Input::update_finished");
+	end_of_data = internal_handler("Input::end_of_data");
 	}
 
 Manager::~Manager()
@@ -1169,8 +1169,12 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 	DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str());
 #endif
 
-	if ( i->stream_type == EVENT_STREAM )  // nothing to do..
+	if ( i->stream_type == EVENT_STREAM )  
+		{
+		// just signal the end of the data source
+		SendEndOfData(i);
 		return;
+		}
 
 	assert(i->stream_type == TABLE_STREAM);
 	TableStream* stream = (TableStream*) i;
@@ -1251,14 +1255,29 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 	stream->currDict->SetDeleteFunc(input_hash_delete_func);
 
 #ifdef DEBUG
-	DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event",
+	DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s",
 		i->name.c_str());
 #endif
 
-	// Send event that the current update is indeed finished.
-	SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source));
+	SendEndOfData(i);
 	}
 
+void Manager::SendEndOfData(ReaderFrontend* reader) {
+	Stream *i = FindStream(reader);
+
+	if ( i == 0 )
+		{
+		reporter->InternalError("Unknown reader in SendEndOfData");
+		return;
+		}
+
+	SendEndOfData(i);
+}
+
+void Manager::SendEndOfData(const Stream *i) {
+	SendEvent(end_of_data, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source));
+}
+
 void Manager::Put(ReaderFrontend* reader, Value* *vals)
 	{
 	Stream *i = FindStream(reader);
diff --git a/src/input/Manager.h b/src/input/Manager.h
index cc81df38b7..b7650d33c6 100644
--- a/src/input/Manager.h
+++ b/src/input/Manager.h
@@ -78,7 +78,7 @@ public:
 	 * input.bif, which just forwards here.
 	 */
 	bool RemoveStream(const string &id);
-
+	
 protected:
 	friend class ReaderFrontend;
 	friend class PutMessage;
@@ -89,6 +89,7 @@ protected:
 	friend class EndCurrentSendMessage;
 	friend class ReaderClosedMessage;
 	friend class DisableMessage;
+	friend class EndOfDataMessage;
 
 	// For readers to write to input stream in direct mode (reporting
 	// new/deleted values directly). Functions take ownership of
@@ -96,6 +97,9 @@ protected:
 	void Put(ReaderFrontend* reader, threading::Value* *vals);
 	void Clear(ReaderFrontend* reader);
 	bool Delete(ReaderFrontend* reader, threading::Value* *vals);
+	// Trigger sending the End-of-Data event when the input source has
+	// finished reading. Just use in direct mode.
+	void SendEndOfData(ReaderFrontend* reader);
 
 	// For readers to write to input stream in indirect mode (manager is
 	// monitoring new/deleted values) Functions take ownership of
@@ -119,7 +123,7 @@ protected:
 	// main thread. This makes sure all data that has ben queued for a
 	// stream is still received.
 	bool RemoveStreamContinuation(ReaderFrontend* reader);
-	
+
 	/**
 	 * Deletes an existing input stream.
 	 *
@@ -154,15 +158,18 @@ private:
 	// equivalend in threading cannot be used, because we have support
 	// different types from the log framework
 	bool IsCompatibleType(BroType* t, bool atomic_only=false);
-
 	// Check if a record is made up of compatible types and return a list
 	// of all fields that are in the record in order. Recursively unrolls
 	// records
 	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend, bool allow_file_func);
+	
 
 	// Send events
 	void SendEvent(EventHandlerPtr ev, const int numvals, ...);
 	void SendEvent(EventHandlerPtr ev, list<Val*> events);
+	
+	// Implementation of SendEndOfData (send end_of_data event)
+	void SendEndOfData(const Stream *i);
 
 	// Call predicate function and return result.
 	bool CallPred(Func* pred_func, const int numvals, ...);
@@ -193,6 +200,7 @@ private:
 	// Converts a Bro ListVal to a RecordVal given the record type.
 	RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position);
 
+
 	Stream* FindStream(const string &name);
 	Stream* FindStream(ReaderFrontend* reader);
 
@@ -200,7 +208,7 @@ private:
 
 	map<ReaderFrontend*, Stream*> readers;
 
-	EventHandlerPtr update_finished;
+	EventHandlerPtr end_of_data;
 };
 
 
diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc
index 81060be7d5..74f5306271 100644
--- a/src/input/ReaderBackend.cc
+++ b/src/input/ReaderBackend.cc
@@ -108,6 +108,20 @@ public:
 private:
 };
 
+class EndOfDataMessage : public threading::OutputMessage<ReaderFrontend> {
+public:
+	EndOfDataMessage(ReaderFrontend* reader)
+		: threading::OutputMessage<ReaderFrontend>("EndOfData", reader) {}
+
+	virtual bool Process()
+		{
+		input_mgr->SendEndOfData(Object());
+		return true;
+		}
+
+private:
+};
+
 class ReaderClosedMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
 	ReaderClosedMessage(ReaderFrontend* reader)
@@ -183,6 +197,11 @@ void ReaderBackend::EndCurrentSend()
 	SendOut(new EndCurrentSendMessage(frontend));
 	}
 
+void ReaderBackend::EndOfData()
+	{
+	SendOut(new EndOfDataMessage(frontend));
+	}
+
 void ReaderBackend::SendEntry(Value* *vals)
 	{
 	SendOut(new SendEntryMessage(frontend, vals));
diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h
index 8ee14c808a..32c668fb0d 100644
--- a/src/input/ReaderBackend.h
+++ b/src/input/ReaderBackend.h
@@ -280,6 +280,16 @@ protected:
 	 *
 	 */
 	void Clear();
+	
+	/**
+	 * Method telling the manager that we finished reading the current
+	 * data source. Will trigger an end_of_data event.
+	 *
+	 * Note: When using SendEntry/the tracking mode this is triggered 
+	 * automatically by EndCurrentSend(). Only use if not using the
+	 * tracking mode. Otherwise the event will be sent twice.
+	 */
+	void EndOfData();
 
 	// Content-sending-functions (tracking mode): Only changed lines are propagated.
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out
index 49c1015198..c3f6d1ceba 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.event/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out
@@ -4,13 +4,6 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
@@ -23,13 +16,6 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
@@ -42,13 +28,6 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
@@ -61,13 +40,6 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
@@ -80,13 +52,6 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
@@ -99,13 +64,6 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
@@ -118,16 +76,10 @@ print outfile, A::description;
 print outfile, A::tpe;
 print outfile, A::i;
 print outfile, A::b;
-try = try + 1;
-if (7 == try) 
-{ 
-close(outfile);
-terminate();
-}
-
 }, config={
 
 }]
 Input::EVENT_NEW
 7
 T
+End-of-data
diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro
index faab303534..dfac84d062 100644
--- a/testing/btest/scripts/base/frameworks/input/basic.bro
+++ b/testing/btest/scripts/base/frameworks/input/basic.bro
@@ -55,7 +55,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	print outfile, to_count(servers[-42]$ns); # try to actually use a string. If null-termination is wrong this will fail.
diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro
index 250f84bbb2..5b93472551 100644
--- a/testing/btest/scripts/base/frameworks/input/bignumber.bro
+++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro
@@ -37,7 +37,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
index b66febba82..c8760b467e 100644
--- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
+++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
@@ -73,7 +73,7 @@ event bro_init()
 	}
 
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, "==========SERVERS============";
 	print outfile, servers;
diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro
index a2a9ba3070..94b0f1b620 100644
--- a/testing/btest/scripts/base/frameworks/input/emptyvals.bro
+++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro
@@ -40,7 +40,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro
index d0088472e7..ba47d5e3f2 100644
--- a/testing/btest/scripts/base/frameworks/input/event.bro
+++ b/testing/btest/scripts/base/frameworks/input/event.bro
@@ -22,7 +22,6 @@
 @load frameworks/communication/listen
 
 global outfile: file;
-global try: count;
 
 module A;
 
@@ -37,18 +36,18 @@ event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: b
 	print outfile, tpe;
 	print outfile, i;
 	print outfile, b;
-	try = try + 1;
-	if ( try == 7 )
-		{
-		close(outfile);
-		terminate();
-		}
 	}
 
 event bro_init()
 	{
-	try = 0;
 	outfile = open("../out");
 	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
 	Input::remove("input");
 	}
+
+event Input::end_of_data(name: string, source:string)
+	{
+	print outfile, "End-of-data";
+	close(outfile);
+	terminate();
+	}
diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
index 3c755f1d08..1deec605ae 100644
--- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
+++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
@@ -41,7 +41,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	terminate();
diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro
index 9707af7f94..c08b1420fb 100644
--- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro
+++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro
@@ -38,7 +38,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro
index 18349f1515..9e420e75fe 100644
--- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro
+++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro
@@ -38,7 +38,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro
index 23e0b1e4d1..2fe0e5c86f 100644
--- a/testing/btest/scripts/base/frameworks/input/optional.bro
+++ b/testing/btest/scripts/base/frameworks/input/optional.bro
@@ -48,7 +48,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro
index 2f061e9507..081c59559b 100644
--- a/testing/btest/scripts/base/frameworks/input/port.bro
+++ b/testing/btest/scripts/base/frameworks/input/port.bro
@@ -43,7 +43,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers[1.2.3.4];
 	print outfile, servers[1.2.3.5];
diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro
index fcd986c9a6..8fb33242e8 100644
--- a/testing/btest/scripts/base/frameworks/input/predicate.bro
+++ b/testing/btest/scripts/base/frameworks/input/predicate.bro
@@ -47,7 +47,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	if ( 1 in servers )
 		print outfile, "VALID";
diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro
index 1d6a54fe38..17467bbc27 100644
--- a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro
+++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro
@@ -51,7 +51,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro
index 9b8758bf3f..5a9e993651 100644
--- a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro
+++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro
@@ -94,7 +94,7 @@ event bro_init()
 				]);
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	try = try + 1;
 	print outfile, fmt("Update_finished for %s, try %d", name, try);
diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
index d572b30090..ba0b468cdc 100644
--- a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
+++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
@@ -48,7 +48,7 @@ event bro_init()
 	Input::remove("input");
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro
index a5a914932c..a966ac064e 100644
--- a/testing/btest/scripts/base/frameworks/input/repeat.bro
+++ b/testing/btest/scripts/base/frameworks/input/repeat.bro
@@ -45,7 +45,7 @@ event bro_init()
 		}
 	}
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, name;
 	print outfile, source;
diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro
index 2db58fc6b0..11aa873f9d 100644
--- a/testing/btest/scripts/base/frameworks/input/reread.bro
+++ b/testing/btest/scripts/base/frameworks/input/reread.bro
@@ -123,7 +123,7 @@ event bro_init()
 	}
 
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print outfile, "==========SERVERS============";
 	print outfile, servers;
diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro
index 5215523ee3..b2b5cea323 100644
--- a/testing/btest/scripts/base/frameworks/input/set.bro
+++ b/testing/btest/scripts/base/frameworks/input/set.bro
@@ -38,7 +38,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro
index 44b9d08d54..b7148d80bd 100644
--- a/testing/btest/scripts/base/frameworks/input/setseparator.bro
+++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro
@@ -38,7 +38,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
index 239bdfe7e7..022eac9731 100644
--- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
+++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
@@ -42,7 +42,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/subrecord.bro b/testing/btest/scripts/base/frameworks/input/subrecord.bro
index 8c845a1842..512b8ec58f 100644
--- a/testing/btest/scripts/base/frameworks/input/subrecord.bro
+++ b/testing/btest/scripts/base/frameworks/input/subrecord.bro
@@ -62,7 +62,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);
diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro
index f404416049..83ae86cd46 100644
--- a/testing/btest/scripts/base/frameworks/input/twotables.bro
+++ b/testing/btest/scripts/base/frameworks/input/twotables.bro
@@ -113,7 +113,7 @@ event bro_init()
 	}
 
 
-event Input::update_finished(name: string, source: string)
+event Input::end_of_data(name: string, source: string)
 	{
 	print fin_out, "==========SERVERS============";
 	#print fin_out, servers;
diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
index 7affa4065d..e1350f61a9 100644
--- a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
+++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
@@ -56,7 +56,7 @@ event bro_init()
 	Input::remove("ssh");
 	}
 
-event Input::update_finished(name: string, source:string)
+event Input::end_of_data(name: string, source:string)
 	{
 	print outfile, servers;
 	close(outfile);

From f64c739751522ec3c99a7b3c3a4c8265db1c078c Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 12 Oct 2012 09:50:46 -0700
Subject: [PATCH 231/238] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index a93ef13735..74e6a5401c 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit a93ef1373512c661ffcd0d0a61bd19b96667e0d5
+Subproject commit 74e6a5401c4228d5293c0e309283f43c389e7c12
diff --git a/aux/bro-aux b/aux/bro-aux
index 6748ec3a96..01bb93cb23 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff
+Subproject commit 01bb93cb23f31a98fb400584e8d2f2fbe8a589ef
diff --git a/aux/broccoli b/aux/broccoli
index ebfa4de45a..907210ce14 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit ebfa4de45a839e58aec200e7e4bad33eaab4f1ed
+Subproject commit 907210ce1470724fb386f939cc1b10a4caa2ae39
diff --git a/aux/broctl b/aux/broctl
index b0e3c0d846..b8cbd5a46f 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit b0e3c0d84643878c135dcb8a9774ed78147dd648
+Subproject commit b8cbd5a46fd275c900b5c67f4c6abd5785b83a8a

From 15033a2b626ea8d932ca6743587ead0b2bd70688 Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 12 Oct 2012 16:54:26 -0500
Subject: [PATCH 232/238] Allow faster rebuilds in certain cases

Previously, when rebuilding with a different "--prefix" or "--scriptdir",
all Bro source files were recompiled.  With this change, only util.cc
is recompiled.  Instead of specifying command-line preprocessor
macros on all source files, a header file is regenerated when needed
which only util.cc includes.
---
 src/CMakeLists.txt   | 5 +----
 src/util-config.h.in | 3 +++
 src/util.cc          | 1 +
 3 files changed, 5 insertions(+), 4 deletions(-)
 create mode 100644 src/util-config.h.in

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index ce440852d7..b77863d107 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -4,6 +4,7 @@ include_directories(BEFORE
 )
 
 configure_file(version.c.in ${CMAKE_CURRENT_BINARY_DIR}/version.c)
+configure_file(util-config.h.in ${CMAKE_CURRENT_BINARY_DIR}/util-config.h)
 
 # This creates a custom command to transform a bison output file (inFile)
 # into outFile in order to avoid symbol conflicts:
@@ -444,10 +445,6 @@ set(bro_SRCS
 
 collect_headers(bro_HEADERS ${bro_SRCS})
 
-add_definitions(-DBRO_SCRIPT_INSTALL_PATH="${BRO_SCRIPT_INSTALL_PATH}")
-add_definitions(-DBRO_SCRIPT_SOURCE_PATH="${BRO_SCRIPT_SOURCE_PATH}")
-add_definitions(-DBRO_BUILD_PATH="${CMAKE_CURRENT_BINARY_DIR}")
-
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
 target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT})
diff --git a/src/util-config.h.in b/src/util-config.h.in
new file mode 100644
index 0000000000..c50c4e6b48
--- /dev/null
+++ b/src/util-config.h.in
@@ -0,0 +1,3 @@
+#define BRO_SCRIPT_INSTALL_PATH "@BRO_SCRIPT_INSTALL_PATH@"
+#define BRO_SCRIPT_SOURCE_PATH "@BRO_SCRIPT_SOURCE_PATH@"
+#define BRO_BUILD_PATH "@CMAKE_CURRENT_BINARY_DIR@"
diff --git a/src/util.cc b/src/util.cc
index 3b6fcac76f..76ca7729df 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1,6 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "config.h"
+#include "util-config.h"
 
 #ifdef TIME_WITH_SYS_TIME
 # include <sys/time.h>

From e835a55229315f61e6994811b0eb6423f14c905a Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 17 Oct 2012 11:11:51 -0500
Subject: [PATCH 233/238] Add IPv6 support to signature header conditions.

- "src-ip" and "dst-ip" conditions can now use IPv6 addresses/subnets.
  They must be written in colon-hexadecimal representation and enclosed
  in square brackets (e.g. [fe80::1]).  Addresses #774.

- "icmp6" is now a valid protocol for use with "ip-proto" and "header"
  conditions.  This allows signatures to be written that can match
  against ICMPv6 payloads.  Addresses #880.

- "ip6" is now a valid protocol for use with the "header" condition.
  (also the "ip-proto" condition, but it results in a no-op in that
  case since signatures apply only to the inner-most IP packet when
  packets are tunneled).  This allows signatures to match specifically
  against IPv6 packets (whereas "ip" only matches against IPv4 packets).

- "ip-proto" conditions can now match against IPv6 packets.  Before,
  IPv6 packets were just silently ignored which meant DPD based on
  signatures did not function for IPv6 -- protocol analyzers would only
  get attached to a connection over IPv6 based on the well-known ports
  set in the "dpd_config" table.
---
 doc/signatures.rst                            |  57 ++--
 src/IPAddr.h                                  |  35 +++
 src/RuleMatcher.cc                            | 293 +++++++++++++-----
 src/RuleMatcher.h                             |   9 +-
 src/rule-parse.y                              |  81 ++++-
 src/rule-scan.l                               |  34 +-
 .../Baseline/signatures.dpd/dpd-ipv4.out      |  79 +++++
 .../Baseline/signatures.dpd/dpd-ipv6.out      | 100 ++++++
 .../Baseline/signatures.dpd/nosig-ipv4.out    |   3 +
 .../Baseline/signatures.dpd/nosig-ipv6.out    |   3 +
 .../dst-ip-eq-list.out                        |   1 +
 .../dst-ip-eq-nomatch.out                     |   0
 .../dst-ip-eq.out                             |   1 +
 .../dst-ip-ne-list-nomatch.out                |   0
 .../dst-ip-ne-list.out                        |   1 +
 .../dst-ip-ne-nomatch.out                     |   0
 .../dst-ip-ne.out                             |   1 +
 .../dst-ip-eq-list.out                        |   1 +
 .../dst-ip-eq-nomatch.out                     |   0
 .../dst-ip-eq.out                             |   1 +
 .../dst-ip-ne-list-nomatch.out                |   0
 .../dst-ip-ne-list.out                        |   1 +
 .../dst-ip-ne-nomatch.out                     |   0
 .../dst-ip-ne.out                             |   1 +
 .../dst-ip-eq-list.out                        |   1 +
 .../dst-ip-eq-nomatch.out                     |   0
 .../dst-ip-eq.out                             |   1 +
 .../dst-ip-ne-list-nomatch.out                |   0
 .../dst-ip-ne-list.out                        |   1 +
 .../dst-ip-ne-nomatch.out                     |   0
 .../dst-ip-ne.out                             |   1 +
 .../dst-ip-eq-list.out                        |   1 +
 .../dst-ip-eq-nomatch.out                     |   0
 .../dst-ip-eq.out                             |   1 +
 .../dst-ip-ne-list-nomatch.out                |   0
 .../dst-ip-ne-list.out                        |   1 +
 .../dst-ip-ne-nomatch.out                     |   0
 .../dst-ip-ne.out                             |   1 +
 .../dst-port-eq-ip6.out                       |   1 +
 .../dst-port-eq-list.out                      |   1 +
 .../dst-port-eq-nomatch.out                   |   0
 .../dst-port-eq.out                           |   1 +
 .../dst-port-gt-nomatch.out                   |   0
 .../dst-port-gt.out                           |   1 +
 .../dst-port-gte-nomatch.out                  |   0
 .../dst-port-gte1.out                         |   1 +
 .../dst-port-gte2.out                         |   1 +
 .../dst-port-lt-nomatch.out                   |   0
 .../dst-port-lt.out                           |   1 +
 .../dst-port-lte-nomatch.out                  |   0
 .../dst-port-lte1.out                         |   1 +
 .../dst-port-lte2.out                         |   1 +
 .../dst-port-ne-list-nomatch.out              |   0
 .../dst-port-ne-list.out                      |   1 +
 .../dst-port-ne-nomatch.out                   |   0
 .../dst-port-ne.out                           |   1 +
 .../icmp.out                                  |   1 +
 .../icmp6.out                                 |   1 +
 .../ip-mask.out                               |   1 +
 .../signatures.header-header-condition/ip.out |   1 +
 .../ip6.out                                   |   1 +
 .../tcp.out                                   |   1 +
 .../udp.out                                   |   1 +
 .../val-mask.out                              |   1 +
 .../Baseline/signatures.id-lookup/id.out      |   1 +
 .../icmp6_in_ip6.out                          |   1 +
 .../icmp_in_ip4.out                           |   1 +
 .../nomatch.out                               |   0
 .../tcp_in_ip4.out                            |   1 +
 .../tcp_in_ip6.out                            |   1 +
 .../udp_in_ip4.out                            |   1 +
 .../udp_in_ip6.out                            |   1 +
 .../src-ip-eq-list.out                        |   1 +
 .../src-ip-eq-nomatch.out                     |   0
 .../src-ip-eq.out                             |   1 +
 .../src-ip-ne-list-nomatch.out                |   0
 .../src-ip-ne-list.out                        |   1 +
 .../src-ip-ne-nomatch.out                     |   0
 .../src-ip-ne.out                             |   1 +
 .../src-ip-eq-list.out                        |   1 +
 .../src-ip-eq-nomatch.out                     |   0
 .../src-ip-eq.out                             |   1 +
 .../src-ip-ne-list-nomatch.out                |   0
 .../src-ip-ne-list.out                        |   1 +
 .../src-ip-ne-nomatch.out                     |   0
 .../src-ip-ne.out                             |   1 +
 .../src-ip-eq-list.out                        |   1 +
 .../src-ip-eq-nomatch.out                     |   0
 .../src-ip-eq.out                             |   1 +
 .../src-ip-ne-list-nomatch.out                |   0
 .../src-ip-ne-list.out                        |   1 +
 .../src-ip-ne-nomatch.out                     |   0
 .../src-ip-ne.out                             |   1 +
 .../src-ip-eq-list.out                        |   1 +
 .../src-ip-eq-nomatch.out                     |   0
 .../src-ip-eq.out                             |   1 +
 .../src-ip-ne-list-nomatch.out                |   0
 .../src-ip-ne-list.out                        |   1 +
 .../src-ip-ne-nomatch.out                     |   0
 .../src-ip-ne.out                             |   1 +
 .../src-port-eq-ip6.out                       |   1 +
 .../src-port-eq-list.out                      |   1 +
 .../src-port-eq-nomatch.out                   |   0
 .../src-port-eq.out                           |   1 +
 .../src-port-gt-nomatch.out                   |   0
 .../src-port-gt.out                           |   1 +
 .../src-port-gte-nomatch.out                  |   0
 .../src-port-gte1.out                         |   1 +
 .../src-port-gte2.out                         |   1 +
 .../src-port-lt-nomatch.out                   |   0
 .../src-port-lt.out                           |   1 +
 .../src-port-lte-nomatch.out                  |   0
 .../src-port-lte1.out                         |   1 +
 .../src-port-lte2.out                         |   1 +
 .../src-port-ne-list-nomatch.out              |   0
 .../src-port-ne-list.out                      |   1 +
 .../src-port-ne-nomatch.out                   |   0
 .../src-port-ne.out                           |   1 +
 testing/btest/signatures/dpd.bro              |  54 ++++
 .../dst-ip-header-condition-v4-masks.bro      |  71 +++++
 .../signatures/dst-ip-header-condition-v4.bro |  71 +++++
 .../dst-ip-header-condition-v6-masks.bro      |  71 +++++
 .../signatures/dst-ip-header-condition-v6.bro |  71 +++++
 .../signatures/dst-port-header-condition.bro  | 164 ++++++++++
 .../signatures/header-header-condition.bro    |  78 +++++
 testing/btest/signatures/id-lookup.bro        |  16 +
 .../signatures/ip-proto-header-condition.bro  |  48 +++
 .../src-ip-header-condition-v4-masks.bro      |  71 +++++
 .../signatures/src-ip-header-condition-v4.bro |  71 +++++
 .../src-ip-header-condition-v6-masks.bro      |  71 +++++
 .../signatures/src-ip-header-condition-v6.bro |  71 +++++
 .../signatures/src-port-header-condition.bro  | 164 ++++++++++
 132 files changed, 1731 insertions(+), 124 deletions(-)
 create mode 100644 testing/btest/Baseline/signatures.dpd/dpd-ipv4.out
 create mode 100644 testing/btest/Baseline/signatures.dpd/dpd-ipv6.out
 create mode 100644 testing/btest/Baseline/signatures.dpd/nosig-ipv4.out
 create mode 100644 testing/btest/Baseline/signatures.dpd/nosig-ipv6.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-ip6.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte1.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte2.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte1.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte2.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/icmp.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/icmp6.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/ip-mask.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/ip.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/ip6.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/tcp.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/udp.out
 create mode 100644 testing/btest/Baseline/signatures.header-header-condition/val-mask.out
 create mode 100644 testing/btest/Baseline/signatures.id-lookup/id.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/icmp6_in_ip6.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/icmp_in_ip4.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip4.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip6.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip4.out
 create mode 100644 testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip6.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-ip6.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte1.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte2.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte1.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte2.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-nomatch.out
 create mode 100644 testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne.out
 create mode 100644 testing/btest/signatures/dpd.bro
 create mode 100644 testing/btest/signatures/dst-ip-header-condition-v4-masks.bro
 create mode 100644 testing/btest/signatures/dst-ip-header-condition-v4.bro
 create mode 100644 testing/btest/signatures/dst-ip-header-condition-v6-masks.bro
 create mode 100644 testing/btest/signatures/dst-ip-header-condition-v6.bro
 create mode 100644 testing/btest/signatures/dst-port-header-condition.bro
 create mode 100644 testing/btest/signatures/header-header-condition.bro
 create mode 100644 testing/btest/signatures/id-lookup.bro
 create mode 100644 testing/btest/signatures/ip-proto-header-condition.bro
 create mode 100644 testing/btest/signatures/src-ip-header-condition-v4-masks.bro
 create mode 100644 testing/btest/signatures/src-ip-header-condition-v4.bro
 create mode 100644 testing/btest/signatures/src-ip-header-condition-v6-masks.bro
 create mode 100644 testing/btest/signatures/src-ip-header-condition-v6.bro
 create mode 100644 testing/btest/signatures/src-port-header-condition.bro

diff --git a/doc/signatures.rst b/doc/signatures.rst
index 36099ba40f..59ca819636 100644
--- a/doc/signatures.rst
+++ b/doc/signatures.rst
@@ -83,9 +83,8 @@ Header Conditions
 ~~~~~~~~~~~~~~~~~
 
 Header conditions limit the applicability of the signature to a subset
-of traffic that contains matching packet headers. For TCP, this match
-is performed only for the first packet of a connection. For other
-protocols, it is done on each individual packet.
+of traffic that contains matching packet headers.  This type of matching
+is performed only for the first packet of a connection.
 
 There are pre-defined header conditions for some of the most used
 header fields. All of them generally have the format ``<keyword> <cmp>
@@ -95,14 +94,22 @@ one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``; and
 against. The following keywords are defined:
 
 ``src-ip``/``dst-ip <cmp> <address-list>``
-    Source and destination address, respectively. Addresses can be
-    given as IP addresses or CIDR masks.
+    Source and destination address, respectively. Addresses can be given
+    as IPv4 or IPv6 addresses or CIDR masks.  For IPv6 addresses/masks
+    the colon-hexadecimal representation of the address must be enclosed
+    in square brackets (e.g. ``[fe80::1]`` or ``[fe80::0]/16``).
 
-``src-port``/``dst-port`` ``<int-list>``
+``src-port``/``dst-port <cmp> <int-list>``
     Source and destination port, respectively.
 
-``ip-proto tcp|udp|icmp``
-    IP protocol.
+``ip-proto <cmp> tcp|udp|icmp|icmp6|ip|ip6``
+    IPv4 header's Protocol field or the Next Header field of the final
+    IPv6 header (i.e. either Next Header field in the fixed IPv6 header
+    if no extension headers are present or that field from the last
+    extension header in the chain).  Note that the IP-in-IP forms of
+    tunneling are automatically decapsulated by default and signatures
+    apply to only the inner-most packet, so specifying ``ip`` or ``ip6``
+    is a no-op.
 
 For lists of multiple values, they are sequentially compared against
 the corresponding header field. If at least one of the comparisons
@@ -116,20 +123,22 @@ condition can be defined either as
 
     header <proto>[<offset>:<size>] [& <integer>] <cmp> <value-list>
 
-This compares the value found at the given position of the packet
-header with a list of values. ``offset`` defines the position of the
-value within the header of the protocol defined by ``proto`` (which
-can be ``ip``, ``tcp``, ``udp`` or ``icmp``). ``size`` is either 1, 2,
-or 4 and specifies the value to have a size of this many bytes. If the
-optional ``& <integer>`` is given, the packet's value is first masked
-with the integer before it is compared to the value-list. ``cmp`` is
-one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. ``value-list`` is
-a list of comma-separated integers similar to those described above.
-The integers within the list may be followed by an additional ``/
-mask`` where ``mask`` is a value from 0 to 32. This corresponds to the
-CIDR notation for netmasks and is translated into a corresponding
-bitmask applied to the packet's value prior to the comparison (similar
-to the optional ``& integer``).
+This compares the value found at the given position of the packet header
+with a list of values. ``offset`` defines the position of the value
+within the header of the protocol defined by ``proto`` (which can be
+``ip``, ``ip6``, ``tcp``, ``udp``, ``icmp`` or ``icmp6``). ``size`` is
+either 1, 2, or 4 and specifies the value to have a size of this many
+bytes. If the optional ``& <integer>`` is given, the packet's value is
+first masked with the integer before it is compared to the value-list.
+``cmp`` is one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``.
+``value-list`` is a list of comma-separated integers similar to those
+described above.  The integers within the list may be followed by an
+additional ``/ mask`` where ``mask`` is a value from 0 to 32. This
+corresponds to the CIDR notation for netmasks and is translated into a
+corresponding bitmask applied to the packet's value prior to the
+comparison (similar to the optional ``& integer``).  IPv6 address values
+are not allowed in the value-list, though you can still inspect any 1,
+2, or 4 byte section of an IPv6 header using this keyword.
 
 Putting it all together, this is an example condition that is
 equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
@@ -138,8 +147,8 @@ equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
 
     header ip[16:4] == 1.2.3.4/16, 5.6.7.8/24
 
-Internally, the predefined header conditions are in fact just
-short-cuts and mapped into a generic condition.
+Note that the analogous example for IPv6 isn't currently possible since
+4 bytes is the max width of a value that can be compared.
 
 Content Conditions
 ~~~~~~~~~~~~~~~~~~
diff --git a/src/IPAddr.h b/src/IPAddr.h
index f664f649f9..6d26ef3fa8 100644
--- a/src/IPAddr.h
+++ b/src/IPAddr.h
@@ -342,6 +342,21 @@ public:
 		return memcmp(&addr1.in6, &addr2.in6, sizeof(in6_addr)) < 0;
 		}
 
+	friend bool operator<=(const IPAddr& addr1, const IPAddr& addr2)
+		{
+		return addr1 < addr2 || addr1 == addr2;
+		}
+
+	friend bool operator>=(const IPAddr& addr1, const IPAddr& addr2)
+		{
+		return ! ( addr1 < addr2 );
+		}
+
+	friend bool operator>(const IPAddr& addr1, const IPAddr& addr2)
+		{
+		return ! ( addr1 <= addr2 );
+		}
+
 	/** Converts the address into the type used internally by the
 	  * inter-thread communication.
 	  */
@@ -583,6 +598,11 @@ public:
 		return net1.Prefix() == net2.Prefix() && net1.Length() == net2.Length();
 		}
 
+	friend bool operator!=(const IPPrefix& net1, const IPPrefix& net2)
+		{
+		return ! (net1 == net2);
+		}
+
 	/**
 	 * Comparison operator IP prefixes. This defines a well-defined order for
 	 * IP prefix. However, the order does not necessarily corresponding to their
@@ -600,6 +620,21 @@ public:
 			return false;
 		}
 
+	friend bool operator<=(const IPPrefix& net1, const IPPrefix& net2)
+		{
+		return net1 < net2 || net1 == net2;
+		}
+
+	friend bool operator>=(const IPPrefix& net1, const IPPrefix& net2)
+		{
+		return ! (net1 < net2 );
+		}
+
+	friend bool operator>(const IPPrefix& net1, const IPPrefix& net2)
+		{
+		return ! ( net1 <= net2 );
+		}
+
 private:
 	IPAddr prefix;	// We store it as an address with the non-prefix bits masked out via Mask().
 	uint8_t length;	// The bit length of the prefix relative to full IPv6 addr.
diff --git a/src/RuleMatcher.cc b/src/RuleMatcher.cc
index c9cf1f5c11..db07dff889 100644
--- a/src/RuleMatcher.cc
+++ b/src/RuleMatcher.cc
@@ -1,4 +1,5 @@
 #include <algorithm>
+#include <functional>
 
 #include "config.h"
 
@@ -41,6 +42,23 @@ RuleHdrTest::RuleHdrTest(Prot arg_prot, uint32 arg_offset, uint32 arg_size,
 	level = 0;
 	}
 
+RuleHdrTest::RuleHdrTest(Prot arg_prot, Comp arg_comp, vector<IPPrefix> arg_v)
+	{
+	prot = arg_prot;
+	offset = 0;
+	size = 0;
+	comp = arg_comp;
+	vals = new maskedvalue_list;
+	prefix_vals = arg_v;
+	sibling = 0;
+	child = 0;
+	pattern_rules = 0;
+	pure_rules = 0;
+	ruleset = new IntSet;
+	id = ++idcounter;
+	level = 0;
+	}
+
 Val* RuleMatcher::BuildRuleStateValue(const Rule* rule,
 					const RuleEndpointState* state) const
 	{
@@ -63,6 +81,8 @@ RuleHdrTest::RuleHdrTest(RuleHdrTest& h)
 	loop_over_list(*h.vals, i)
 		vals->append(new MaskedValue(*(*h.vals)[i]));
 
+	prefix_vals = h.prefix_vals;
+
 	for ( int j = 0; j < Rule::TYPES; ++j )
 		{
 		loop_over_list(h.psets[j], k)
@@ -114,6 +134,10 @@ bool RuleHdrTest::operator==(const RuleHdrTest& h)
 		     (*vals)[i]->mask != (*h.vals)[i]->mask )
 			return false;
 
+	for ( size_t i = 0; i < prefix_vals.size(); ++i )
+		if ( ! (prefix_vals[i] == h.prefix_vals[i]) )
+			return false;
+
 	return true;
 	}
 
@@ -129,6 +153,9 @@ void RuleHdrTest::PrintDebug()
 		fprintf(stderr, " 0x%08x/0x%08x",
 				(*vals)[i]->val, (*vals)[i]->mask);
 
+	for ( size_t i = 0; i < prefix_vals.size(); ++i )
+		fprintf(stderr, " %s", prefix_vals[i].AsString().c_str());
+
 	fprintf(stderr, "\n");
 	}
 
@@ -410,29 +437,129 @@ static inline uint32 getval(const u_char* data, int size)
 	}
 
 
-// A line which can be inserted into the macros below for debugging
-// fprintf(stderr, "%.06f %08x & %08x %s %08x\n", network_time, v, (mvals)[i]->mask, #op, (mvals)[i]->val);
-
 // Evaluate a value list (matches if at least one value matches).
-#define DO_MATCH_OR( mvals, v, op )	\
-	{	\
-	loop_over_list((mvals), i)	\
-		{	\
-		if ( ((v) & (mvals)[i]->mask) op (mvals)[i]->val )	\
-			goto match;	\
-		}	\
-	goto no_match;	\
+template <typename FuncT>
+static inline bool match_or(const maskedvalue_list& mvals, uint32 v, FuncT comp)
+	{
+	loop_over_list(mvals, i)
+		{
+		if ( comp(v & mvals[i]->mask, mvals[i]->val) )
+			return true;
+		}
+	return false;
+	}
+
+// Evaluate a prefix list (matches if at least one value matches).
+template <typename FuncT>
+static inline bool match_or(const vector<IPPrefix>& prefixes, const IPAddr& a,
+                            FuncT comp)
+	{
+	for ( size_t i = 0; i < prefixes.size(); ++i )
+		{
+		IPAddr masked(a);
+		masked.Mask(prefixes[i].LengthIPv6());
+		if ( comp(masked, prefixes[i].Prefix()) )
+			return true;
+		}
+	return false;
 	}
 
 // Evaluate a value list (doesn't match if any value matches).
-#define DO_MATCH_NOT_AND( mvals, v, op )	\
-	{	\
-	loop_over_list((mvals), i)	\
-		{	\
-		if ( ((v) & (mvals)[i]->mask) op (mvals)[i]->val )	\
-			goto no_match;	\
-		}	\
-	goto match;	\
+template <typename FuncT>
+static inline bool match_not_and(const maskedvalue_list& mvals, uint32 v,
+                                 FuncT comp)
+	{
+	loop_over_list(mvals, i)
+		{
+		if ( comp(v & mvals[i]->mask, mvals[i]->val) )
+			return false;
+		}
+	return true;
+	}
+
+// Evaluate a prefix list (doesn't match if any value matches).
+template <typename FuncT>
+static inline bool match_not_and(const vector<IPPrefix>& prefixes,
+                                 const IPAddr& a, FuncT comp)
+	{
+	for ( size_t i = 0; i < prefixes.size(); ++i )
+		{
+		IPAddr masked(a);
+		masked.Mask(prefixes[i].LengthIPv6());
+		if ( comp(masked, prefixes[i].Prefix()) )
+			return false;
+		}
+	return true;
+	}
+
+static inline bool compare(const maskedvalue_list& mvals, uint32 v,
+                           RuleHdrTest::Comp comp)
+	{
+	switch ( comp ) {
+		case RuleHdrTest::EQ:
+			return match_or(mvals, v, std::equal_to<uint32>());
+			break;
+
+		case RuleHdrTest::NE:
+			return match_not_and(mvals, v, std::equal_to<uint32>());
+			break;
+
+		case RuleHdrTest::LT:
+			return match_or(mvals, v, std::less<uint32>());
+			break;
+
+		case RuleHdrTest::GT:
+			return match_or(mvals, v, std::greater<uint32>());
+			break;
+
+		case RuleHdrTest::LE:
+			return match_or(mvals, v, std::less_equal<uint32>());
+			break;
+
+		case RuleHdrTest::GE:
+			return match_or(mvals, v, std::greater_equal<uint32>());
+			break;
+
+		default:
+			reporter->InternalError("unknown comparison type");
+			break;
+	}
+	return false;
+	}
+
+static inline bool compare(const vector<IPPrefix>& prefixes, const IPAddr& a,
+                           RuleHdrTest::Comp comp)
+	{
+	switch ( comp ) {
+		case RuleHdrTest::EQ:
+			return match_or(prefixes, a, std::equal_to<IPAddr>());
+			break;
+
+		case RuleHdrTest::NE:
+			return match_not_and(prefixes, a, std::equal_to<IPAddr>());
+			break;
+
+		case RuleHdrTest::LT:
+			return match_or(prefixes, a, std::less<IPAddr>());
+			break;
+
+		case RuleHdrTest::GT:
+			return match_or(prefixes, a, std::greater<IPAddr>());
+			break;
+
+		case RuleHdrTest::LE:
+			return match_or(prefixes, a, std::less_equal<IPAddr>());
+			break;
+
+		case RuleHdrTest::GE:
+			return match_or(prefixes, a, std::greater_equal<IPAddr>());
+			break;
+
+		default:
+			reporter->InternalError("unknown comparison type");
+			break;
+	}
+	return false;
 	}
 
 RuleEndpointState* RuleMatcher::InitEndpoint(Analyzer* analyzer,
@@ -492,66 +619,52 @@ RuleEndpointState* RuleMatcher::InitEndpoint(Analyzer* analyzer,
 
 		if ( ip )
 			{
-			// Get start of transport layer.
-			const u_char* transport = ip->Payload();
-
 			// Descend the RuleHdrTest tree further.
 			for ( RuleHdrTest* h = hdr_test->child; h;
 			      h = h->sibling )
 				{
-				const u_char* data;
+				bool match = false;
 
 				// Evaluate the header test.
 				switch ( h->prot ) {
+				case RuleHdrTest::NEXT:
+					match = compare(*h->vals, ip->NextProto(), h->comp);
+					break;
+
 				case RuleHdrTest::IP:
-					data = (const u_char*) ip->IP4_Hdr();
+					if ( ! ip->IP4_Hdr() )
+						continue;
+					match = compare(*h->vals, getval((const u_char*)ip->IP4_Hdr() + h->offset, h->size), h->comp);
+					break;
+
+				case RuleHdrTest::IPv6:
+					if ( ! ip->IP6_Hdr() )
+						continue;
+					match = compare(*h->vals, getval((const u_char*)ip->IP6_Hdr() + h->offset, h->size), h->comp);
 					break;
 
 				case RuleHdrTest::ICMP:
+				case RuleHdrTest::ICMPv6:
 				case RuleHdrTest::TCP:
 				case RuleHdrTest::UDP:
-					data = transport;
+					match = compare(*h->vals, getval(ip->Payload() + h->offset, h->size), h->comp);
+					break;
+
+				case RuleHdrTest::IPSrc:
+					match = compare(h->prefix_vals, ip->IPHeaderSrcAddr(), h->comp);
+					break;
+
+				case RuleHdrTest::IPDst:
+					match = compare(h->prefix_vals, ip->IPHeaderDstAddr(), h->comp);
 					break;
 
 				default:
-					data = 0;
 					reporter->InternalError("unknown protocol");
+					break;
 				}
 
-				// ### data can be nil here if it's an
-				// IPv6 packet and we're doing an IP test.
-				if ( ! data )
-					continue;
-
-				// Sorry for the hidden gotos :-)
-				switch ( h->comp ) {
-					case RuleHdrTest::EQ:
-						DO_MATCH_OR(*h->vals, getval(data + h->offset, h->size), ==);
-
-					case RuleHdrTest::NE:
-						DO_MATCH_NOT_AND(*h->vals, getval(data + h->offset, h->size), ==);
-
-					case RuleHdrTest::LT:
-						DO_MATCH_OR(*h->vals, getval(data + h->offset, h->size), <);
-
-					case RuleHdrTest::GT:
-						DO_MATCH_OR(*h->vals, getval(data + h->offset, h->size), >);
-
-					case RuleHdrTest::LE:
-						DO_MATCH_OR(*h->vals, getval(data + h->offset, h->size), <=);
-
-					case RuleHdrTest::GE:
-						DO_MATCH_OR(*h->vals, getval(data + h->offset, h->size), >=);
-
-					default:
-						reporter->InternalError("unknown comparision type");
-				}
-
-no_match:
-				continue;
-
-match:
-				tests.append(h);
+				if ( match )
+					tests.append(h);
 				}
 			}
 		}
@@ -1050,8 +1163,11 @@ static Val* get_bro_val(const char* label)
 	}
 
 
-// Converts an atomic Val and appends it to the list
-static bool val_to_maskedval(Val* v, maskedvalue_list* append_to)
+// Converts an atomic Val and appends it to the list.  For subnet types,
+// if the prefix_vector param isn't null, appending to that is preferred
+// over appending to the masked val list.
+static bool val_to_maskedval(Val* v, maskedvalue_list* append_to,
+                             vector<IPPrefix>* prefix_vector)
 	{
 	MaskedValue* mval = new MaskedValue;
 
@@ -1071,29 +1187,37 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to)
 
 		case TYPE_SUBNET:
 			{
-			const uint32* n;
-			uint32 m[4];
-			v->AsSubNet().Prefix().GetBytes(&n);
-			v->AsSubNetVal()->Mask().CopyIPv6(m);
-
-			for ( unsigned int i = 0; i < 4; ++i )
-				m[i] = ntohl(m[i]);
-
-			bool is_v4_mask = m[0] == 0xffffffff &&
-						m[1] == m[0] && m[2] == m[0];
-
-			if ( v->AsSubNet().Prefix().GetFamily() == IPv4 &&
-			     is_v4_mask )
+			if ( prefix_vector )
 				{
-				mval->val = ntohl(*n);
-				mval->mask = m[3];
+				prefix_vector->push_back(v->AsSubNet());
+				delete mval;
+				return true;
 				}
-
 			else
 				{
-				rules_error("IPv6 subnets not supported");
-				mval->val = 0;
-				mval->mask = 0;
+				const uint32* n;
+				uint32 m[4];
+				v->AsSubNet().Prefix().GetBytes(&n);
+				v->AsSubNetVal()->Mask().CopyIPv6(m);
+
+				for ( unsigned int i = 0; i < 4; ++i )
+					m[i] = ntohl(m[i]);
+
+				bool is_v4_mask = m[0] == 0xffffffff &&
+				                          m[1] == m[0] && m[2] == m[0];
+
+
+				if ( v->AsSubNet().Prefix().GetFamily() == IPv4 && is_v4_mask )
+					{
+					mval->val = ntohl(*n);
+					mval->mask = m[3];
+					}
+				else
+					{
+					rules_error("IPv6 subnets not supported");
+					mval->val = 0;
+					mval->mask = 0;
+					}
 				}
 			}
 			break;
@@ -1108,7 +1232,8 @@ static bool val_to_maskedval(Val* v, maskedvalue_list* append_to)
 	return true;
 	}
 
-void id_to_maskedvallist(const char* id, maskedvalue_list* append_to)
+void id_to_maskedvallist(const char* id, maskedvalue_list* append_to,
+                         vector<IPPrefix>* prefix_vector)
 	{
 	Val* v = get_bro_val(id);
 	if ( ! v )
@@ -1118,7 +1243,7 @@ void id_to_maskedvallist(const char* id, maskedvalue_list* append_to)
 		{
 		val_list* vals = v->AsTableVal()->ConvertToPureList()->Vals();
 		loop_over_list(*vals, i )
-			if ( ! val_to_maskedval((*vals)[i], append_to) )
+			if ( ! val_to_maskedval((*vals)[i], append_to, prefix_vector) )
 			{
 				delete_vals(vals);
 				return;
@@ -1128,7 +1253,7 @@ void id_to_maskedvallist(const char* id, maskedvalue_list* append_to)
 		}
 
 	else
-		val_to_maskedval(v, append_to);
+		val_to_maskedval(v, append_to, prefix_vector);
 	}
 
 char* id_to_str(const char* id)
diff --git a/src/RuleMatcher.h b/src/RuleMatcher.h
index 5bba69e130..b8895513b4 100644
--- a/src/RuleMatcher.h
+++ b/src/RuleMatcher.h
@@ -2,7 +2,9 @@
 #define sigs_h
 
 #include <limits.h>
+#include <vector>
 
+#include "IPAddr.h"
 #include "BroString.h"
 #include "List.h"
 #include "RE.h"
@@ -59,17 +61,19 @@ declare(PList, BroString);
 typedef PList(BroString) bstr_list;
 
 // Get values from Bro's script-level variables.
-extern void id_to_maskedvallist(const char* id, maskedvalue_list* append_to);
+extern void id_to_maskedvallist(const char* id, maskedvalue_list* append_to,
+                                vector<IPPrefix>* prefix_vector = 0);
 extern char* id_to_str(const char* id);
 extern uint32 id_to_uint(const char* id);
 
 class RuleHdrTest {
 public:
 	enum Comp { LE, GE, LT, GT, EQ, NE };
-	enum Prot { NOPROT, IP, ICMP, TCP, UDP };
+	enum Prot { NOPROT, IP, IPv6, ICMP, ICMPv6, TCP, UDP, NEXT, IPSrc, IPDst };
 
 	RuleHdrTest(Prot arg_prot, uint32 arg_offset, uint32 arg_size,
 			Comp arg_comp, maskedvalue_list* arg_vals);
+	RuleHdrTest(Prot arg_prot, Comp arg_comp, vector<IPPrefix> arg_v);
 	~RuleHdrTest();
 
 	void PrintDebug();
@@ -86,6 +90,7 @@ private:
 	Prot prot;
 	Comp comp;
 	maskedvalue_list* vals;
+	vector<IPPrefix> prefix_vals; // for use with IPSrc/IPDst comparisons
 	uint32 offset;
 	uint32 size;
 
diff --git a/src/rule-parse.y b/src/rule-parse.y
index c8770c3e22..d85aea2835 100644
--- a/src/rule-parse.y
+++ b/src/rule-parse.y
@@ -1,13 +1,27 @@
 %{
 #include <stdio.h>
+#include <netinet/in.h>
+#include <vector>
+#include "config.h"
 #include "RuleMatcher.h"
 #include "Reporter.h"
+#include "IPAddr.h"
+#include "net_util.h"
 
 extern void begin_PS();
 extern void end_PS();
 
 Rule* current_rule = 0;
 const char* current_rule_file = 0;
+
+static uint8_t mask_to_len(uint32_t mask)
+	{
+	if ( mask == 0xffffffff ) return 32;
+	uint32_t x = ~mask + 1;
+	uint8_t len;
+	for ( len = 0; len < 32 && (! (x & (1<<len))); ++len );
+	return len;
+	}
 %}
 
 %token TOK_COMP
@@ -21,6 +35,7 @@ const char* current_rule_file = 0;
 %token TOK_IDENT
 %token TOK_INT
 %token TOK_IP
+%token TOK_IP6
 %token TOK_IP_OPTIONS
 %token TOK_IP_OPTION_SYM
 %token TOK_IP_PROTO
@@ -49,7 +64,9 @@ const char* current_rule_file = 0;
 %type <hdr_test> hdr_expr
 %type <range> range rangeopt
 %type <vallist> value_list
+%type <prefix_val_list> prefix_value_list
 %type <mval> TOK_IP value
+%type <prefixval> TOK_IP6 prefix_value
 %type <prot> TOK_PROT
 %type <ptype> TOK_PATTERN_TYPE
 
@@ -57,6 +74,8 @@ const char* current_rule_file = 0;
 	Rule* rule;
 	RuleHdrTest* hdr_test;
 	maskedvalue_list* vallist;
+	vector<IPPrefix>* prefix_val_list;
+	IPPrefix* prefixval;
 
 	bool bl;
 	int val;
@@ -91,11 +110,11 @@ rule_attr_list:
 	;
 
 rule_attr:
-		TOK_DST_IP TOK_COMP value_list
+		TOK_DST_IP TOK_COMP prefix_value_list
 			{
 			current_rule->AddHdrTest(new RuleHdrTest(
-				RuleHdrTest::IP, 16, 4,
-				(RuleHdrTest::Comp) $2, $3));
+				RuleHdrTest::IPDst,
+				(RuleHdrTest::Comp) $2, *($3)));
 			}
 
 	|	TOK_DST_PORT TOK_COMP value_list
@@ -123,10 +142,14 @@ rule_attr:
 			{
 			int proto = 0;
 			switch ( $3 ) {
-			case RuleHdrTest::ICMP: proto = 1; break;
+			case RuleHdrTest::ICMP: proto = IPPROTO_ICMP; break;
+			case RuleHdrTest::ICMPv6: proto = IPPROTO_ICMPV6; break;
+			// signature matching against outer packet headers of IP-in-IP
+			// tunneling not supported, so do a no-op there
 			case RuleHdrTest::IP: proto = 0; break;
-			case RuleHdrTest::TCP: proto = 6; break;
-			case RuleHdrTest::UDP: proto = 17; break;
+			case RuleHdrTest::IPv6: proto = 0; break;
+			case RuleHdrTest::TCP: proto = IPPROTO_TCP; break;
+			case RuleHdrTest::UDP: proto = IPPROTO_UDP; break;
 			default:
 				rules_error("internal_error: unknown protocol");
 			}
@@ -140,16 +163,20 @@ rule_attr:
 				val->mask = 0xffffffff;
 				vallist->append(val);
 
+				// offset & size params are dummies, actual next proto value in
+				// header is retrieved dynamically via IP_Hdr::NextProto()
 				current_rule->AddHdrTest(new RuleHdrTest(
-					RuleHdrTest::IP, 9, 1,
+					RuleHdrTest::NEXT, 0, 0,
 					(RuleHdrTest::Comp) $2, vallist));
 				}
 			}
 
 	|	TOK_IP_PROTO TOK_COMP value_list
 			{
+			// offset & size params are dummies, actual next proto value in
+			// header is retrieved dynamically via IP_Hdr::NextProto()
 			current_rule->AddHdrTest(new RuleHdrTest(
-				RuleHdrTest::IP, 9, 1,
+				RuleHdrTest::NEXT, 0, 0,
 				(RuleHdrTest::Comp) $2, $3));
 			}
 
@@ -193,11 +220,11 @@ rule_attr:
 	|	TOK_SAME_IP
 			{ current_rule->AddCondition(new RuleConditionSameIP()); }
 
-	|	TOK_SRC_IP TOK_COMP value_list
+	|	TOK_SRC_IP TOK_COMP prefix_value_list
 			{
 			current_rule->AddHdrTest(new RuleHdrTest(
-				RuleHdrTest::IP, 12, 4,
-				(RuleHdrTest::Comp) $2, $3));
+				RuleHdrTest::IPSrc,
+				(RuleHdrTest::Comp) $2, *($3)));
 			}
 
 	|	TOK_SRC_PORT TOK_COMP value_list
@@ -254,6 +281,38 @@ value_list:
 			}
 	;
 
+prefix_value_list:
+		prefix_value_list ',' prefix_value
+			{
+			$$ = $1;
+			$$->push_back(*($3));
+			}
+	|	prefix_value_list ',' TOK_IDENT
+			{
+			$$ = $1;
+			id_to_maskedvallist($3, 0, $1);
+			}
+	|	prefix_value
+			{
+			$$ = new vector<IPPrefix>();
+			$$->push_back(*($1));
+			}
+	|	TOK_IDENT
+			{
+			$$ = new vector<IPPrefix>();
+			id_to_maskedvallist($1, 0, $$);
+			}
+	;
+
+prefix_value:
+		TOK_IP
+			{
+			$$ = new IPPrefix(IPAddr(IPv4, &($1.val), IPAddr::Host),
+			                  mask_to_len($1.mask));
+			}
+	|	TOK_IP6
+	;
+
 value:
 		TOK_INT
 			{ $$.val = $1; $$.mask = 0xffffffff; }
diff --git a/src/rule-scan.l b/src/rule-scan.l
index 781c477ff2..48995f8ccd 100644
--- a/src/rule-scan.l
+++ b/src/rule-scan.l
@@ -1,24 +1,37 @@
 %{
-typedef unsigned int uint32;
-
 #include <string.h>
+#include <string>
 #include <sys/types.h>
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
 #include "RuleMatcher.h"
+#include "IPAddr.h"
+#include "util.h"
 #include "rule-parse.h"
 
 int rules_line_number = 0;
+
+static string extract_ipv6(string s)
+	{
+	if ( s.substr(0, 3) == "[0x" )
+		s = s.substr(3, s.find("]") - 3);
+	else
+		s = s.substr(1, s.find("]") - 1);
+	return s;
+	}
 %}
 
 %x PS
 
+OWS	[ \t]*
 WS	[ \t]+
 D	[0-9]+
 H	[0-9a-fA-F]+
+HEX {H}
 STRING	\"([^\n\"]|\\\")*\"
 ID	([0-9a-zA-Z_-]+::)*[0-9a-zA-Z_-]+
+IP6	("["({HEX}:){7}{HEX}"]")|("["0x{HEX}({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*({D}"."){3}{D}"]")
 RE	\/(\\\/)?([^/]|[^\\]\\\/)*\/
 META	\.[^ \t]+{WS}[^\n]+
 PID	([0-9a-zA-Z_-]|"::")+
@@ -34,6 +47,19 @@ PID	([0-9a-zA-Z_-]|"::")+
 	\n	++rules_line_number;
 }
 
+{IP6}	{
+	rules_lval.prefixval = new IPPrefix(IPAddr(extract_ipv6(yytext)), 128);
+	return TOK_IP6;
+	}
+
+{IP6}{OWS}"/"{OWS}{D}	{
+	char* l = strchr(yytext, '/');
+	*l++ = '\0';
+	int len = atoi(l);
+	rules_lval.prefixval = new IPPrefix(IPAddr(extract_ipv6(yytext)), len);
+	return TOK_IP6;
+	}
+
 [!\]\[{}&:,]	return rules_text[0];
 
 "<="	{ rules_lval.val = RuleHdrTest::LE; return TOK_COMP; }
@@ -45,7 +71,9 @@ PID	([0-9a-zA-Z_-]|"::")+
 "!="	{ rules_lval.val = RuleHdrTest::NE; return TOK_COMP; }
 
 ip	{ rules_lval.val = RuleHdrTest::IP; return TOK_PROT; }
+ip6	{ rules_lval.val = RuleHdrTest::IPv6; return TOK_PROT; }
 icmp	{ rules_lval.val = RuleHdrTest::ICMP; return TOK_PROT; }
+icmp6	{ rules_lval.val = RuleHdrTest::ICMPv6; return TOK_PROT; }
 tcp	{ rules_lval.val = RuleHdrTest::TCP; return TOK_PROT; }
 udp	{ rules_lval.val = RuleHdrTest::UDP; return TOK_PROT; }
 
@@ -123,7 +151,7 @@ http	{ rules_lval.val = Rule::HTTP_REQUEST; return TOK_PATTERN_TYPE; }
 ftp	{ rules_lval.val = Rule::FTP; return TOK_PATTERN_TYPE; }
 finger	{ rules_lval.val = Rule::FINGER; return TOK_PATTERN_TYPE; }
 
-{D}("."{D}){3}"/"{D}	{
+{D}("."{D}){3}{OWS}"/"{OWS}{D}	{
 	char* s = strchr(yytext, '/');
 	*s++ = '\0';
 
diff --git a/testing/btest/Baseline/signatures.dpd/dpd-ipv4.out b/testing/btest/Baseline/signatures.dpd/dpd-ipv4.out
new file mode 100644
index 0000000000..abb41f330c
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dpd/dpd-ipv4.out
@@ -0,0 +1,79 @@
+dpd_config, {
+
+}
+signature_match [orig_h=141.142.220.235, orig_p=50003/tcp, resp_h=199.233.217.249, resp_p=21/tcp] - matched my_ftp_client
+ftp_reply 199.233.217.249:21 - 220 ftp.NetBSD.org FTP server (NetBSD-ftpd 20100320) ready.
+ftp_request 141.142.220.235:50003 - USER anonymous
+ftp_reply 199.233.217.249:21 - 331 Guest login ok, type your name as password.
+signature_match [orig_h=141.142.220.235, orig_p=50003/tcp, resp_h=199.233.217.249, resp_p=21/tcp] - matched my_ftp_server
+ftp_request 141.142.220.235:50003 - PASS test
+ftp_reply 199.233.217.249:21 - 230 
+ftp_reply 199.233.217.249:21 - 0     The NetBSD Project FTP Server located in Redwood City, CA, USA
+ftp_reply 199.233.217.249:21 - 0     1 Gbps connectivity courtesy of                          ,        ,
+ftp_reply 199.233.217.249:21 - 0     Internet Systems Consortium                 WELCOME!    /(        )`
+ftp_reply 199.233.217.249:21 - 0                                                             \ \___   / |
+ftp_reply 199.233.217.249:21 - 0       +--- Currently Supported Platforms ----+              /- _  `-/  '
+ftp_reply 199.233.217.249:21 - 0       |  acorn[26,32], algor, alpha, amd64,  |             (/\/ \ \   /\
+ftp_reply 199.233.217.249:21 - 0       |   amiga[,ppc], arc, atari, bebox,    |             / /   | `    \
+ftp_reply 199.233.217.249:21 - 0       |   cats, cesfic, cobalt, dreamcast,   |             O O   ) /    |
+ftp_reply 199.233.217.249:21 - 0       |  evb[arm,mips,ppc,sh3], hp[300,700], |             `-^--'`<     '
+ftp_reply 199.233.217.249:21 - 0       |       hpc[arm,mips,sh], i386,        |            (_.)  _  )   /
+ftp_reply 199.233.217.249:21 - 0       |      ibmnws, iyonix, luna68k,        |              .___/`    /
+ftp_reply 199.233.217.249:21 - 0       |    mac[m68k,ppc], mipsco, mmeye,     |               `-----' /
+ftp_reply 199.233.217.249:21 - 0       |      mvme[m68k,ppc], netwinders,     |  <----.     __ / __   \
+ftp_reply 199.233.217.249:21 - 0       |   news[m68k,mips], next68k, ofppc,   |  <----|====O)))==) \) /====
+ftp_reply 199.233.217.249:21 - 0       | playstation2, pmax, prep, sandpoint, |  <----'    `--' `.__,' \
+ftp_reply 199.233.217.249:21 - 0       |  sbmips, sgimips, shark, sparc[,64], |               |        |
+ftp_reply 199.233.217.249:21 - 0       |      sun[2,3], vax, x68k, xen        |                \       /
+ftp_reply 199.233.217.249:21 - 0       +--------------------------------------+           ______( (_  / \_____
+ftp_reply 199.233.217.249:21 - 0       See our website at http://www.NetBSD.org/        ,'  ,-----'   |       \
+ftp_reply 199.233.217.249:21 - 0        We log all FTP transfers and commands.          `--{__________)  (FL) \/
+ftp_reply 199.233.217.249:21 - 0 230-
+ftp_reply 199.233.217.249:21 - 0     EXPORT NOTICE
+ftp_reply 199.233.217.249:21 - 0     
+ftp_reply 199.233.217.249:21 - 0     Please note that portions of this FTP site contain cryptographic
+ftp_reply 199.233.217.249:21 - 0     software controlled under the Export Administration Regulations (EAR).
+ftp_reply 199.233.217.249:21 - 0     
+ftp_reply 199.233.217.249:21 - 0     None of this software may be downloaded or otherwise exported or
+ftp_reply 199.233.217.249:21 - 0     re-exported into (or to a national or resident of) Cuba, Iran, Libya,
+ftp_reply 199.233.217.249:21 - 0     Sudan, North Korea, Syria or any other country to which the U.S. has
+ftp_reply 199.233.217.249:21 - 0     embargoed goods.
+ftp_reply 199.233.217.249:21 - 0     
+ftp_reply 199.233.217.249:21 - 0     By downloading or using said software, you are agreeing to the
+ftp_reply 199.233.217.249:21 - 0     foregoing and you are representing and warranting that you are not
+ftp_reply 199.233.217.249:21 - 0     located in, under the control of, or a national or resident of any
+ftp_reply 199.233.217.249:21 - 0     such country or on any such list.
+ftp_reply 199.233.217.249:21 - 230 Guest login ok, access restrictions apply.
+ftp_request 141.142.220.235:50003 - SYST 
+ftp_reply 199.233.217.249:21 - 215 UNIX Type: L8 Version: NetBSD-ftpd 20100320
+ftp_request 141.142.220.235:50003 - PASV 
+ftp_reply 199.233.217.249:21 - 227 Entering Passive Mode (199,233,217,249,221,90)
+ftp_request 141.142.220.235:50003 - LIST 
+ftp_reply 199.233.217.249:21 - 150 Opening ASCII mode data connection for '/bin/ls'.
+ftp_reply 199.233.217.249:21 - 226 Transfer complete.
+ftp_request 141.142.220.235:50003 - TYPE I
+ftp_reply 199.233.217.249:21 - 200 Type set to I.
+ftp_request 141.142.220.235:50003 - PASV 
+ftp_reply 199.233.217.249:21 - 227 Entering Passive Mode (199,233,217,249,221,91)
+ftp_request 141.142.220.235:50003 - RETR robots.txt
+ftp_reply 199.233.217.249:21 - 150 Opening BINARY mode data connection for 'robots.txt' (77 bytes).
+ftp_reply 199.233.217.249:21 - 226 Transfer complete.
+ftp_request 141.142.220.235:50003 - TYPE A
+ftp_reply 199.233.217.249:21 - 200 Type set to A.
+ftp_request 141.142.220.235:50003 - PORT 141,142,220,235,131,46
+ftp_reply 199.233.217.249:21 - 200 PORT command successful.
+ftp_request 141.142.220.235:50003 - LIST 
+ftp_reply 199.233.217.249:21 - 150 Opening ASCII mode data connection for '/bin/ls'.
+ftp_reply 199.233.217.249:21 - 226 Transfer complete.
+ftp_request 141.142.220.235:50003 - TYPE I
+ftp_reply 199.233.217.249:21 - 200 Type set to I.
+ftp_request 141.142.220.235:50003 - PORT 141,142,220,235,147,203
+ftp_reply 199.233.217.249:21 - 200 PORT command successful.
+ftp_request 141.142.220.235:50003 - RETR robots.txt
+ftp_reply 199.233.217.249:21 - 150 Opening BINARY mode data connection for 'robots.txt' (77 bytes).
+ftp_reply 199.233.217.249:21 - 226 Transfer complete.
+ftp_request 141.142.220.235:50003 - QUIT 
+ftp_reply 199.233.217.249:21 - 221 
+ftp_reply 199.233.217.249:21 - 0     Data traffic for this session was 154 bytes in 2 files.
+ftp_reply 199.233.217.249:21 - 0     Total traffic for this session was 4037 bytes in 4 transfers.
+ftp_reply 199.233.217.249:21 - 221 Thank you for using the FTP service on ftp.NetBSD.org.
diff --git a/testing/btest/Baseline/signatures.dpd/dpd-ipv6.out b/testing/btest/Baseline/signatures.dpd/dpd-ipv6.out
new file mode 100644
index 0000000000..a2227ee890
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dpd/dpd-ipv6.out
@@ -0,0 +1,100 @@
+dpd_config, {
+
+}
+signature_match [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] - matched my_ftp_client
+ftp_reply [2001:470:4867:99::21]:21 - 220 ftp.NetBSD.org FTP server (NetBSD-ftpd 20100320) ready.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - USER anonymous
+ftp_reply [2001:470:4867:99::21]:21 - 331 Guest login ok, type your name as password.
+signature_match [orig_h=2001:470:1f11:81f:c999:d94:aa7c:2e3e, orig_p=49185/tcp, resp_h=2001:470:4867:99::21, resp_p=21/tcp] - matched my_ftp_server
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - PASS test
+ftp_reply [2001:470:4867:99::21]:21 - 230 
+ftp_reply [2001:470:4867:99::21]:21 - 0     The NetBSD Project FTP Server located in Redwood City, CA, USA
+ftp_reply [2001:470:4867:99::21]:21 - 0     1 Gbps connectivity courtesy of                          ,        ,
+ftp_reply [2001:470:4867:99::21]:21 - 0     Internet Systems Consortium                 WELCOME!    /(        )`
+ftp_reply [2001:470:4867:99::21]:21 - 0                                                             \ \___   / |
+ftp_reply [2001:470:4867:99::21]:21 - 0       +--- Currently Supported Platforms ----+              /- _  `-/  '
+ftp_reply [2001:470:4867:99::21]:21 - 0       |  acorn[26,32], algor, alpha, amd64,  |             (/\/ \ \   /\
+ftp_reply [2001:470:4867:99::21]:21 - 0       |   amiga[,ppc], arc, atari, bebox,    |             / /   | `    \
+ftp_reply [2001:470:4867:99::21]:21 - 0       |   cats, cesfic, cobalt, dreamcast,   |             O O   ) /    |
+ftp_reply [2001:470:4867:99::21]:21 - 0       |  evb[arm,mips,ppc,sh3], hp[300,700], |             `-^--'`<     '
+ftp_reply [2001:470:4867:99::21]:21 - 0       |       hpc[arm,mips,sh], i386,        |            (_.)  _  )   /
+ftp_reply [2001:470:4867:99::21]:21 - 0       |      ibmnws, iyonix, luna68k,        |              .___/`    /
+ftp_reply [2001:470:4867:99::21]:21 - 0       |    mac[m68k,ppc], mipsco, mmeye,     |               `-----' /
+ftp_reply [2001:470:4867:99::21]:21 - 0       |      mvme[m68k,ppc], netwinders,     |  <----.     __ / __   \
+ftp_reply [2001:470:4867:99::21]:21 - 0       |   news[m68k,mips], next68k, ofppc,   |  <----|====O)))==) \) /====
+ftp_reply [2001:470:4867:99::21]:21 - 0       | playstation2, pmax, prep, sandpoint, |  <----'    `--' `.__,' \
+ftp_reply [2001:470:4867:99::21]:21 - 0       |  sbmips, sgimips, shark, sparc[,64], |               |        |
+ftp_reply [2001:470:4867:99::21]:21 - 0       |      sun[2,3], vax, x68k, xen        |                \       /
+ftp_reply [2001:470:4867:99::21]:21 - 0       +--------------------------------------+           ______( (_  / \_____
+ftp_reply [2001:470:4867:99::21]:21 - 0       See our website at http://www.NetBSD.org/        ,'  ,-----'   |       \
+ftp_reply [2001:470:4867:99::21]:21 - 0        We log all FTP transfers and commands.          `--{__________)  (FL) \/
+ftp_reply [2001:470:4867:99::21]:21 - 0 230-
+ftp_reply [2001:470:4867:99::21]:21 - 0     EXPORT NOTICE
+ftp_reply [2001:470:4867:99::21]:21 - 0     
+ftp_reply [2001:470:4867:99::21]:21 - 0     Please note that portions of this FTP site contain cryptographic
+ftp_reply [2001:470:4867:99::21]:21 - 0     software controlled under the Export Administration Regulations (EAR).
+ftp_reply [2001:470:4867:99::21]:21 - 0     
+ftp_reply [2001:470:4867:99::21]:21 - 0     None of this software may be downloaded or otherwise exported or
+ftp_reply [2001:470:4867:99::21]:21 - 0     re-exported into (or to a national or resident of) Cuba, Iran, Libya,
+ftp_reply [2001:470:4867:99::21]:21 - 0     Sudan, North Korea, Syria or any other country to which the U.S. has
+ftp_reply [2001:470:4867:99::21]:21 - 0     embargoed goods.
+ftp_reply [2001:470:4867:99::21]:21 - 0     
+ftp_reply [2001:470:4867:99::21]:21 - 0     By downloading or using said software, you are agreeing to the
+ftp_reply [2001:470:4867:99::21]:21 - 0     foregoing and you are representing and warranting that you are not
+ftp_reply [2001:470:4867:99::21]:21 - 0     located in, under the control of, or a national or resident of any
+ftp_reply [2001:470:4867:99::21]:21 - 0     such country or on any such list.
+ftp_reply [2001:470:4867:99::21]:21 - 230 Guest login ok, access restrictions apply.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - SYST 
+ftp_reply [2001:470:4867:99::21]:21 - 215 UNIX Type: L8 Version: NetBSD-ftpd 20100320
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - FEAT 
+ftp_reply [2001:470:4867:99::21]:21 - 211 Features supported
+ftp_reply [2001:470:4867:99::21]:21 - 0  MDTM
+ftp_reply [2001:470:4867:99::21]:21 - 0  MLST Type*;Size*;Modify*;Perm*;Unique*;
+ftp_reply [2001:470:4867:99::21]:21 - 0  REST STREAM
+ftp_reply [2001:470:4867:99::21]:21 - 0  SIZE
+ftp_reply [2001:470:4867:99::21]:21 - 0  TVFS
+ftp_reply [2001:470:4867:99::21]:21 - 211 End
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - PWD 
+ftp_reply [2001:470:4867:99::21]:21 - 257 "/" is the current directory.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - EPSV 
+ftp_reply [2001:470:4867:99::21]:21 - 229 Entering Extended Passive Mode (|||57086|)
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - LIST 
+ftp_reply [2001:470:4867:99::21]:21 - 150 Opening ASCII mode data connection for '/bin/ls'.
+ftp_reply [2001:470:4867:99::21]:21 - 226 Transfer complete.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - EPSV 
+ftp_reply [2001:470:4867:99::21]:21 - 229 Entering Extended Passive Mode (|||57087|)
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - NLST 
+ftp_reply [2001:470:4867:99::21]:21 - 150 Opening ASCII mode data connection for 'file list'.
+ftp_reply [2001:470:4867:99::21]:21 - 226 Transfer complete.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - TYPE I
+ftp_reply [2001:470:4867:99::21]:21 - 200 Type set to I.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - SIZE robots.txt
+ftp_reply [2001:470:4867:99::21]:21 - 213 77
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - EPSV 
+ftp_reply [2001:470:4867:99::21]:21 - 229 Entering Extended Passive Mode (|||57088|)
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - RETR robots.txt
+ftp_reply [2001:470:4867:99::21]:21 - 150 Opening BINARY mode data connection for 'robots.txt' (77 bytes).
+ftp_reply [2001:470:4867:99::21]:21 - 226 Transfer complete.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - MDTM robots.txt
+ftp_reply [2001:470:4867:99::21]:21 - 213 20090816112038
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - SIZE robots.txt
+ftp_reply [2001:470:4867:99::21]:21 - 213 77
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49189|
+ftp_reply [2001:470:4867:99::21]:21 - 200 EPRT command successful.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - RETR robots.txt
+ftp_reply [2001:470:4867:99::21]:21 - 150 Opening BINARY mode data connection for 'robots.txt' (77 bytes).
+ftp_reply [2001:470:4867:99::21]:21 - 226 Transfer complete.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - MDTM robots.txt
+ftp_reply [2001:470:4867:99::21]:21 - 213 20090816112038
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - TYPE A
+ftp_reply [2001:470:4867:99::21]:21 - 200 Type set to A.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - EPRT |2|2001:470:1f11:81f:c999:d94:aa7c:2e3e|49190|
+ftp_reply [2001:470:4867:99::21]:21 - 200 EPRT command successful.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - LIST 
+ftp_reply [2001:470:4867:99::21]:21 - 150 Opening ASCII mode data connection for '/bin/ls'.
+ftp_reply [2001:470:4867:99::21]:21 - 226 Transfer complete.
+ftp_request [2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185 - QUIT 
+ftp_reply [2001:470:4867:99::21]:21 - 221 
+ftp_reply [2001:470:4867:99::21]:21 - 0     Data traffic for this session was 154 bytes in 2 files.
+ftp_reply [2001:470:4867:99::21]:21 - 0     Total traffic for this session was 4512 bytes in 5 transfers.
+ftp_reply [2001:470:4867:99::21]:21 - 221 Thank you for using the FTP service on ftp.NetBSD.org.
diff --git a/testing/btest/Baseline/signatures.dpd/nosig-ipv4.out b/testing/btest/Baseline/signatures.dpd/nosig-ipv4.out
new file mode 100644
index 0000000000..55566505d8
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dpd/nosig-ipv4.out
@@ -0,0 +1,3 @@
+dpd_config, {
+
+}
diff --git a/testing/btest/Baseline/signatures.dpd/nosig-ipv6.out b/testing/btest/Baseline/signatures.dpd/nosig-ipv6.out
new file mode 100644
index 0000000000..55566505d8
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dpd/nosig-ipv6.out
@@ -0,0 +1,3 @@
+dpd_config, {
+
+}
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-list.out
new file mode 100644
index 0000000000..06d3c27188
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq.out
new file mode 100644
index 0000000000..8bad163eeb
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-eq
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list.out
new file mode 100644
index 0000000000..a1c0ea8927
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne.out
new file mode 100644
index 0000000000..8249781376
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4-masks/dst-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-ne
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-list.out
new file mode 100644
index 0000000000..06d3c27188
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq.out
new file mode 100644
index 0000000000..8bad163eeb
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-eq
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list.out
new file mode 100644
index 0000000000..a1c0ea8927
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne.out
new file mode 100644
index 0000000000..8249781376
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v4/dst-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - dst-ip-ne
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-list.out
new file mode 100644
index 0000000000..7396460f22
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq.out
new file mode 100644
index 0000000000..3241ccdf6f
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-eq
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list.out
new file mode 100644
index 0000000000..f875da226e
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne.out
new file mode 100644
index 0000000000..b074df8891
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6-masks/dst-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-ne
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-list.out
new file mode 100644
index 0000000000..7396460f22
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq.out
new file mode 100644
index 0000000000..3241ccdf6f
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-eq
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list.out
new file mode 100644
index 0000000000..f875da226e
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne.out b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne.out
new file mode 100644
index 0000000000..b074df8891
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-ip-header-condition-v6/dst-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-ip-ne
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-ip6.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-ip6.out
new file mode 100644
index 0000000000..db9d71f669
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-ip6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-eq
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-list.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-list.out
new file mode 100644
index 0000000000..0df42f6000
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - dst-port-eq-list
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq.out
new file mode 100644
index 0000000000..52321f7777
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - dst-port-eq
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt.out
new file mode 100644
index 0000000000..87c0c75514
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gt.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-gt
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte1.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte1.out
new file mode 100644
index 0000000000..a6eb48c84c
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte1.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-gte1
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte2.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte2.out
new file mode 100644
index 0000000000..2d13632cd6
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-gte2.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-gte2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt.out
new file mode 100644
index 0000000000..5d06777caf
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lt.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-lt
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte1.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte1.out
new file mode 100644
index 0000000000..4102fdfd9a
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte1.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-lte1
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte2.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte2.out
new file mode 100644
index 0000000000..b14823b92e
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-lte2.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-lte2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list.out
new file mode 100644
index 0000000000..7b68c06787
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-ne-list
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-nomatch.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne.out b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne.out
new file mode 100644
index 0000000000..c92dcb8b31
--- /dev/null
+++ b/testing/btest/Baseline/signatures.dst-port-header-condition/dst-port-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - dst-port-ne
diff --git a/testing/btest/Baseline/signatures.header-header-condition/icmp.out b/testing/btest/Baseline/signatures.header-header-condition/icmp.out
new file mode 100644
index 0000000000..a626bf85a5
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/icmp.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - icmp
diff --git a/testing/btest/Baseline/signatures.header-header-condition/icmp6.out b/testing/btest/Baseline/signatures.header-header-condition/icmp6.out
new file mode 100644
index 0000000000..61b7c927e9
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/icmp6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=128/icmp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=129/icmp] - icmp6
diff --git a/testing/btest/Baseline/signatures.header-header-condition/ip-mask.out b/testing/btest/Baseline/signatures.header-header-condition/ip-mask.out
new file mode 100644
index 0000000000..bc8045180f
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/ip-mask.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - ip-mask
diff --git a/testing/btest/Baseline/signatures.header-header-condition/ip.out b/testing/btest/Baseline/signatures.header-header-condition/ip.out
new file mode 100644
index 0000000000..5a7f51a6e3
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/ip.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - ip
diff --git a/testing/btest/Baseline/signatures.header-header-condition/ip6.out b/testing/btest/Baseline/signatures.header-header-condition/ip6.out
new file mode 100644
index 0000000000..d3d8aeae90
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/ip6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - ip6
diff --git a/testing/btest/Baseline/signatures.header-header-condition/tcp.out b/testing/btest/Baseline/signatures.header-header-condition/tcp.out
new file mode 100644
index 0000000000..48241068d4
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/tcp.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/tcp, resp_h=127.0.0.1, resp_p=80/tcp] - tcp
diff --git a/testing/btest/Baseline/signatures.header-header-condition/udp.out b/testing/btest/Baseline/signatures.header-header-condition/udp.out
new file mode 100644
index 0000000000..fd54308e9f
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/udp.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - udp
diff --git a/testing/btest/Baseline/signatures.header-header-condition/val-mask.out b/testing/btest/Baseline/signatures.header-header-condition/val-mask.out
new file mode 100644
index 0000000000..ad7a66e202
--- /dev/null
+++ b/testing/btest/Baseline/signatures.header-header-condition/val-mask.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - val-mask
diff --git a/testing/btest/Baseline/signatures.id-lookup/id.out b/testing/btest/Baseline/signatures.id-lookup/id.out
new file mode 100644
index 0000000000..4a5310a3b2
--- /dev/null
+++ b/testing/btest/Baseline/signatures.id-lookup/id.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - id
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/icmp6_in_ip6.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/icmp6_in_ip6.out
new file mode 100644
index 0000000000..61b7c927e9
--- /dev/null
+++ b/testing/btest/Baseline/signatures.ip-proto-header-condition/icmp6_in_ip6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=128/icmp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=129/icmp] - icmp6
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/icmp_in_ip4.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/icmp_in_ip4.out
new file mode 100644
index 0000000000..a626bf85a5
--- /dev/null
+++ b/testing/btest/Baseline/signatures.ip-proto-header-condition/icmp_in_ip4.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - icmp
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/nomatch.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip4.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip4.out
new file mode 100644
index 0000000000..48241068d4
--- /dev/null
+++ b/testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip4.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/tcp, resp_h=127.0.0.1, resp_p=80/tcp] - tcp
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip6.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip6.out
new file mode 100644
index 0000000000..8a5d5f17fc
--- /dev/null
+++ b/testing/btest/Baseline/signatures.ip-proto-header-condition/tcp_in_ip6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/tcp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=80/tcp] - tcp
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip4.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip4.out
new file mode 100644
index 0000000000..fd54308e9f
--- /dev/null
+++ b/testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip4.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - udp
diff --git a/testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip6.out b/testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip6.out
new file mode 100644
index 0000000000..f843e44d2d
--- /dev/null
+++ b/testing/btest/Baseline/signatures.ip-proto-header-condition/udp_in_ip6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - udp
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-list.out
new file mode 100644
index 0000000000..60fa5de636
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq.out
new file mode 100644
index 0000000000..ce46d4b3df
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-eq
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list.out
new file mode 100644
index 0000000000..3ca3aab914
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne.out
new file mode 100644
index 0000000000..c0876257e3
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4-masks/src-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-ne
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-list.out
new file mode 100644
index 0000000000..60fa5de636
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq.out
new file mode 100644
index 0000000000..ce46d4b3df
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-eq
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list.out
new file mode 100644
index 0000000000..3ca3aab914
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne.out
new file mode 100644
index 0000000000..c0876257e3
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v4/src-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=192.168.1.100, orig_p=8/icmp, resp_h=192.168.1.101, resp_p=0/icmp] - src-ip-ne
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-list.out
new file mode 100644
index 0000000000..15e7b9848c
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq.out
new file mode 100644
index 0000000000..12b0192a28
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-eq
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list.out
new file mode 100644
index 0000000000..2e10e62cec
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne.out
new file mode 100644
index 0000000000..be5325c4e9
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6-masks/src-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-ne
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-list.out
new file mode 100644
index 0000000000..15e7b9848c
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-eq-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq.out
new file mode 100644
index 0000000000..12b0192a28
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-eq
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list.out
new file mode 100644
index 0000000000..2e10e62cec
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-ne-list
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-nomatch.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne.out b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne.out
new file mode 100644
index 0000000000..be5325c4e9
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-ip-header-condition-v6/src-ip-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-ip-ne
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-ip6.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-ip6.out
new file mode 100644
index 0000000000..9a16e2d533
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-ip6.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-eq
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-list.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-list.out
new file mode 100644
index 0000000000..c8a6579af1
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - src-port-eq-list
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq.out
new file mode 100644
index 0000000000..8e44853a14
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-eq.out
@@ -0,0 +1 @@
+signature_match [orig_h=127.0.0.1, orig_p=30000/udp, resp_h=127.0.0.1, resp_p=13000/udp] - src-port-eq
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt.out
new file mode 100644
index 0000000000..235b9a0f11
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gt.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-gt
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte1.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte1.out
new file mode 100644
index 0000000000..82b1a39aab
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte1.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-gte1
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte2.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte2.out
new file mode 100644
index 0000000000..4816fe1947
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-gte2.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-gte2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt.out
new file mode 100644
index 0000000000..b124a1616d
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lt.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-lt
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte1.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte1.out
new file mode 100644
index 0000000000..67b2665619
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte1.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-lte1
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte2.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte2.out
new file mode 100644
index 0000000000..758b5f1241
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-lte2.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-lte2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list.out
new file mode 100644
index 0000000000..c98df730a8
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-list.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-ne-list
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-nomatch.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne-nomatch.out
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne.out b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne.out
new file mode 100644
index 0000000000..f2ec15a667
--- /dev/null
+++ b/testing/btest/Baseline/signatures.src-port-header-condition/src-port-ne.out
@@ -0,0 +1 @@
+signature_match [orig_h=2001:4f8:4:7:2e0:81ff:fe52:ffff, orig_p=30000/udp, resp_h=2001:4f8:4:7:2e0:81ff:fe52:9a6b, resp_p=13000/udp] - src-port-ne
diff --git a/testing/btest/signatures/dpd.bro b/testing/btest/signatures/dpd.bro
new file mode 100644
index 0000000000..d6ae02cb50
--- /dev/null
+++ b/testing/btest/signatures/dpd.bro
@@ -0,0 +1,54 @@
+# @TEST-EXEC: bro -b -s myftp -r $TRACES/ftp-ipv4.trace %INPUT >dpd-ipv4.out
+# @TEST-EXEC: bro -b -s myftp -r $TRACES/ipv6-ftp.trace %INPUT >dpd-ipv6.out
+# @TEST-EXEC: bro -b -r $TRACES/ftp-ipv4.trace %INPUT >nosig-ipv4.out
+# @TEST-EXEC: bro -b -r $TRACES/ipv6-ftp.trace %INPUT >nosig-ipv6.out
+# @TEST-EXEC: btest-diff dpd-ipv4.out
+# @TEST-EXEC: btest-diff dpd-ipv6.out
+# @TEST-EXEC: btest-diff nosig-ipv4.out
+# @TEST-EXEC: btest-diff nosig-ipv6.out
+
+# DPD based on 'ip-proto' and 'payload' signatures should be independent
+# of IP protocol.
+
+@TEST-START-FILE myftp.sig
+signature my_ftp_client {
+  ip-proto == tcp
+  payload /(|.*[\n\r]) *[uU][sS][eE][rR] /
+  tcp-state originator
+  event "matched my_ftp_client"
+}
+
+signature my_ftp_server {
+  ip-proto == tcp
+  payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/
+  tcp-state responder
+  requires-reverse-signature my_ftp_client
+  enable "ftp"
+  event "matched my_ftp_server"
+}
+@TEST-END-FILE
+
+@load base/utils/addrs
+
+event bro_init()
+	{
+	# no analyzer attached to any port by default, depends entirely on sigs
+	print "dpd_config", dpd_config;
+	}
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
+
+event ftp_request(c: connection, command: string, arg: string)
+	{
+	print fmt("ftp_request %s:%s - %s %s", addr_to_uri(c$id$orig_h),
+	          port_to_count(c$id$orig_p), command, arg);
+	}
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	print fmt("ftp_reply %s:%s - %s %s", addr_to_uri(c$id$resp_h),
+	          port_to_count(c$id$resp_p), code, msg);
+	}
diff --git a/testing/btest/signatures/dst-ip-header-condition-v4-masks.bro b/testing/btest/signatures/dst-ip-header-condition-v4-masks.bro
new file mode 100644
index 0000000000..dc5b0f48b8
--- /dev/null
+++ b/testing/btest/signatures/dst-ip-header-condition-v4-masks.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s dst-ip-eq -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-eq.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s dst-ip-ne -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne-list.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff dst-ip-eq.out
+# @TEST-EXEC: btest-diff dst-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff dst-ip-ne.out
+# @TEST-EXEC: btest-diff dst-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list-nomatch.out
+
+@TEST-START-FILE dst-ip-eq.sig
+signature id {
+  dst-ip == 192.168.1.0/24
+  event "dst-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-nomatch.sig
+signature id {
+  dst-ip == 10.0.0.0/8
+  event "dst-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-list.sig
+signature id {
+  dst-ip == 10.0.0.0/8,[fe80::0]/16,192.168.1.0/24
+  event "dst-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne.sig
+signature id {
+  dst-ip != 10.0.0.0/8
+  event "dst-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-nomatch.sig
+signature id {
+  dst-ip != 192.168.1.0/24
+  event "dst-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list.sig
+signature id {
+  dst-ip != 10.0.0.0/8,[fe80::0]/16
+  event "dst-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list-nomatch.sig
+signature id {
+  dst-ip != 10.0.0.0/8,[fe80::0]/16,192.168.1.0/24
+  event "dst-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/dst-ip-header-condition-v4.bro b/testing/btest/signatures/dst-ip-header-condition-v4.bro
new file mode 100644
index 0000000000..0d0d3e644c
--- /dev/null
+++ b/testing/btest/signatures/dst-ip-header-condition-v4.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s dst-ip-eq -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-eq.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s dst-ip-ne -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne-list.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >dst-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff dst-ip-eq.out
+# @TEST-EXEC: btest-diff dst-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff dst-ip-ne.out
+# @TEST-EXEC: btest-diff dst-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list-nomatch.out
+
+@TEST-START-FILE dst-ip-eq.sig
+signature id {
+  dst-ip == 192.168.1.101
+  event "dst-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-nomatch.sig
+signature id {
+  dst-ip == 10.0.0.1
+  event "dst-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-list.sig
+signature id {
+  dst-ip == 10.0.0.1,10.0.0.2,[fe80::1],192.168.1.101
+  event "dst-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne.sig
+signature id {
+  dst-ip != 10.0.0.1
+  event "dst-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-nomatch.sig
+signature id {
+  dst-ip != 192.168.1.101
+  event "dst-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list.sig
+signature id {
+  dst-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1]
+  event "dst-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list-nomatch.sig
+signature id {
+  dst-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1],192.168.1.101
+  event "dst-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/dst-ip-header-condition-v6-masks.bro b/testing/btest/signatures/dst-ip-header-condition-v6-masks.bro
new file mode 100644
index 0000000000..d82a76e78d
--- /dev/null
+++ b/testing/btest/signatures/dst-ip-header-condition-v6-masks.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s dst-ip-eq -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-eq.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s dst-ip-ne -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne-list.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff dst-ip-eq.out
+# @TEST-EXEC: btest-diff dst-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff dst-ip-ne.out
+# @TEST-EXEC: btest-diff dst-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list-nomatch.out
+
+@TEST-START-FILE dst-ip-eq.sig
+signature id {
+  dst-ip == [2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "dst-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-nomatch.sig
+signature id {
+  dst-ip == [fe80::0]/16
+  event "dst-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-list.sig
+signature id {
+  dst-ip == 10.0.0.0/8,[fe80::0]/16,[2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "dst-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne.sig
+signature id {
+  dst-ip != [2001:4f8:4:7:2e0:81ff:fe52:0]/120
+  event "dst-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-nomatch.sig
+signature id {
+  dst-ip != [2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "dst-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list.sig
+signature id {
+  dst-ip != 10.0.0.0/8,[fe80::0]/16
+  event "dst-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list-nomatch.sig
+signature id {
+  dst-ip != 10.0.0.0/8,[fe80::1]/16,[2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "dst-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/dst-ip-header-condition-v6.bro b/testing/btest/signatures/dst-ip-header-condition-v6.bro
new file mode 100644
index 0000000000..e629fb4462
--- /dev/null
+++ b/testing/btest/signatures/dst-ip-header-condition-v6.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s dst-ip-eq -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-eq.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-eq-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s dst-ip-ne -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne-list.out
+# @TEST-EXEC: bro -b -s dst-ip-ne-list-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff dst-ip-eq.out
+# @TEST-EXEC: btest-diff dst-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff dst-ip-ne.out
+# @TEST-EXEC: btest-diff dst-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list.out
+# @TEST-EXEC: btest-diff dst-ip-ne-list-nomatch.out
+
+@TEST-START-FILE dst-ip-eq.sig
+signature id {
+  dst-ip == [2001:4f8:4:7:2e0:81ff:fe52:9a6b]
+  event "dst-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-nomatch.sig
+signature id {
+  dst-ip == 10.0.0.1
+  event "dst-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-eq-list.sig
+signature id {
+  dst-ip == 10.0.0.1,10.0.0.2,[fe80::1],[2001:4f8:4:7:2e0:81ff:fe52:9a6b]
+  event "dst-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne.sig
+signature id {
+  dst-ip != 10.0.0.1
+  event "dst-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-nomatch.sig
+signature id {
+  dst-ip != [2001:4f8:4:7:2e0:81ff:fe52:9a6b]
+  event "dst-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list.sig
+signature id {
+  dst-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1]
+  event "dst-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-ip-ne-list-nomatch.sig
+signature id {
+  dst-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1],[2001:4f8:4:7:2e0:81ff:fe52:9a6b]
+  event "dst-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/dst-port-header-condition.bro b/testing/btest/signatures/dst-port-header-condition.bro
new file mode 100644
index 0000000000..08ba07b0de
--- /dev/null
+++ b/testing/btest/signatures/dst-port-header-condition.bro
@@ -0,0 +1,164 @@
+# @TEST-EXEC: bro -b -s dst-port-eq -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >dst-port-eq.out
+# @TEST-EXEC: bro -b -s dst-port-eq-nomatch -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >dst-port-eq-nomatch.out
+# @TEST-EXEC: bro -b -s dst-port-eq-list -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >dst-port-eq-list.out
+# @TEST-EXEC: bro -b -s dst-port-eq -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-eq-ip6.out
+
+# @TEST-EXEC: bro -b -s dst-port-ne -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-ne.out
+# @TEST-EXEC: bro -b -s dst-port-ne-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-ne-nomatch.out
+# @TEST-EXEC: bro -b -s dst-port-ne-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-ne-list.out
+# @TEST-EXEC: bro -b -s dst-port-ne-list-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-ne-list-nomatch.out
+
+# @TEST-EXEC: bro -b -s dst-port-lt -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-lt.out
+# @TEST-EXEC: bro -b -s dst-port-lt-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-lt-nomatch.out
+# @TEST-EXEC: bro -b -s dst-port-lte1 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-lte1.out
+# @TEST-EXEC: bro -b -s dst-port-lte2 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-lte2.out
+# @TEST-EXEC: bro -b -s dst-port-lte-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-lte-nomatch.out
+
+# @TEST-EXEC: bro -b -s dst-port-gt -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-gt.out
+# @TEST-EXEC: bro -b -s dst-port-gt-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-gt-nomatch.out
+# @TEST-EXEC: bro -b -s dst-port-gte1 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-gte1.out
+# @TEST-EXEC: bro -b -s dst-port-gte2 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-gte2.out
+# @TEST-EXEC: bro -b -s dst-port-gte-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >dst-port-gte-nomatch.out
+
+# @TEST-EXEC: btest-diff dst-port-eq.out
+# @TEST-EXEC: btest-diff dst-port-eq-nomatch.out
+# @TEST-EXEC: btest-diff dst-port-eq-list.out
+# @TEST-EXEC: btest-diff dst-port-eq-ip6.out
+# @TEST-EXEC: btest-diff dst-port-ne.out
+# @TEST-EXEC: btest-diff dst-port-ne-nomatch.out
+# @TEST-EXEC: btest-diff dst-port-ne-list.out
+# @TEST-EXEC: btest-diff dst-port-ne-list-nomatch.out
+# @TEST-EXEC: btest-diff dst-port-lt.out
+# @TEST-EXEC: btest-diff dst-port-lt-nomatch.out
+# @TEST-EXEC: btest-diff dst-port-lte1.out
+# @TEST-EXEC: btest-diff dst-port-lte2.out
+# @TEST-EXEC: btest-diff dst-port-lte-nomatch.out
+# @TEST-EXEC: btest-diff dst-port-gt.out
+# @TEST-EXEC: btest-diff dst-port-gt-nomatch.out
+# @TEST-EXEC: btest-diff dst-port-gte1.out
+# @TEST-EXEC: btest-diff dst-port-gte2.out
+# @TEST-EXEC: btest-diff dst-port-gte-nomatch.out
+
+@TEST-START-FILE dst-port-eq.sig
+signature id {
+  dst-port == 13000
+  event "dst-port-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-eq-nomatch.sig
+signature id {
+  dst-port == 22
+  event "dst-port-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-eq-list.sig
+signature id {
+  dst-port == 22,23,24,13000
+  event "dst-port-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-ne.sig
+signature id {
+  dst-port != 22
+  event "dst-port-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-ne-nomatch.sig
+signature id {
+  dst-port != 13000
+  event "dst-port-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-ne-list.sig
+signature id {
+  dst-port != 22,23,24,25
+  event "dst-port-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-ne-list-nomatch.sig
+signature id {
+  dst-port != 22,23,24,25,13000
+  event "dst-port-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-lt.sig
+signature id {
+  dst-port < 13001
+  event "dst-port-lt"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-lt-nomatch.sig
+signature id {
+  dst-port < 13000
+  event "dst-port-lt-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-lte1.sig
+signature id {
+  dst-port <= 13000
+  event "dst-port-lte1"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-lte2.sig
+signature id {
+  dst-port <= 13001
+  event "dst-port-lte2"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-lte-nomatch.sig
+signature id {
+  dst-port <= 12999
+  event "dst-port-lte-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-gt.sig
+signature id {
+  dst-port > 12999
+  event "dst-port-gt"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-gt-nomatch.sig
+signature id {
+  dst-port > 13000
+  event "dst-port-gt-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-gte1.sig
+signature id {
+  dst-port >= 13000
+  event "dst-port-gte1"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-gte2.sig
+signature id {
+  dst-port >= 12999
+  event "dst-port-gte2"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE dst-port-gte-nomatch.sig
+signature id {
+  dst-port >= 13001
+  event "dst-port-gte-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/header-header-condition.bro b/testing/btest/signatures/header-header-condition.bro
new file mode 100644
index 0000000000..ad78ba4513
--- /dev/null
+++ b/testing/btest/signatures/header-header-condition.bro
@@ -0,0 +1,78 @@
+# @TEST-EXEC: bro -b -s ip -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >ip.out
+# @TEST-EXEC: bro -b -s ip-mask -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >ip-mask.out
+# @TEST-EXEC: bro -b -s ip6 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >ip6.out
+# @TEST-EXEC: bro -b -s udp -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >udp.out
+# @TEST-EXEC: bro -b -s tcp -r $TRACES/chksums/ip4-tcp-good-chksum.pcap %INPUT >tcp.out
+# @TEST-EXEC: bro -b -s icmp -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >icmp.out
+# @TEST-EXEC: bro -b -s icmp6 -r $TRACES/chksums/ip6-icmp6-good-chksum.pcap %INPUT >icmp6.out
+# @TEST-EXEC: bro -b -s val-mask -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >val-mask.out
+
+# @TEST-EXEC: btest-diff ip.out
+# @TEST-EXEC: btest-diff ip-mask.out
+# @TEST-EXEC: btest-diff ip6.out
+# @TEST-EXEC: btest-diff udp.out
+# @TEST-EXEC: btest-diff tcp.out
+# @TEST-EXEC: btest-diff icmp.out
+# @TEST-EXEC: btest-diff icmp6.out
+# @TEST-EXEC: btest-diff val-mask.out
+
+@TEST-START-FILE ip.sig
+signature id {
+  header ip[10:1] == 0x7c
+  event "ip"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE ip-mask.sig
+signature id {
+  header ip[16:4] == 127.0.0.0/24
+  event "ip-mask"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE ip6.sig
+signature id {
+  header ip6[10:1] == 0x04
+  event "ip6"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE udp.sig
+signature id {
+  header udp[2:1] == 0x32
+  event "udp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE tcp.sig
+signature id {
+  header tcp[3:4] == 0x50000000
+  event "tcp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE icmp.sig
+signature id {
+  header icmp[2:2] == 0xf7ff
+  event "icmp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE icmp6.sig
+signature id {
+  header icmp6[0:1] == 0x80
+  event "icmp6"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE val-mask.sig
+signature id {
+  header udp[2:1] & 0x0f == 0x02
+  event "val-mask"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/id-lookup.bro b/testing/btest/signatures/id-lookup.bro
new file mode 100644
index 0000000000..2e32224bc8
--- /dev/null
+++ b/testing/btest/signatures/id-lookup.bro
@@ -0,0 +1,16 @@
+# @TEST-EXEC: bro -b -s id -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >id.out
+# @TEST-EXEC: btest-diff id.out
+
+@TEST-START-FILE id.sig
+signature id {
+  ip-proto == udp_proto_number
+  event "id"
+}
+@TEST-END-FILE
+
+const udp_proto_number = 17;
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/ip-proto-header-condition.bro b/testing/btest/signatures/ip-proto-header-condition.bro
new file mode 100644
index 0000000000..52d58ea223
--- /dev/null
+++ b/testing/btest/signatures/ip-proto-header-condition.bro
@@ -0,0 +1,48 @@
+# @TEST-EXEC: bro -b -s tcp -r $TRACES/chksums/ip4-tcp-good-chksum.pcap %INPUT >tcp_in_ip4.out
+# @TEST-EXEC: bro -b -s udp -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >udp_in_ip4.out
+# @TEST-EXEC: bro -b -s icmp -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >icmp_in_ip4.out
+# @TEST-EXEC: bro -b -s tcp -r $TRACES/chksums/ip6-tcp-good-chksum.pcap %INPUT >tcp_in_ip6.out
+# @TEST-EXEC: bro -b -s udp -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >udp_in_ip6.out
+# @TEST-EXEC: bro -b -s icmp6 -r $TRACES/chksums/ip6-icmp6-good-chksum.pcap %INPUT >icmp6_in_ip6.out
+# @TEST-EXEC: bro -b -s icmp -r $TRACES/chksums/ip6-icmp6-good-chksum.pcap %INPUT >nomatch.out
+
+# @TEST-EXEC: btest-diff tcp_in_ip4.out
+# @TEST-EXEC: btest-diff udp_in_ip4.out
+# @TEST-EXEC: btest-diff icmp_in_ip4.out
+# @TEST-EXEC: btest-diff tcp_in_ip6.out
+# @TEST-EXEC: btest-diff udp_in_ip6.out
+# @TEST-EXEC: btest-diff icmp6_in_ip6.out
+# @TEST-EXEC: btest-diff nomatch.out
+
+@TEST-START-FILE tcp.sig
+signature tcp_transport {
+  ip-proto == tcp
+  event "tcp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE udp.sig
+signature udp_transport {
+  ip-proto == udp
+  event "udp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE icmp.sig
+signature icmp_transport {
+  ip-proto == icmp
+  event "icmp"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE icmp6.sig
+signature icmp6_transport {
+  ip-proto == icmp6
+  event "icmp6"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/src-ip-header-condition-v4-masks.bro b/testing/btest/signatures/src-ip-header-condition-v4-masks.bro
new file mode 100644
index 0000000000..1e272c81ee
--- /dev/null
+++ b/testing/btest/signatures/src-ip-header-condition-v4-masks.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s src-ip-eq -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-eq.out
+# @TEST-EXEC: bro -b -s src-ip-eq-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-eq-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s src-ip-ne -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne.out
+# @TEST-EXEC: bro -b -s src-ip-ne-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne-list.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff src-ip-eq.out
+# @TEST-EXEC: btest-diff src-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff src-ip-ne.out
+# @TEST-EXEC: btest-diff src-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-ne-list.out
+# @TEST-EXEC: btest-diff src-ip-ne-list-nomatch.out
+
+@TEST-START-FILE src-ip-eq.sig
+signature id {
+  src-ip == 192.168.1.0/24
+  event "src-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-nomatch.sig
+signature id {
+  src-ip == 10.0.0.0/8
+  event "src-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-list.sig
+signature id {
+  src-ip == 10.0.0.0/8,[fe80::0]/16,192.168.1.0/24
+  event "src-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne.sig
+signature id {
+  src-ip != 10.0.0.0/8
+  event "src-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-nomatch.sig
+signature id {
+  src-ip != 192.168.1.0/24
+  event "src-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list.sig
+signature id {
+  src-ip != 10.0.0.0/8,[fe80::0]/16
+  event "src-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list-nomatch.sig
+signature id {
+  src-ip != 10.0.0.0/8,[fe80::0]/16,192.168.1.0/24
+  event "src-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/src-ip-header-condition-v4.bro b/testing/btest/signatures/src-ip-header-condition-v4.bro
new file mode 100644
index 0000000000..746e41a4be
--- /dev/null
+++ b/testing/btest/signatures/src-ip-header-condition-v4.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s src-ip-eq -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-eq.out
+# @TEST-EXEC: bro -b -s src-ip-eq-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-eq-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s src-ip-ne -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne.out
+# @TEST-EXEC: bro -b -s src-ip-ne-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne-list.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list-nomatch -r $TRACES/chksums/ip4-icmp-good-chksum.pcap %INPUT >src-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff src-ip-eq.out
+# @TEST-EXEC: btest-diff src-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff src-ip-ne.out
+# @TEST-EXEC: btest-diff src-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-ne-list.out
+# @TEST-EXEC: btest-diff src-ip-ne-list-nomatch.out
+
+@TEST-START-FILE src-ip-eq.sig
+signature id {
+  src-ip == 192.168.1.100
+  event "src-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-nomatch.sig
+signature id {
+  src-ip == 10.0.0.1
+  event "src-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-list.sig
+signature id {
+  src-ip == 10.0.0.1,10.0.0.2,[fe80::1],192.168.1.100
+  event "src-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne.sig
+signature id {
+  src-ip != 10.0.0.1
+  event "src-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-nomatch.sig
+signature id {
+  src-ip != 192.168.1.100
+  event "src-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list.sig
+signature id {
+  src-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1]
+  event "src-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list-nomatch.sig
+signature id {
+  src-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1],192.168.1.100
+  event "src-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/src-ip-header-condition-v6-masks.bro b/testing/btest/signatures/src-ip-header-condition-v6-masks.bro
new file mode 100644
index 0000000000..3c4fbf5526
--- /dev/null
+++ b/testing/btest/signatures/src-ip-header-condition-v6-masks.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s src-ip-eq -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-eq.out
+# @TEST-EXEC: bro -b -s src-ip-eq-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-eq-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s src-ip-ne -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne.out
+# @TEST-EXEC: bro -b -s src-ip-ne-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne-list.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff src-ip-eq.out
+# @TEST-EXEC: btest-diff src-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff src-ip-ne.out
+# @TEST-EXEC: btest-diff src-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-ne-list.out
+# @TEST-EXEC: btest-diff src-ip-ne-list-nomatch.out
+
+@TEST-START-FILE src-ip-eq.sig
+signature id {
+  src-ip == [2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "src-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-nomatch.sig
+signature id {
+  src-ip == [fe80::0]/16
+  event "src-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-list.sig
+signature id {
+  src-ip == 10.0.0.0/8,[fe80::0]/16,[2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "src-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne.sig
+signature id {
+  src-ip != [2001:4f8:4:7:2e0:81ff:fe52:0]/120
+  event "src-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-nomatch.sig
+signature id {
+  src-ip != [2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "src-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list.sig
+signature id {
+  src-ip != 10.0.0.0/8,[fe80::0]/16
+  event "src-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list-nomatch.sig
+signature id {
+  src-ip != 10.0.0.0/8,[fe80::1]/16,[2001:4f8:4:7:2e0:81ff:fe52:0]/112
+  event "src-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/src-ip-header-condition-v6.bro b/testing/btest/signatures/src-ip-header-condition-v6.bro
new file mode 100644
index 0000000000..613a3dd4c1
--- /dev/null
+++ b/testing/btest/signatures/src-ip-header-condition-v6.bro
@@ -0,0 +1,71 @@
+# @TEST-EXEC: bro -b -s src-ip-eq -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-eq.out
+# @TEST-EXEC: bro -b -s src-ip-eq-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-eq-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-eq-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-eq-list.out
+
+# @TEST-EXEC: bro -b -s src-ip-ne -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne.out
+# @TEST-EXEC: bro -b -s src-ip-ne-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne-nomatch.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne-list.out
+# @TEST-EXEC: bro -b -s src-ip-ne-list-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-ip-ne-list-nomatch.out
+
+# @TEST-EXEC: btest-diff src-ip-eq.out
+# @TEST-EXEC: btest-diff src-ip-eq-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-eq-list.out
+
+# @TEST-EXEC: btest-diff src-ip-ne.out
+# @TEST-EXEC: btest-diff src-ip-ne-nomatch.out
+# @TEST-EXEC: btest-diff src-ip-ne-list.out
+# @TEST-EXEC: btest-diff src-ip-ne-list-nomatch.out
+
+@TEST-START-FILE src-ip-eq.sig
+signature id {
+  src-ip == [2001:4f8:4:7:2e0:81ff:fe52:ffff]
+  event "src-ip-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-nomatch.sig
+signature id {
+  src-ip == 10.0.0.1
+  event "src-ip-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-eq-list.sig
+signature id {
+  src-ip == 10.0.0.1,10.0.0.2,[fe80::1],[2001:4f8:4:7:2e0:81ff:fe52:ffff]
+  event "src-ip-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne.sig
+signature id {
+  src-ip != 10.0.0.1
+  event "src-ip-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-nomatch.sig
+signature id {
+  src-ip != [2001:4f8:4:7:2e0:81ff:fe52:ffff]
+  event "src-ip-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list.sig
+signature id {
+  src-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1]
+  event "src-ip-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-ip-ne-list-nomatch.sig
+signature id {
+  src-ip != 10.0.0.1,10.0.0.2,10.0.0.3,[fe80::1],[2001:4f8:4:7:2e0:81ff:fe52:ffff]
+  event "src-ip-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}
diff --git a/testing/btest/signatures/src-port-header-condition.bro b/testing/btest/signatures/src-port-header-condition.bro
new file mode 100644
index 0000000000..ea9e08ce2b
--- /dev/null
+++ b/testing/btest/signatures/src-port-header-condition.bro
@@ -0,0 +1,164 @@
+# @TEST-EXEC: bro -b -s src-port-eq -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >src-port-eq.out
+# @TEST-EXEC: bro -b -s src-port-eq-nomatch -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >src-port-eq-nomatch.out
+# @TEST-EXEC: bro -b -s src-port-eq-list -r $TRACES/chksums/ip4-udp-good-chksum.pcap %INPUT >src-port-eq-list.out
+# @TEST-EXEC: bro -b -s src-port-eq -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-eq-ip6.out
+
+# @TEST-EXEC: bro -b -s src-port-ne -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-ne.out
+# @TEST-EXEC: bro -b -s src-port-ne-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-ne-nomatch.out
+# @TEST-EXEC: bro -b -s src-port-ne-list -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-ne-list.out
+# @TEST-EXEC: bro -b -s src-port-ne-list-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-ne-list-nomatch.out
+
+# @TEST-EXEC: bro -b -s src-port-lt -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-lt.out
+# @TEST-EXEC: bro -b -s src-port-lt-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-lt-nomatch.out
+# @TEST-EXEC: bro -b -s src-port-lte1 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-lte1.out
+# @TEST-EXEC: bro -b -s src-port-lte2 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-lte2.out
+# @TEST-EXEC: bro -b -s src-port-lte-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-lte-nomatch.out
+
+# @TEST-EXEC: bro -b -s src-port-gt -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-gt.out
+# @TEST-EXEC: bro -b -s src-port-gt-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-gt-nomatch.out
+# @TEST-EXEC: bro -b -s src-port-gte1 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-gte1.out
+# @TEST-EXEC: bro -b -s src-port-gte2 -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-gte2.out
+# @TEST-EXEC: bro -b -s src-port-gte-nomatch -r $TRACES/chksums/ip6-udp-good-chksum.pcap %INPUT >src-port-gte-nomatch.out
+
+# @TEST-EXEC: btest-diff src-port-eq.out
+# @TEST-EXEC: btest-diff src-port-eq-nomatch.out
+# @TEST-EXEC: btest-diff src-port-eq-list.out
+# @TEST-EXEC: btest-diff src-port-eq-ip6.out
+# @TEST-EXEC: btest-diff src-port-ne.out
+# @TEST-EXEC: btest-diff src-port-ne-nomatch.out
+# @TEST-EXEC: btest-diff src-port-ne-list.out
+# @TEST-EXEC: btest-diff src-port-ne-list-nomatch.out
+# @TEST-EXEC: btest-diff src-port-lt.out
+# @TEST-EXEC: btest-diff src-port-lt-nomatch.out
+# @TEST-EXEC: btest-diff src-port-lte1.out
+# @TEST-EXEC: btest-diff src-port-lte2.out
+# @TEST-EXEC: btest-diff src-port-lte-nomatch.out
+# @TEST-EXEC: btest-diff src-port-gt.out
+# @TEST-EXEC: btest-diff src-port-gt-nomatch.out
+# @TEST-EXEC: btest-diff src-port-gte1.out
+# @TEST-EXEC: btest-diff src-port-gte2.out
+# @TEST-EXEC: btest-diff src-port-gte-nomatch.out
+
+@TEST-START-FILE src-port-eq.sig
+signature id {
+  src-port == 30000
+  event "src-port-eq"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-eq-nomatch.sig
+signature id {
+  src-port == 22
+  event "src-port-eq-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-eq-list.sig
+signature id {
+  src-port == 22,23,24,30000
+  event "src-port-eq-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-ne.sig
+signature id {
+  src-port != 22
+  event "src-port-ne"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-ne-nomatch.sig
+signature id {
+  src-port != 30000
+  event "src-port-ne-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-ne-list.sig
+signature id {
+  src-port != 22,23,24,25
+  event "src-port-ne-list"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-ne-list-nomatch.sig
+signature id {
+  src-port != 22,23,24,25,30000
+  event "src-port-ne-list-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-lt.sig
+signature id {
+  src-port < 30001
+  event "src-port-lt"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-lt-nomatch.sig
+signature id {
+  src-port < 30000
+  event "src-port-lt-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-lte1.sig
+signature id {
+  src-port <= 30000
+  event "src-port-lte1"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-lte2.sig
+signature id {
+  src-port <= 30001
+  event "src-port-lte2"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-lte-nomatch.sig
+signature id {
+  src-port <= 29999
+  event "src-port-lte-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-gt.sig
+signature id {
+  src-port > 29999
+  event "src-port-gt"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-gt-nomatch.sig
+signature id {
+  src-port > 30000
+  event "src-port-gt-nomatch"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-gte1.sig
+signature id {
+  src-port >= 30000
+  event "src-port-gte1"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-gte2.sig
+signature id {
+  src-port >= 29999
+  event "src-port-gte2"
+}
+@TEST-END-FILE
+
+@TEST-START-FILE src-port-gte-nomatch.sig
+signature id {
+  src-port >= 30001
+  event "src-port-gte-nomatch"
+}
+@TEST-END-FILE
+
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	print fmt("signature_match %s - %s", state$conn$id, msg);
+	}

From 18f84275793e3eb50f0fb23c9978621cddac4117 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 17 Oct 2012 12:09:12 -0500
Subject: [PATCH 234/238] Change how "gridftp" gets added to service field of
 connection records.

In addition to checking for a finished SSL handshake over an FTP
connection, it now also requires that the SSL handshake occurs after
the FTP client requested AUTH GSSAPI, more specifically identifying the
characteristics of GridFTP control channels.

Addresses #891.
---
 scripts/base/protocols/ftp/gridftp.bro | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/scripts/base/protocols/ftp/gridftp.bro b/scripts/base/protocols/ftp/gridftp.bro
index 7413fa24c6..57752b1cbd 100644
--- a/scripts/base/protocols/ftp/gridftp.bro
+++ b/scripts/base/protocols/ftp/gridftp.bro
@@ -68,6 +68,16 @@ export {
 	const data_channel_initial_criteria: function(c: connection): bool &redef;
 }
 
+redef record FTP::Info += {
+	last_auth_requested: string &optional;
+};
+
+event ftp_request(c: connection, command: string, arg: string) &priority=4
+	{
+	if ( command == "AUTH" && c?$ftp )
+		c$ftp$last_auth_requested = arg;
+	}
+
 function size_callback(c: connection, cnt: count): interval
 	{
 	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
@@ -89,8 +99,10 @@ function size_callback(c: connection, cnt: count): interval
 
 event ssl_established(c: connection) &priority=5
 	{
-	# Add service label to control channels.
-	if ( "FTP" in c$service )
+	# If an FTP client requests AUTH GSSAPI and later an SSL handshake
+	# finishes, it's likely a GridFTP control channel, so add service label.
+	if ( c?$ftp && c$ftp?$last_auth_requested &&
+	     /GSSAPI/ in c$ftp$last_auth_requested )
 		add c$service["gridftp"];
 	}
 

From d157759ff2f7741d116ebbb3add637ef6bce5163 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Fri, 19 Oct 2012 02:07:34 -0400
Subject: [PATCH 235/238] Added a BiF to wrap the strptime function.

---
 src/bro.bif                                   | 25 +++++++++++++++++++
 testing/btest/Baseline/bifs.strptime/.stdout  |  2 ++
 .../btest/Baseline/bifs.strptime/reporter.log | 10 ++++++++
 testing/btest/bifs/strptime.bro               | 10 ++++++++
 4 files changed, 47 insertions(+)
 create mode 100644 testing/btest/Baseline/bifs.strptime/.stdout
 create mode 100644 testing/btest/Baseline/bifs.strptime/reporter.log
 create mode 100644 testing/btest/bifs/strptime.bro

diff --git a/src/bro.bif b/src/bro.bif
index 8ddde6ef86..d59ae36b28 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -11,6 +11,7 @@
 #include <cmath>
 #include <sys/stat.h>
 #include <cstdio>
+#include <time.h>
 
 #include "digest.h"
 #include "Reporter.h"
@@ -3285,6 +3286,30 @@ function strftime%(fmt: string, d: time%) : string
 	return new StringVal(buffer);
 	%}
 
+
+## Parse a textual representation of a date/time value into a ``time`` type value.
+##
+## fmt: The format string used to parse the following *d* argument. See ``man strftime``
+##      for the syntax.
+##
+## d: The string representing the time.
+##
+## Returns: The time value calculated from parsing *d* with *fmt*.
+function strptime%(fmt: string, d: string%) : time
+	%{
+	const time_t timeval = time_t(NULL);
+	struct tm t = *localtime(&timeval);
+	if ( strptime(d->CheckString(), fmt->CheckString(), &t) == NULL )
+		{
+		reporter->Warning("strptime conversion failed: fmt:%s d:%s", fmt->CheckString(), d->CheckString());
+		return new Val((double) 0, TYPE_TIME);
+		}
+
+	double ret = mktime(&t);
+	return new Val(ret, TYPE_TIME);
+	%}
+
+
 # ===========================================================================
 #
 #                           Network Type Processing
diff --git a/testing/btest/Baseline/bifs.strptime/.stdout b/testing/btest/Baseline/bifs.strptime/.stdout
new file mode 100644
index 0000000000..179612d4c4
--- /dev/null
+++ b/testing/btest/Baseline/bifs.strptime/.stdout
@@ -0,0 +1,2 @@
+1350604800.0
+0.0
diff --git a/testing/btest/Baseline/bifs.strptime/reporter.log b/testing/btest/Baseline/bifs.strptime/reporter.log
new file mode 100644
index 0000000000..367dbd63c1
--- /dev/null
+++ b/testing/btest/Baseline/bifs.strptime/reporter.log
@@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#open	2012-10-19-06-06-36
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::WARNING	strptime conversion failed: fmt:%m d:1980-10-24	(empty)
+#close	2012-10-19-06-06-36
diff --git a/testing/btest/bifs/strptime.bro b/testing/btest/bifs/strptime.bro
new file mode 100644
index 0000000000..7a58989679
--- /dev/null
+++ b/testing/btest/bifs/strptime.bro
@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff .stdout
+# @TEST-EXEC: btest-diff reporter.log
+
+event bro_init()
+	{
+	print strptime("%Y-%m-%d", "2012-10-19");
+	print strptime("%m", "1980-10-24");
+	}
\ No newline at end of file

From 46d225cc5bd4f2cba953a14925432080615d948e Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Mon, 22 Oct 2012 15:57:21 -0500
Subject: [PATCH 236/238] Add parsing rules for IPv4/IPv6 subnet literal
 constants, addresses #888

This fixes specifying IPv4 subnets in IPv4-mapped-IPv6 format with a
mask length relative to the 128 bits of the mapped IPv6 address.
---
 src/IPAddr.cc                              |  4 +--
 src/IPAddr.h                               |  9 ++++++-
 src/rule-scan.l                            | 19 +++----------
 src/scan.l                                 | 31 +++++++++++-----------
 src/util.cc                                | 29 ++++++++++++++++++++
 src/util.h                                 |  3 +++
 testing/btest/Baseline/language.addr/out   |  2 ++
 testing/btest/Baseline/language.subnet/out |  8 ++++++
 testing/btest/language/addr.bro            |  7 ++++-
 testing/btest/language/subnet.bro          | 19 ++++++++++++-
 10 files changed, 96 insertions(+), 35 deletions(-)

diff --git a/src/IPAddr.cc b/src/IPAddr.cc
index 0ba5589fff..51fb37c4d5 100644
--- a/src/IPAddr.cc
+++ b/src/IPAddr.cc
@@ -248,10 +248,10 @@ IPPrefix::IPPrefix(const in6_addr& in6, uint8_t length)
 	prefix.Mask(this->length);
 	}
 
-IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length)
+IPPrefix::IPPrefix(const IPAddr& addr, uint8_t length, bool len_is_v6_relative)
 	: prefix(addr)
 	{
-	if ( prefix.GetFamily() == IPv4 )
+	if ( prefix.GetFamily() == IPv4 && ! len_is_v6_relative )
 		{
 		if ( length > 32 )
 			reporter->InternalError("Bad IPAddr(v4) IPPrefix length : %d",
diff --git a/src/IPAddr.h b/src/IPAddr.h
index 6d26ef3fa8..5ddee70fb8 100644
--- a/src/IPAddr.h
+++ b/src/IPAddr.h
@@ -496,8 +496,15 @@ public:
 	 * @param addr The IP address.
 	 *
 	 * @param length The prefix length in the range from 0 to 128
+	 *
+	 * @param len_is_v6_relative Whether \a length is relative to the full
+	 * 128 bits of an IPv6 address.  If false and \a addr is an IPv4
+	 * address, then \a length is expected to range from 0 to 32.  If true
+	 * \a length is expected to range from 0 to 128 even if \a addr is IPv4,
+	 * meaning that the mask is to apply to the IPv4-mapped-IPv6 representation.
 	 */
-	IPPrefix(const IPAddr& addr, uint8_t length);
+	IPPrefix(const IPAddr& addr, uint8_t length,
+	         bool len_is_v6_relative = false);
 
 	/**
 	 * Copy constructor.
diff --git a/src/rule-scan.l b/src/rule-scan.l
index d516a98e89..9c755d04e3 100644
--- a/src/rule-scan.l
+++ b/src/rule-scan.l
@@ -11,16 +11,6 @@
 #include "rule-parse.h"
 
 int rules_line_number = 0;
-
-static string extract_ipv6(string s)
-	{
-	if ( s.substr(0, 3) == "[0x" )
-		s = s.substr(3, s.find("]") - 3);
-	else
-		s = s.substr(1, s.find("]") - 1);
-
-	return s;
-	}
 %}
 
 %x PS
@@ -49,15 +39,14 @@ PID	([0-9a-zA-Z_-]|"::")+
 }
 
 {IP6}	{
-	rules_lval.prefixval = new IPPrefix(IPAddr(extract_ipv6(yytext)), 128);
+	rules_lval.prefixval = new IPPrefix(IPAddr(extract_ip(yytext)), 128, true);
 	return TOK_IP6;
 	}
 
 {IP6}{OWS}"/"{OWS}{D}	{
-	char* l = strchr(yytext, '/');
-	*l++ = '\0';
-	int len = atoi(l);
-	rules_lval.prefixval = new IPPrefix(IPAddr(extract_ipv6(yytext)), len);
+	int len = 0;
+	string ip = extract_ip_and_len(yytext, &len);
+	rules_lval.prefixval = new IPPrefix(IPAddr(ip), len, true);
 	return TOK_IP6;
 	}
 
diff --git a/src/scan.l b/src/scan.l
index 6c87766781..8ff33e7d24 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -148,6 +148,7 @@ D	[0-9]+
 HEX	[0-9a-fA-F]+
 IDCOMPONENT [A-Za-z_][A-Za-z_0-9]*
 ID	{IDCOMPONENT}(::{IDCOMPONENT})*
+IP6 ("["({HEX}:){7}{HEX}"]")|("["0x{HEX}({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*"]")|("["({HEX}|:)*"::"({HEX}|:)*({D}"."){3}{D}"]")
 FILE	[^ \t\n]+
 PREFIX	[^ \t\n]+
 FLOAT	(({D}*"."?{D})|({D}"."?{D}*))([eE][-+]?{D})?
@@ -229,21 +230,23 @@ ESCSEQ	(\\([^\n]|[0-7]+|x[[:xdigit:]]+))
 			}
 
 	/* IPv6 literal constant patterns */
-"["({HEX}:){7}{HEX}"]" {
-	string s(yytext+1);
-	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+{IP6}	{
+	RET_CONST(new AddrVal(extract_ip(yytext)))
 }
-"["0x{HEX}({HEX}|:)*"::"({HEX}|:)*"]" {
-	string s(yytext+3);
-	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+
+{IP6}{OWS}"/"{OWS}{D}	{
+	int len = 0;
+	string ip = extract_ip_and_len(yytext, &len);
+	RET_CONST(new SubNetVal(IPPrefix(IPAddr(ip), len, true)))
 }
-"["({HEX}|:)*"::"({HEX}|:)*"]" {
-	string s(yytext+1);
-	RET_CONST(new AddrVal(s.erase(s.size()-1)))
-}
-"["({HEX}|:)*"::"({HEX}|:)*({D}"."){3}{D}"]" {
-	string s(yytext+1);
-	RET_CONST(new AddrVal(s.erase(s.size()-1)))
+
+	/* IPv4 literal constant patterns */
+({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
+
+({D}"."){3}{D}{OWS}"/"{OWS}{D}	{
+	int len = 0;
+	string ip = extract_ip_and_len(yytext, &len);
+	RET_CONST(new SubNetVal(IPPrefix(IPAddr(ip), len)))
 }
 
 [!%*/+\-,:;<=>?()\[\]{}~$|]	return yytext[0];
@@ -484,8 +487,6 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 {FLOAT}{OWS}msec(s?)	RET_CONST(new IntervalVal(atof(yytext),Milliseconds))
 {FLOAT}{OWS}usec(s?)	RET_CONST(new IntervalVal(atof(yytext),Microseconds))
 
-({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
-
 "0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
 
 {H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
diff --git a/src/util.cc b/src/util.cc
index 76ca7729df..ba7b9cfa46 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -43,6 +43,35 @@
 #include "Net.h"
 #include "Reporter.h"
 
+/**
+ * Return IP address without enclosing brackets and any leading 0x.
+ */
+std::string extract_ip(const std::string& i)
+	{
+	std::string s(skip_whitespace(i.c_str()));
+	if ( s.size() > 0 && s[0] == '[' )
+		s.erase(0, 1);
+	if ( s.size() > 1 && s.substr(0, 2) == "0x" )
+		s.erase(0, 2);
+	size_t pos = 0;
+	if ( (pos = s.find(']')) != std::string::npos )
+		s = s.substr(0, pos);
+	return s;
+	}
+
+/**
+ * Given a subnet string, return IP address and subnet length separately.
+ */
+std::string extract_ip_and_len(const std::string& i, int* len)
+	{
+	size_t pos = i.find('/');
+	if ( pos == std::string::npos )
+		return i;
+	if ( len )
+		*len = atoi(i.substr(pos + 1).c_str());
+	return extract_ip(i.substr(0, pos));
+	}
+
 /**
  * Takes a string, unescapes all characters that are escaped as hex codes
  * (\x##) and turns them into the equivalent ascii-codes. Returns a string
diff --git a/src/util.h b/src/util.h
index e69167abce..71b9c494e8 100644
--- a/src/util.h
+++ b/src/util.h
@@ -91,6 +91,9 @@ void delete_each(T* t)
 		delete *it;
 	}
 
+std::string extract_ip(const std::string& i);
+std::string extract_ip_and_len(const std::string& i, int* len);
+
 std::string get_unescaped_string(const std::string& str);
 std::string get_escaped_string(const std::string& str, bool escape_all);
 
diff --git a/testing/btest/Baseline/language.addr/out b/testing/btest/Baseline/language.addr/out
index b04aac5ce3..b0ecdd3605 100644
--- a/testing/btest/Baseline/language.addr/out
+++ b/testing/btest/Baseline/language.addr/out
@@ -13,3 +13,5 @@ IPv6 address not case-sensitive (PASS)
 size of IPv6 address (PASS)
 IPv6 address type inference (PASS)
 IPv4 and IPv6 address inequality (PASS)
+IPv4-mapped-IPv6 equality to IPv4 (PASS)
+IPv4-mapped-IPv6 is IPv4 (PASS)
diff --git a/testing/btest/Baseline/language.subnet/out b/testing/btest/Baseline/language.subnet/out
index 45900a291e..e8c4ba354f 100644
--- a/testing/btest/Baseline/language.subnet/out
+++ b/testing/btest/Baseline/language.subnet/out
@@ -10,3 +10,11 @@ IPv6 subnet !in operator (PASS)
 IPv6 subnet type inference (PASS)
 IPv4 and IPv6 subnet inequality (PASS)
 IPv4 address and IPv6 subnet (PASS)
+IPv4 in IPv4-mapped-IPv6 subnet (PASS)
+IPv6 !in IPv4-mapped-IPv6 subnet (PASS)
+IPv4-mapped-IPv6 in IPv4-mapped-IPv6 subnet (PASS)
+IPv4-mapped-IPv6 subnet equality (PASS)
+subnet literal const whitespace (PASS)
+subnet literal const whitespace (PASS)
+subnet literal const whitespace (PASS)
+subnet literal const whitespace (PASS)
diff --git a/testing/btest/language/addr.bro b/testing/btest/language/addr.bro
index 1cd93bad03..dd7e5e1dff 100644
--- a/testing/btest/language/addr.bro
+++ b/testing/btest/language/addr.bro
@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 function test_case(msg: string, expect: bool)
@@ -43,5 +43,10 @@ event bro_init()
 
 	test_case( "IPv4 and IPv6 address inequality", a1 != b1 );
 
+	# IPv4-mapped-IPv6 (internally treated as IPv4)
+	local c1: addr = [::ffff:1.2.3.4];
+
+	test_case( "IPv4-mapped-IPv6 equality to IPv4", c1 == 1.2.3.4 );
+	test_case( "IPv4-mapped-IPv6 is IPv4", is_v4_addr(c1) == T );
 }
 
diff --git a/testing/btest/language/subnet.bro b/testing/btest/language/subnet.bro
index ea641f6983..b3b50e085f 100644
--- a/testing/btest/language/subnet.bro
+++ b/testing/btest/language/subnet.bro
@@ -1,4 +1,4 @@
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro -b %INPUT >out
 # @TEST-EXEC: btest-diff out
 
 function test_case(msg: string, expect: bool)
@@ -43,5 +43,22 @@ event bro_init()
 	test_case( "IPv4 and IPv6 subnet inequality", s1 != t1 );
 	test_case( "IPv4 address and IPv6 subnet", a1 !in t2 );
 
+	# IPv4-mapped-IPv6 subnets
+	local u1: subnet = [::ffff:0:0]/96;
+
+	test_case( "IPv4 in IPv4-mapped-IPv6 subnet", 1.2.3.4 in u1 );
+	test_case( "IPv6 !in IPv4-mapped-IPv6 subnet", [fe80::1] !in u1 );
+	test_case( "IPv4-mapped-IPv6 in IPv4-mapped-IPv6 subnet",
+	           [::ffff:1.2.3.4] in u1 );
+	test_case( "IPv4-mapped-IPv6 subnet equality",
+	           [::ffff:1.2.3.4]/112 == 1.2.0.0/16 );
+	test_case( "subnet literal const whitespace",
+	           [::ffff:1.2.3.4] / 112 == 1.2.0.0 / 16 );
+	test_case( "subnet literal const whitespace",
+	           [::ffff:1.2.3.4]/ 128 == 1.2.3.4/ 32 );
+	test_case( "subnet literal const whitespace",
+	           [::ffff:1.2.3.4] /96 == 1.2.3.4 /0 );
+	test_case( "subnet literal const whitespace",
+	           [::ffff:1.2.3.4]   /    92 == [::fffe:1.2.3.4]    /   92 );
 }
 

From 54084d0744e606c566053b9f793c0d3c8c8b93de Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 24 Oct 2012 01:05:01 -0400
Subject: [PATCH 237/238] Adding a test for PPPoE support.

---
 testing/btest/Baseline/core.pppoe/conn.log |  16 ++++++++++++++++
 testing/btest/Traces/pppoe.trace           | Bin 0 -> 6296 bytes
 testing/btest/core/pppoe.test              |   2 ++
 3 files changed, 18 insertions(+)
 create mode 100644 testing/btest/Baseline/core.pppoe/conn.log
 create mode 100644 testing/btest/Traces/pppoe.trace
 create mode 100644 testing/btest/core/pppoe.test

diff --git a/testing/btest/Baseline/core.pppoe/conn.log b/testing/btest/Baseline/core.pppoe/conn.log
new file mode 100644
index 0000000000..002b8a7ca0
--- /dev/null
+++ b/testing/btest/Baseline/core.pppoe/conn.log
@@ -0,0 +1,16 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2012-10-24-05-04-16
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1284385418.014560	TEfuqmmG4bh	fe80::c801:eff:fe88:8	547	fe80::ce05:eff:fe88:0	546	udp	-	0.096000	192	0	S0	-	0	D	2	288	0	0	(empty)
+1284385417.962560	j4u32Pc5bif	fe80::ce05:eff:fe88:0	546	ff02::1:2	547	udp	-	0.078000	114	0	S0	-	0	D	2	210	0	0	(empty)
+1284385411.091560	arKYeMETxOg	fe80::c801:eff:fe88:8	136	ff02::1	135	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	(empty)
+1284385411.035560	UWkUyAuUGXf	fe80::c801:eff:fe88:8	143	ff02::16	0	icmp	-	0.835000	160	0	OTH	-	0	-	8	608	0	0	(empty)
+1284385451.658560	FrJExwHcSal	fc00:0:2:100::1:1	128	fc00::1	129	icmp	-	0.156000	260	260	OTH	-	0	-	5	500	5	500	(empty)
+1284385413.027560	nQcgTWjvg4c	fe80::c801:eff:fe88:8	134	fe80::ce05:eff:fe88:0	133	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	(empty)
+1284385412.963560	k6kgXLOoSKl	fe80::ce05:eff:fe88:0	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	48	0	0	(empty)
+#close	2012-10-24-05-04-16
diff --git a/testing/btest/Traces/pppoe.trace b/testing/btest/Traces/pppoe.trace
new file mode 100644
index 0000000000000000000000000000000000000000..4de67175c36894bf7337311bb5240b6767a51832
GIT binary patch
literal 6296
zcmchbU2GIp6vxk<kNub~+uc%NTSeM#l@>v2Ewx#TDUC5TF@&h0pwTq+!xDqkmJ~73
zEJATDL5T^r@j;4-q=?a~FZy8I#KaIarVo8kfmmaVCGiO#BvM(=+4;EL<<3W$)*&;U
zxjXwi|MQ=F&YflE>rc-}pn$bi0WiFHWy2ef#;%k=E1uJy%WvPS(F{PQJ08ZN2*Vqh
z3`)<+kpy1bZp0ct6tP%3<y(fV0J4Jk#kz$ohJxB3+x9q9AhxwseJxOX^y)8-UtIq8
z;FnLOk(KzIvvi83t70*%!^o=Jq~}LD7U}ARxOZK|x>h4wITq;Zi374C*=o#EQv>S!
z%x+t=@B?C9s8uXhfi9g@o@bRYMV*nAY*G!d`R}fan8ZpqII>We<XP9`G-nBN)Uwqw
zgAnSvBvQl2NEiIcSvEOJKsjD=&vD{9X9;wnUy^55OS5WuFJP5rllUBCYWLou?*00a
z=e6+cRm^d^Cup$*IR>*y^j=-~E%7<Ac2tF&Sdnyoe#rq%FJUnVM~T(-fab)Cr8_al
zDAO*mXzoS^LUu4>&28r_fnSy`JBB&#fo19XLCxzjzk1VOVh!wZ(?#>%n_n(0)P*dM
ze$5W?zABg#=oi*dUY8WRDX|3ulw)Yjk!9)f;#a8Lrmq_P3+I>QMb{CU6QcloP%G_%
z9JfQ~^OQ*~up4PQ0WfD;PSUT9<LVTsIKtpfc;_IF;i7=6QQ<ujMVxqh86pk~BQnTG
zY&*LQ5eKb^3%ddXnR2}L$udOz!-~jVWjU;Qia6QJU?<){yWlyt8_zM3%DFs8mVq5d
z(nu-3kD?_0VNTr$D+0T>hl(Ipf6TG!quJ^(g9xp`c55bz#En>)XE;lcJ+d@p5yu1T
zqV_*m#IE0#A)@2_oVprW`Wzi=aH3<y(nqwdh@&_s=01zU!#F=^7p!)TMn!~pMVP&}
zxND8TnR7JLMvyxBoL*y#uQRir%qYj;&_`_$v{r1DUgtrbc^#v`R1{vsE7}FxdkpQV
zxK+(2wLm-eKdDAaLaOpvHeY)l-wq`83btqsWMF!oKmh+$z#2&4F#_TIj|vh3qo`xH
z1pu38eizi$TZWE)uESg%inEoHQVsNPos~!j_6Smt&n#4=Cjq6(-hts}<v{Py0l9ns
zK0smhhOh@w%S%v+BLFYc-ZGCzz$M+>NcF(G0LO5Ec7c0?hUyNk+R?p^!EfaGzlX^^
z1lqY*n3KrMkJ*xX*;gJURfTa2)%o=DIC?q#tK)mI(WR-YI{#WCRyx7oiv`cc=+eY-
zW}QIR$q)GZl$aHkysRF?s{5U@1hbrUA<KRBI7K<0;yH?0q|4neVohJ<EOCyc%iXUr
zWKFb|S}V7h6_&mDHIA&zTASbE`uivOg)Dc!P9tmXNuHyPUk1y0P7rJA1ZRnJBwg-)
zy@#y+l;i&q(7ac?_(iPA-JB)PQA3uyU+0iD9t~U1QOqJ;?(>US4Uce^jbBEN&fcF#
zRyv|#XQu8f?74l=*@d4#YpSiFV>4j+Yx+#zSQ>tQXx$iIFiBR_Krj^6N+K(wrLlO~
z%JNmK*CZ+`Z@Yc%9qX#9Ym&8h-nG82e#6}xH{G+j;oipkNXf(#C6*F_L&ZwWxEQZk
z*<tv}my%tVxe|fD#Y)UoOz26r9z9{VU`Z2sQWc9VnI~WQQgU)VSK`-`vr+U!TKp_&
z+CrXmZRSe+&Xf0jDVb~MO8k0q{4exGUUcC!ZD}S?26l2KemxoWr6ktJmH74Kg&)xq
zWwE^kExDgOncc^g`1NGqmX&;nO5kb!E++7V)+E)-nnbMr)0`#9iq>#sxvxq80e!98
AiU0rr

literal 0
HcmV?d00001

diff --git a/testing/btest/core/pppoe.test b/testing/btest/core/pppoe.test
new file mode 100644
index 0000000000..35be84d657
--- /dev/null
+++ b/testing/btest/core/pppoe.test
@@ -0,0 +1,2 @@
+# @TEST-EXEC: bro -r $TRACES/pppoe.trace %INPUT 
+# @TEST-EXEC: btest-diff conn.log

From ae38aad2bbba572f5a3750d730064ef7b5525c9a Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 24 Oct 2012 15:40:06 -0700
Subject: [PATCH 238/238] Adding missing &redef for some TCP options.

Reported by Carsten Langer.

Closes #905.
Closes #906.
Closes #907.
---
 CHANGES                    | 5 +++++
 VERSION                    | 2 +-
 scripts/base/init-bare.bro | 6 +++---
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/CHANGES b/CHANGES
index 5ffeff242a..28b98e638b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,9 @@
 
+2.1-87 | 2012-10-24 15:40:06 -0700
+
+  * Adding missing &redef for some TCP options. Addresses #905, #906,
+    #907. (Carsten Langer)
+
 2.1-86 | 2012-10-24 15:37:11 -0700
 
   * Add parsing rules for IPv4/IPv6 subnet literal constants.
diff --git a/VERSION b/VERSION
index 8892e94465..1b9530da06 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-86
+2.1-87
diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro
index 70026394e9..598fdf9098 100644
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@@ -826,7 +826,7 @@ const tcp_storm_interarrival_thresh = 1 sec &redef;
 ## peer's ACKs.  Set to zero to turn off this determination.
 ##
 ## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks
-const tcp_max_initial_window = 4096;
+const tcp_max_initial_window = 4096 &redef;
 
 ## If we're not seeing our peer's ACKs, the maximum volume of data above a sequence
 ## hole that we'll tolerate before assuming that there's been a packet drop and we
@@ -834,7 +834,7 @@ const tcp_max_initial_window = 4096;
 ## up.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks
-const tcp_max_above_hole_without_any_acks = 4096;
+const tcp_max_above_hole_without_any_acks = 4096 &redef;
 
 ## If we've seen this much data without any of it being acked, we give up
 ## on that connection to avoid memory exhaustion due to buffering all that
@@ -843,7 +843,7 @@ const tcp_max_above_hole_without_any_acks = 4096;
 ## has in fact gone too far, but for now we just make this quite beefy.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
-const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024;
+const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 
 ## For services without an a handler, these sets define originator-side ports that
 ## still trigger reassembly.