Merge branch 'topic/robin/master-test'

* topic/robin/master-test: (60 commits)
  Script fix for Linux.
  Updating test base line.
  Another small change to MsgThread API.
  Bug fix for BasicThread.
  make version_ok return true for TLSv12
  Sed usage in canonifier script didn't work on non-Linux systems.
  Changing HTTP DPD port 3138 to 3128.
  Temporarily removing tuning/logs-to-elasticsearch.bro from the test-all-policy.
  More documentation updates.
  Revert "Fixing calc_next_rotate to use UTC based time functions."
  Some documentation updates for elasticsearch plugin.
  Give configure a --disable-perftools option.
  Updating tests for the #start/#end change.
  Further threading and API restructuring for logging and input frameworks.
  Reworking forceful thread termination.
  Moving the ASCII writer over to use UNIX I/O rather than stdio.
  Further reworking the thread API.
  Reworking thread termination logic.
  If a thread doesn't terminate, we log that but not longer proceed (because it could hang later still).
  Removing the thread kill functionality.
  ...

CMakeLists.txt

@@ -91,7 +91,9 @@ endif ()
 set(USE_PERFTOOLS false)
 set(USE_PERFTOOLS_DEBUG false)
 
+if (NOT DISABLE_PERFTOOLS)
    find_package(GooglePerftools)
+endif ()
 
 if (GOOGLEPERFTOOLS_FOUND)
     include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
@@ -122,6 +124,17 @@ if (LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
 	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
 endif()
 
+set(USE_ELASTICSEARCH false)
+set(USE_CURL false)
+find_package(CURL)
+
+if (CURL_FOUND)
+  set(USE_ELASTICSEARCH true)
+  set(USE_CURL true)
+  include_directories(BEFORE ${CURL_INCLUDE_DIR})
+  list(APPEND OPTLIBS ${CURL_LIBRARIES})
+endif()
+
 if (ENABLE_PERFTOOLS_DEBUG)
     # Just a no op to prevent CMake from complaining about manually-specified
     # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
@@ -213,7 +226,10 @@ message(
     "\nGeoIP:             ${USE_GEOIP}"
     "\nGoogle perftools:  ${USE_PERFTOOLS}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
+    "\ncURL:              ${USE_CURL}"
+    "\n"
     "\nDataSeries:        ${USE_DATASERIES}"
+    "\nElasticSearch:     ${USE_ELASTICSEARCH}"
     "\n"
     "\n================================================================\n"
 )

NEWS

@@ -140,6 +140,11 @@ the full set.
   Bro now supports decapsulating tunnels directly for protocols it
   understands.
 
+- ASCII logs now record the time when they were opened/closed at the
+  beginning and end of the file, respectively. The options
+  LogAscii::header_prefix and LogAscii::include_header have been
+  renamed to LogAscii::meta_prefix and LogAscii::include_meta,
+  respectively.
 
 Bro 2.0
 -------

config.h.in

@@ -114,9 +114,15 @@
 /* Analyze Mobile IPv6 traffic */
 #cmakedefine ENABLE_MOBILE_IPV6
 
+/* Use libCurl. */
+#cmakedefine USE_CURL
+
 /* Use the DataSeries writer. */
 #cmakedefine USE_DATASERIES
 
+/* Use the ElasticSearch writer. */
+#cmakedefine USE_ELASTICSEARCH
+
 /* Version number of package */
 #define VERSION "@VERSION@"
 

configure

@@ -33,6 +33,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
+    --disable-perftools    don't try to build python with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
 
@@ -105,6 +106,7 @@ append_cache_entry INSTALL_BROCCOLI     BOOL   true
 append_cache_entry INSTALL_BROCTL       BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
+append_cache_entry DISABLE_PERFTOOLS    BOOL   false
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -156,6 +158,9 @@ while [ $# -ne 0 ]; do
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS    BOOL   false
             ;;
+        --disable-perftools)
+            append_cache_entry DISABLE_PERFTOOLS    BOOL   true
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;

doc/logging-elasticsearch.rst

@@ -0,0 +1,89 @@
+
+=========================================
+Indexed Logging Output with ElasticSearch
+=========================================
+
+.. rst-class:: opening
+
+   Bro's default ASCII log format is not exactly the most efficient
+   way for searching large volumes of data. ElasticSearch
+   is a new data storage technology for dealing with tons of data.
+   It's also a search engine built on top of Apache's Lucene
+   project. It scales very well, both for distributed indexing and 
+   distributed searching.
+
+.. contents::
+
+Warning
+-------
+
+This writer plugin is still in testing and is not yet recommended for
+production use!  The approach to how logs are handled in the plugin is "fire
+and forget" at this time, there is no error handling if the server fails to
+respond successfully to the insertion request.
+
+Installing ElasticSearch
+------------------------
+
+Download the latest version from: <http://www.elasticsearch.org/download/>.
+Once extracted, start ElasticSearch with::
+
+# ./bin/elasticsearch
+
+For more detailed information, refer to the ElasticSearch installation
+documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html
+
+Compiling Bro with ElasticSearch Support
+----------------------------------------
+
+First, ensure that you have libcurl installed the run configure.::
+
+    # ./configure
+    [...]
+    ====================|  Bro Build Summary  |=====================
+    [...]
+    cURL:              true
+    [...]
+    ElasticSearch:     true
+    [...]
+    ================================================================
+
+Activating ElasticSearch
+------------------------
+
+The easiest way to enable ElasticSearch output is to load the tuning/logs-to-
+elasticsearch.bro script.  If you are using BroControl, the following line in
+local.bro will enable it.
+
+.. console::
+
+    @load tuning/logs-to-elasticsearch
+
+With that, Bro will now write most of its logs into ElasticSearch in addition
+to maintaining the Ascii logs like it would do by default.  That script has
+some tunable options for choosing which logs to send to ElasticSearch, refer
+to the autogenerated script documentation for those options.
+
+There is an interface being written specifically to integrate with the data
+that Bro outputs into ElasticSearch named Brownian.  It can be found here::
+
+    https://github.com/grigorescu/Brownian
+
+Tuning
+------
+
+A common problem encountered with ElasticSearch is too many files being held
+open.  The ElasticSearch website has some suggestions on how to increase the
+open file limit.
+
+  - http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html
+
+TODO
+----
+
+Lots.
+
+- Perform multicast discovery for server.
+- Better error detection.
+- Better defaults (don't index loaded-plugins, for instance).
+- 

doc/scripts/DocSourcesList.cmake

@@ -42,6 +42,7 @@ rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
+rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
 rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
@@ -145,6 +146,7 @@ rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
+rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
 rest_target(${psd} site/local-manager.bro)
 rest_target(${psd} site/local-proxy.bro)

scripts/base/frameworks/logging/__load__.bro

@@ -2,4 +2,5 @@
 @load ./postprocessors
 @load ./writers/ascii
 @load ./writers/dataseries
+@load ./writers/elasticsearch
 @load ./writers/none

scripts/base/frameworks/logging/writers/ascii.bro

@@ -8,12 +8,13 @@ export {
 	## into files. This is primarily for debugging purposes.
 	const output_to_stdout = F &redef;
 
-	## If true, include a header line with column names and description
+	## If true, include lines with log meta information such as column names with
-	## of the other ASCII logging options that were used.
+	## types, the values of ASCII logging options that in use, and the time when the
-	const include_header = T &redef;
+	## file was opened and closes (the latter at the end). 
+	const include_meta = T &redef;
 
-	## Prefix for the header line if included.
+	## Prefix for lines with meta information.
-	const header_prefix = "#" &redef;
+	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
 	const separator = "\t" &redef;

scripts/base/frameworks/logging/writers/elasticsearch.bro

@@ -0,0 +1,46 @@
+##! Log writer for sending logs to an ElasticSearch server.
+##!
+##! Note: This module is in testing and is not yet considered stable!
+##!
+##! There is one known memory issue.  If your elasticsearch server is
+##! running slowly and taking too long to return from bulk insert
+##! requests, the message queue to the writer thread will continue
+##! growing larger and larger giving the appearance of a memory leak.
+
+module LogElasticSearch;
+
+export {
+	## Name of the ES cluster
+	const cluster_name = "elasticsearch" &redef;
+
+	## ES Server
+	const server_host = "127.0.0.1" &redef;
+
+	## ES Port
+	const server_port = 9200 &redef;
+
+	## Name of the ES index
+	const index_prefix = "bro" &redef;
+
+	## The ES type prefix comes before the name of the related log.
+	## e.g. prefix = "bro_" would create types of bro_dns, bro_software, etc.
+	const type_prefix = "" &redef;
+
+	## The time before an ElasticSearch transfer will timeout.
+	## This is not working!
+	const transfer_timeout = 2secs;
+
+	## The batch size is the number of messages that will be queued up before
+	## they are sent to be bulk indexed.
+	const max_batch_size = 1000 &redef;
+
+	## The maximum amount of wall-clock time that is allowed to pass without
+	## finishing a bulk log send.  This represents the maximum delay you
+	## would like to have with your logs before they are sent to ElasticSearch.
+	const max_batch_interval = 1min &redef;
+
+	## The maximum byte size for a buffered JSON string to send to the bulk
+	## insert API.
+	const max_byte_size = 1024 * 1024 &redef;
+}
+

scripts/base/protocols/dns/main.bro

@@ -162,11 +162,11 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
 
 	c$dns = c$dns_state$pending[msg$id];
 
+	if ( ! is_query )
+		{
 		c$dns$rcode = msg$rcode;
 		c$dns$rcode_name = base_errors[msg$rcode];
 
-	if ( ! is_query )
-		{
 		if ( ! c$dns?$total_answers )
 			c$dns$total_answers = msg$num_answers;
 

scripts/base/protocols/http/main.bro

@@ -114,7 +114,7 @@ event bro_init() &priority=5
 
 # DPD configuration.
 const ports = {
-	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3138/tcp,
+	80/tcp, 81/tcp, 631/tcp, 1080/tcp, 3128/tcp,
 	8000/tcp, 8080/tcp, 8888/tcp,
 };
 redef dpd_config += { 

scripts/policy/tuning/logs-to-elasticsearch.bro

@@ -0,0 +1,45 @@
+##! Load this script to enable global log output to an ElasticSearch database.
+
+module LogElasticSearch;
+
+export {
+	## An elasticsearch specific rotation interval.
+	const rotation_interval = 24hr &redef;
+
+	## Optionally ignore any :bro:enum:`Log::ID` from being sent to
+	## ElasticSearch with this script.
+	const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
+
+	## If you want to explicitly only send certain :bro:enum:`Log::ID` 
+	## streams, add them to this set.  If the set remains empty, all will 
+	## be sent.  The :bro:id:`excluded_log_ids` option will remain in 
+	## effect as well.
+	const send_logs: set[string] = set() &redef;
+}
+
+module Log;
+
+event bro_init() &priority=-5
+	{
+	local my_filters: table[ID, string] of Filter = table();
+
+	for ( [id, name] in filters )
+		{
+		local filter = filters[id, name];
+		if ( fmt("%s", id) in LogElasticSearch::excluded_log_ids ||
+		     (|LogElasticSearch::send_logs| > 0 && fmt("%s", id) !in LogElasticSearch::send_logs) )
+			next;
+
+		filter$name = cat(name, "-es");
+		filter$writer = Log::WRITER_ELASTICSEARCH;
+		filter$interv = LogElasticSearch::rotation_interval;
+		my_filters[id, name] = filter;
+		}
+
+	# This had to be done separately to avoid an ever growing filters list
+	# where the for loop would never end.
+	for ( [id, name] in my_filters )
+		{
+		Log::add_filter(id, filter);
+		}
+	}

scripts/test-all-policy.bro

@@ -60,4 +60,5 @@
 @load tuning/defaults/__load__.bro
 @load tuning/defaults/packet-fragments.bro
 @load tuning/defaults/warnings.bro
+# @load tuning/logs-to-elasticsearch.bro
 @load tuning/track-all-assets.bro

src/CMakeLists.txt

@@ -428,6 +428,7 @@ set(bro_SRCS
     logging/WriterFrontend.cc
     logging/writers/Ascii.cc
     logging/writers/DataSeries.cc
+    logging/writers/ElasticSearch.cc
     logging/writers/None.cc
 
     input/Manager.cc

src/RemoteSerializer.cc

@@ -2692,12 +2692,12 @@ bool RemoteSerializer::ProcessLogCreateWriter()
 
 	int id, writer;
 	int num_fields;
-	logging::WriterBackend::WriterInfo info;
+	logging::WriterBackend::WriterInfo* info = new logging::WriterBackend::WriterInfo();
 
 	bool success = fmt.Read(&id, "id") &&
 		fmt.Read(&writer, "writer") &&
 		fmt.Read(&num_fields, "num_fields") &&
-		info.Read(&fmt);
+		info->Read(&fmt);
 
 	if ( ! success )
 		goto error;

src/input/Manager.cc

@@ -71,7 +71,7 @@ declare(PDict, InputHash);
 class Manager::Stream {
 public:
 	string name;
-	ReaderBackend::ReaderInfo info;
+	ReaderBackend::ReaderInfo* info;
 	bool removed;
 
 	StreamType stream_type; // to distinguish between event and table streams
@@ -257,7 +257,6 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type)
 
 	assert(ir->factory);
 
-	frontend->SetTypeName(ir->name);
 	ReaderBackend* backend = (*ir->factory)(frontend);
 	assert(backend);
 
@@ -291,9 +290,6 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 
 	EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal();
 
-	ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt());
-	assert(reader_obj);
-
 	// get the source ...
 	Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source"));
 	assert ( sourceval != 0 );
@@ -301,21 +297,22 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 	string source((const char*) bsource->Bytes(), bsource->Len());
 	Unref(sourceval);
 
-	EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal();
+	ReaderBackend::ReaderInfo* rinfo = new ReaderBackend::ReaderInfo();
-	Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
+	rinfo->source = copy_string(source.c_str());
 
+	EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal();
 	switch ( mode->InternalInt() )
 		{
 		case 0:
-			info->info.mode = MODE_MANUAL;
+			rinfo->mode = MODE_MANUAL;
 			break;
 
 		case 1:
-			info->info.mode = MODE_REREAD;
+			rinfo->mode = MODE_REREAD;
 			break;
 
 		case 2:
-			info->info.mode = MODE_STREAM;
+			rinfo->mode = MODE_STREAM;
 			break;
 
 		default:
@@ -324,12 +321,16 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 
 	Unref(mode);
 
+	Val* config = description->LookupWithDefault(rtype->FieldOffset("config"));
+
+	ReaderFrontend* reader_obj = new ReaderFrontend(*rinfo, reader);
+	assert(reader_obj);
+
 	info->reader = reader_obj;
 	info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault
 	info->name = name;
 	info->config = config->AsTableVal(); // ref'd by LookupWithDefault
-
+	info->info = rinfo;
-	info->info.source = source;
 
 	Ref(description);
 	info->description = description;
@@ -344,7 +345,7 @@ bool Manager::CreateStream(Stream* info, RecordVal* description)
 			ListVal* index = info->config->RecoverIndex(k);
 			string key = index->Index(0)->AsString()->CheckString();
 			string value = v->Value()->AsString()->CheckString();
-			info->info.config.insert(std::make_pair(key, value));
+			info->info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
 			Unref(index);
 			delete k;
 			}
@@ -475,7 +476,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	assert(stream->reader);
 
-	stream->reader->Init(stream->info, stream->num_fields, logf );
+	stream->reader->Init(stream->num_fields, logf );
 
 	readers[stream->reader] = stream;
 
@@ -652,7 +653,7 @@ bool Manager::CreateTableStream(RecordVal* fval)
 
 
 	assert(stream->reader);
-	stream->reader->Init(stream->info, fieldsV.size(), fields );
+	stream->reader->Init(fieldsV.size(), fields );
 
 	readers[stream->reader] = stream;
 
@@ -726,8 +727,6 @@ bool Manager::RemoveStream(Stream *i)
 
 	i->removed = true;
 
-	i->reader->Close();
-
 	DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s",
 		i->name.c_str());
 
@@ -793,17 +792,19 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 
 		else
 			{
-			Field* field = new Field();
+			string name = nameprepend + rec->FieldName(i);
-			field->name = nameprepend + rec->FieldName(i);
+			const char* secondary = 0;
-			field->type = rec->FieldType(i)->Tag();
+			TypeTag ty = rec->FieldType(i)->Tag();
+			TypeTag st = TYPE_VOID;
+			bool optional = false;
 
-			if ( field->type == TYPE_TABLE )
+			if ( ty == TYPE_TABLE )
-				field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag();
+				st = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag();
 
-			else if ( field->type == TYPE_VECTOR )
+			else if ( ty == TYPE_VECTOR )
-				field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag();
+				st = rec->FieldType(i)->AsVectorType()->YieldType()->Tag();
 
-			else if ( field->type == TYPE_PORT &&
+			else if ( ty == TYPE_PORT &&
 				  rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) )
 				{
 				// we have an annotation for the second column
@@ -813,12 +814,13 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 				assert(c);
 				assert(c->Type()->Tag() == TYPE_STRING);
 
-				field->secondary_name = c->AsStringVal()->AsString()->CheckString();
+				secondary = c->AsStringVal()->AsString()->CheckString();
 				}
 
 			if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) )
-				field->optional = true;
+				optional = true;
 
+			Field* field = new Field(name.c_str(), secondary, ty, st, optional);
 			fields->push_back(field);
 			}
 		}
@@ -1232,7 +1234,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 #endif
 
 	// Send event that the current update is indeed finished.
-	SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info.source.c_str()));
+	SendEvent(update_finished, 2, new StringVal(i->name.c_str()), new StringVal(i->info->source));
 	}
 
 void Manager::Put(ReaderFrontend* reader, Value* *vals)
@@ -1709,7 +1711,7 @@ int Manager::GetValueLength(const Value* val) {
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		length += val->val.string_val->size();
+		length += val->val.string_val.length;
 		break;
 		}
 
@@ -1808,8 +1810,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length());
+		memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length);
-		return val->val.string_val->size();
+		return val->val.string_val.length;
 		}
 
 	case TYPE_ADDR:
@@ -1957,7 +1959,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 
 	case TYPE_STRING:
 		{
-		BroString *s = new BroString(*(val->val.string_val));
+		BroString *s = new BroString((const u_char*)val->val.string_val.data, val->val.string_val.length, 0);
 		return new StringVal(s);
 		}
 
@@ -2041,8 +2043,8 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 	case TYPE_ENUM: {
 		// well, this is kind of stupid, because EnumType just mangles the module name and the var name together again...
 		// but well
-		string module = extract_module_name(val->val.string_val->c_str());
+		string module = extract_module_name(val->val.string_val.data);
-		string var = extract_var_name(val->val.string_val->c_str());
+		string var = extract_var_name(val->val.string_val.data);
 		bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str());
 		if ( index == -1 )
 			reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s",

src/input/ReaderBackend.cc

@@ -56,22 +56,24 @@ private:
 
 class SendEventMessage : public threading::OutputMessage<ReaderFrontend> {
 public:
-	SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val)
+	SendEventMessage(ReaderFrontend* reader, const char* name, const int num_vals, Value* *val)
 		: threading::OutputMessage<ReaderFrontend>("SendEvent", reader),
-		name(name), num_vals(num_vals), val(val) {}
+		name(copy_string(name)), num_vals(num_vals), val(val) {}
+
+	virtual ~SendEventMessage()	{ delete [] name; }
 
 	virtual bool Process()
 		{
 		bool success = input_mgr->SendEvent(name, num_vals, val);
 
 		if ( ! success )
-			reporter->Error("SendEvent for event %s failed", name.c_str());
+			reporter->Error("SendEvent for event %s failed", name);
 
 		return true; // We do not want to die if sendEvent fails because the event did not return.
 		}
 
 private:
-	const string name;
+	const char* name;
 	const int num_vals;
 	Value* *val;
 };
@@ -146,12 +148,14 @@ ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread()
 	{
 	disabled = true; // disabled will be set correcty in init.
 	frontend = arg_frontend;
+	info = new ReaderInfo(frontend->Info());
 
 	SetName(frontend->Name());
 	}
 
 ReaderBackend::~ReaderBackend()
 	{
+	delete info;
 	}
 
 void ReaderBackend::Put(Value* *val)
@@ -169,7 +173,7 @@ void ReaderBackend::Clear()
 	SendOut(new ClearMessage(frontend));
 	}
 
-void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals)
+void ReaderBackend::SendEvent(const char* name, const int num_vals, Value* *vals)
 	{
 	SendOut(new SendEventMessage(frontend, name, num_vals, vals));
 	}
@@ -184,17 +188,14 @@ void ReaderBackend::SendEntry(Value* *vals)
 	SendOut(new SendEntryMessage(frontend, vals));
 	}
 
-bool ReaderBackend::Init(const ReaderInfo& arg_info, const int arg_num_fields,
+bool ReaderBackend::Init(const int arg_num_fields,
 		         const threading::Field* const* arg_fields)
 	{
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
-	SetName("InputReader/"+info.source);
-
 	// disable if DoInit returns error.
-	int success = DoInit(arg_info, arg_num_fields, arg_fields);
+	int success = DoInit(*info, arg_num_fields, arg_fields);
 
 	if ( ! success )
 		{
@@ -207,7 +208,7 @@ bool ReaderBackend::Init(const ReaderInfo& arg_info, const int arg_num_fields,
 	return success;
 	}
 
-void ReaderBackend::Close()
+bool ReaderBackend::OnFinish(double network_time)
 	{
 	DoClose();
 	disabled = true; // frontend disables itself when it gets the Close-message.
@@ -221,6 +222,8 @@ void ReaderBackend::Close()
 		delete [] (fields);
 		fields = 0;
 		}
+
+	return true;
 	}
 
 bool ReaderBackend::Update()
@@ -243,10 +246,9 @@ void ReaderBackend::DisableFrontend()
 	SendOut(new DisableMessage(frontend));
 	}
 
-bool ReaderBackend::DoHeartbeat(double network_time, double current_time)
+bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
 	{
-	MsgThread::DoHeartbeat(network_time, current_time);
+	return DoHeartbeat(network_time, current_time);
-	return true;
 	}
 
 TransportProto ReaderBackend::StringToProto(const string &proto)

src/input/ReaderBackend.h

@@ -34,7 +34,10 @@ enum ReaderMode {
 	 * for new appended data. When new data is appended is has to be sent
 	 * using the Put api functions.
 	 */
-	MODE_STREAM
+	MODE_STREAM,
+
+	/** Internal dummy mode for initialization. */
+	MODE_NONE
 };
 
 class ReaderFrontend;
@@ -70,14 +73,17 @@ public:
 	 */
 	struct ReaderInfo
 		{
-		typedef std::map<string, string> config_map;
+		// Structure takes ownership of the strings.
+		typedef std::map<const char*, const char*> config_map;
 
 		/**
 		 * A string left to the interpretation of the reader
 		 * implementation; it corresponds to the value configured on
 		 * the script-level for the logging filter.
+		 *
+		 * Structure takes ownership of the string.
 		 */
-		string source;
+		const char* source;
 
 		/**
 		 * A map of key/value pairs corresponding to the relevant
@@ -89,6 +95,35 @@ public:
 		 * The opening mode for the input source.
 		 */
 		ReaderMode mode;
+
+		ReaderInfo()
+			{
+			source = 0;
+			mode = MODE_NONE;
+			}
+
+		ReaderInfo(const ReaderInfo& other)
+			{
+			source = other.source ? copy_string(other.source) : 0;
+			mode = other.mode;
+
+			for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
+				config.insert(std::make_pair(copy_string(i->first), copy_string(i->second)));
+			}
+
+		~ReaderInfo()
+			{
+			delete [] source;
+
+			for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
+				{
+				delete [] i->first;
+				delete [] i->second;
+				}
+			}
+
+		private:
+		const ReaderInfo& operator=(const ReaderInfo& other); // Disable.
 		};
 
 	/**
@@ -106,16 +141,7 @@ public:
 	 *
 	 * @return False if an error occured.
 	 */
-	bool Init(const ReaderInfo& info, int num_fields, const threading::Field* const* fields);
+	bool Init(int num_fields, const threading::Field* const* fields);
-
-	/**
-	 * Finishes reading from this input stream in a regular fashion. Must
-	 * not be called if an error has been indicated earlier. After
-	 * calling this, no further reading from the stream can be performed.
-	 *
-	 * @return False if an error occured.
-	 */
-	void Close();
 
 	/**
 	 * Force trigger an update of the input stream. The action that will
@@ -142,13 +168,16 @@ public:
 	/**
 	 * Returns the additional reader information into the constructor.
 	 */
-	const ReaderInfo& Info() const	{ return info; }
+	const ReaderInfo& Info() const	{ return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
 	 */
 	int NumFields() const	{ return num_fields; }
 
+	// Overridden from MsgThread.
+	virtual bool OnHeartbeat(double network_time, double current_time);
+	virtual bool OnFinish(double network_time);
 
 protected:
 	// Methods that have to be overwritten by the individual readers
@@ -200,6 +229,11 @@ protected:
 	 */
 	virtual bool DoUpdate() = 0;
 
+	/**
+	 * Triggered by regular heartbeat messages from the main thread.
+	 */
+	virtual bool DoHeartbeat(double network_time, double current_time) = 0;
+
 	/**
 	 * Method allowing a reader to send a specified Bro event. Vals must
 	 * match the values expected by the bro event.
@@ -210,7 +244,7 @@ protected:
 	 *
 	 * @param vals the values to be given to the event
 	 */
-	void SendEvent(const string& name, const int num_vals, threading::Value* *vals);
+	void SendEvent(const char* name, const int num_vals, threading::Value* *vals);
 
 	// Content-sending-functions (simple mode). Include table-specific
 	// functionality that simply is not used if we have no table.
@@ -271,14 +305,6 @@ protected:
 	 */
 	void EndCurrentSend();
 
-	/**
-	 * Triggered by regular heartbeat messages from the main thread.
-	 *
-	 * This method can be overridden but once must call
-	 * ReaderBackend::DoHeartbeat().
-	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
-
 	/**
 	 *  Convert a string into a TransportProto. This is just a utility
 	 *  function for Readers.
@@ -300,7 +326,7 @@ private:
 	// from this class, it's running in a different thread!
 	ReaderFrontend* frontend;
 
-	ReaderInfo info;
+	ReaderInfo* info;
 	unsigned int num_fields;
 	const threading::Field* const * fields; // raw mapping
 

src/input/ReaderFrontend.cc

@@ -11,18 +11,17 @@ namespace input {
 class InitMessage : public threading::InputMessage<ReaderBackend>
 {
 public:
-	InitMessage(ReaderBackend* backend, const ReaderBackend::ReaderInfo& info,
+	InitMessage(ReaderBackend* backend,
 		    const int num_fields, const threading::Field* const* fields)
 		: threading::InputMessage<ReaderBackend>("Init", backend),
-		info(info), num_fields(num_fields), fields(fields) { }
+		num_fields(num_fields), fields(fields) { }
 
 	virtual bool Process()
 		{
-		return Object()->Init(info, num_fields, fields);
+		return Object()->Init(num_fields, fields);
 		}
 
 private:
-	const ReaderBackend::ReaderInfo info;
 	const int num_fields;
 	const threading::Field* const* fields;
 };
@@ -37,32 +36,26 @@ public:
 	virtual bool Process() { return Object()->Update(); }
 };
 
-class CloseMessage : public threading::InputMessage<ReaderBackend>
+ReaderFrontend::ReaderFrontend(const ReaderBackend::ReaderInfo& arg_info, EnumVal* type)
-{
-public:
-	CloseMessage(ReaderBackend* backend)
-		: threading::InputMessage<ReaderBackend>("Close", backend)
-		 { }
-
-	virtual bool Process() { Object()->Close(); return true; }
-};
-
-
-ReaderFrontend::ReaderFrontend(bro_int_t type)
 	{
 	disabled = initialized = false;
-	ty_name = "<not set>";
+	info = new ReaderBackend::ReaderInfo(arg_info);
-	backend = input_mgr->CreateBackend(this, type);
 
+	const char* t = type->Type()->AsEnumType()->Lookup(type->InternalInt());
+	name = copy_string(fmt("%s/%s", arg_info.source, t));
+
+	backend = input_mgr->CreateBackend(this, type->InternalInt());
 	assert(backend);
 	backend->Start();
 	}
 
 ReaderFrontend::~ReaderFrontend()
 	{
+	delete [] name;
+	delete info;
 	}
 
-void ReaderFrontend::Init(const ReaderBackend::ReaderInfo& arg_info, const int arg_num_fields,
+void ReaderFrontend::Init(const int arg_num_fields,
 		          const threading::Field* const* arg_fields)
 	{
 	if ( disabled )
@@ -71,12 +64,11 @@ void ReaderFrontend::Init(const ReaderBackend::ReaderInfo& arg_info, const int a
 	if ( initialized )
 		reporter->InternalError("reader initialize twice");
 
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 	initialized = true;
 
-	backend->SendIn(new InitMessage(backend, info, num_fields, fields));
+	backend->SendIn(new InitMessage(backend, num_fields, fields));
 	}
 
 void ReaderFrontend::Update()
@@ -93,27 +85,9 @@ void ReaderFrontend::Update()
 	backend->SendIn(new UpdateMessage(backend));
 	}
 
-void ReaderFrontend::Close()
+const char* ReaderFrontend::Name() const
 	{
-	if ( disabled )
+	return name;
-		return;
-
-	if ( ! initialized )
-		{
-		reporter->Error("Tried to call finish on uninitialized reader");
-		return;
-		}
-
-	disabled = true;
-	backend->SendIn(new CloseMessage(backend));
-	}
-
-string ReaderFrontend::Name() const
-	{
-	if ( ! info.source.size() )
-		return ty_name;
-
-	return ty_name + "/" + info.source;
 	}
 
 }

src/input/ReaderFrontend.h

@@ -4,10 +4,11 @@
 #define INPUT_READERFRONTEND_H
 
 #include "ReaderBackend.h"
-
 #include "threading/MsgThread.h"
 #include "threading/SerialTypes.h"
 
+#include "Val.h"
+
 namespace input {
 
 class Manager;
@@ -25,6 +26,8 @@ public:
 	/**
 	 * Constructor.
 	 *
+	 * info: The meta information struct for the writer.
+	 *
 	 * type: The backend writer type, with the value corresponding to the
 	 * script-level \c Input::Reader enum (e.g., \a READER_ASCII). The
 	 * frontend will internally instantiate a ReaderBackend of the
@@ -32,7 +35,7 @@ public:
 	 *
 	 * Frontends must only be instantiated by the main thread.
 	 */
-	ReaderFrontend(bro_int_t type);
+	ReaderFrontend(const ReaderBackend::ReaderInfo& info, EnumVal* type);
 
 	/**
 	 * Destructor.
@@ -52,7 +55,7 @@ public:
 	 *
 	 * This method must only be called from the main thread.
 	 */
-	void Init(const ReaderBackend::ReaderInfo& info, const int arg_num_fields, const threading::Field* const* fields);
+	void Init(const int arg_num_fields, const threading::Field* const* fields);
 
 	/**
 	 * Force an update of the current input source. Actual action depends
@@ -100,12 +103,12 @@ public:
 	 *
 	 * This method is safe to call from any thread.
 	 */
-	string Name() const;
+	const char* Name() const;
 
 	/**
 	 * Returns the additional reader information passed into the constructor.
 	 */
-	const ReaderBackend::ReaderInfo& Info() const	{ return info; }
+	const ReaderBackend::ReaderInfo& Info() const	{ assert(info); return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -120,24 +123,14 @@ public:
 protected:
 	friend class Manager;
 
-	/**
-	 * Returns the name of the backend's type.
-	 */
-	const string& TypeName() const	{ return ty_name; }
-
-	/**
-	 * Sets the name of the backend's type.
-	 */
-	void SetTypeName(const string& name)	{ ty_name = name; }
-
 private:
 	ReaderBackend* backend;	// The backend we have instanatiated.
-	ReaderBackend::ReaderInfo info;	// Meta information as passed to Init().
+	ReaderBackend::ReaderInfo* info;	// Meta information.
 	const threading::Field* const*  fields;	// The input fields.
 	int num_fields;		// Information as passed to Init().
-	string ty_name;		// Backend type, set by manager.
 	bool disabled;		// True if disabled.
 	bool initialized;	// True if initialized.
+	const char* name;	// Descriptive name.
 };
 
 }

src/input/readers/Ascii.cc

@@ -87,10 +87,10 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
 	{
 	mtime = 0;
 
-	file = new ifstream(info.source.c_str());
+	file = new ifstream(info.source);
 	if ( ! file->is_open() )
 		{
-		Error(Fmt("Init: cannot open %s", info.source.c_str()));
+		Error(Fmt("Init: cannot open %s", info.source));
 		delete(file);
 		file = 0;
 		return false;
@@ -98,7 +98,7 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f
 
 	if ( ReadHeader(false) == false )
 		{
-		Error(Fmt("Init: cannot open %s; headers are incorrect", info.source.c_str()));
+		Error(Fmt("Init: cannot open %s; headers are incorrect", info.source));
 		file->close();
 		delete(file);
 		file = 0;
@@ -164,20 +164,20 @@ bool Ascii::ReadHeader(bool useCached)
 				}
 
 			Error(Fmt("Did not find requested field %s in input data file %s.",
-				  field->name.c_str(), Info().source.c_str()));
+				  field->name, Info().source));
 			return false;
 			}
 
 
 		FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]);
 
-		if ( field->secondary_name != "" )
+		if ( field->secondary_name && strlen(field->secondary_name) != 0 )
 			{
 			map<string, uint32_t>::iterator fit2 = ifields.find(field->secondary_name);
 			if ( fit2 == ifields.end() )
 				{
 				Error(Fmt("Could not find requested port type field %s in input data file.",
-					  field->secondary_name.c_str()));
+					  field->secondary_name));
 				return false;
 				}
 
@@ -220,7 +220,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 	switch ( field.type ) {
 	case TYPE_ENUM:
 	case TYPE_STRING:
-		val->val.string_val = new string(s);
+		val->val.string_val.length = s.size();
+		val->val.string_val.data = copy_string(s.c_str());
 		break;
 
 	case TYPE_BOOL:
@@ -367,9 +368,9 @@ bool Ascii::DoUpdate()
 			{
 			// check if the file has changed
 			struct stat sb;
-			if ( stat(Info().source.c_str(), &sb) == -1 )
+			if ( stat(Info().source, &sb) == -1 )
 				{
-				Error(Fmt("Could not get stat for %s", Info().source.c_str()));
+				Error(Fmt("Could not get stat for %s", Info().source));
 				return false;
 				}
 
@@ -403,10 +404,10 @@ bool Ascii::DoUpdate()
 				file = 0;
 				}
 
-			file = new ifstream(Info().source.c_str());
+			file = new ifstream(Info().source);
 			if ( ! file->is_open() )
 				{
-				Error(Fmt("cannot open %s", Info().source.c_str()));
+				Error(Fmt("cannot open %s", Info().source));
 				return false;
 				}
 
@@ -506,8 +507,6 @@ bool Ascii::DoUpdate()
 
 bool Ascii::DoHeartbeat(double network_time, double current_time)
 {
-	ReaderBackend::DoHeartbeat(network_time, current_time);
-
 	switch ( Info().mode  ) {
 		case MODE_MANUAL:
 			// yay, we do nothing :)

src/input/readers/Benchmark.cc

@@ -38,7 +38,7 @@ void Benchmark::DoClose()
 
 bool Benchmark::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields)
 	{
-	num_lines = atoi(info.source.c_str());
+	num_lines = atoi(info.source);
 
 	if ( autospread != 0.0 )
 		autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) );
@@ -126,8 +126,12 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 		assert(false); // no enums, please.
 
 	case TYPE_STRING:
-		val->val.string_val = new string(RandomString(10));
+		{
+		string rnd = RandomString(10);
+		val->val.string_val.data = copy_string(rnd.c_str());
+		val->val.string_val.length = rnd.size();
 		break;
+		}
 
 	case TYPE_BOOL:
 		val->val.int_val = 1; // we never lie.
@@ -222,7 +226,6 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 
 bool Benchmark::DoHeartbeat(double network_time, double current_time)
 {
-	ReaderBackend::DoHeartbeat(network_time, current_time);
 	num_lines = (int) ( (double) num_lines*multiplication_factor);
 	num_lines += add;
 	heartbeatstarttime = CurrTime();

src/input/readers/Raw.cc

@@ -108,7 +108,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
 	firstrun = true;
 	bool result;
 
-	if ( info.source.length() == 0 )
+	if ( ! info.source || strlen(info.source) == 0 )
 		{
 		Error("No source path provided");
 		return false;
@@ -129,11 +129,12 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
 		}
 
 	// do Initialization
-	char last = info.source[info.source.length()-1];
+	string source = string(info.source);
+	char last = info.source[source.length() - 1];
 	if ( last == '|' )
 		{
 		execute = true;
-		fname = info.source.substr(0, fname.length() - 1);
+		fname = source.substr(0, fname.length() - 1);
 
 		if ( (info.mode != MODE_MANUAL) )
 			{
@@ -237,7 +238,8 @@ bool Raw::DoUpdate()
 
 		// filter has exactly one text field. convert to it.
 		Value* val = new Value(TYPE_STRING, true);
-		val->val.string_val = new string(line);
+		val->val.string_val.data = copy_string(line.c_str());
+		val->val.string_val.length = line.size();
 		fields[0] = val;
 
 		Put(fields);
@@ -252,8 +254,6 @@ bool Raw::DoUpdate()
 
 bool Raw::DoHeartbeat(double network_time, double current_time)
 	{
-	ReaderBackend::DoHeartbeat(network_time, current_time);
-
 	switch ( Info().mode ) {
 		case MODE_MANUAL:
 			// yay, we do nothing :)

src/logging.bif

@@ -65,8 +65,8 @@ function Log::__flush%(id: Log::ID%): bool
 module LogAscii;
 
 const output_to_stdout: bool;
-const include_header: bool;
+const include_meta: bool;
-const header_prefix: string;
+const meta_prefix: string;
 const separator: string;
 const set_separator: string;
 const empty_field: string;
@@ -82,6 +82,20 @@ const dump_schema: bool;
 const use_integer_for_time: bool;
 const num_threads: count;
 
+# Options for the ElasticSearch writer.
+
+module LogElasticSearch;
+
+const cluster_name: string;
+const server_host: string;
+const server_port: count;
+const index_prefix: string;
+const type_prefix: string;
+const transfer_timeout: interval;
+const max_batch_size: count;
+const max_batch_interval: interval;
+const max_byte_size: count;
+
 # Options for the None writer.
 
 module LogNone;

src/logging/Manager.cc

@@ -6,6 +6,7 @@
 #include "../EventHandler.h"
 #include "../NetVar.h"
 #include "../Net.h"
+#include "../Type.h"
 
 #include "threading/Manager.h"
 #include "threading/SerialTypes.h"
@@ -17,6 +18,10 @@
 #include "writers/Ascii.h"
 #include "writers/None.h"
 
+#ifdef USE_ELASTICSEARCH
+#include "writers/ElasticSearch.h"
+#endif
+
 #ifdef USE_DATASERIES
 #include "writers/DataSeries.h"
 #endif
@@ -35,6 +40,11 @@ struct WriterDefinition {
 WriterDefinition log_writers[] = {
 	{ BifEnum::Log::WRITER_NONE,  "None", 0, writer::None::Instantiate },
 	{ BifEnum::Log::WRITER_ASCII, "Ascii", 0, writer::Ascii::Instantiate },
+
+#ifdef USE_ELASTICSEARCH
+	{ BifEnum::Log::WRITER_ELASTICSEARCH, "ElasticSearch", 0, writer::ElasticSearch::Instantiate },
+#endif
+
 #ifdef USE_DATASERIES
 	{ BifEnum::Log::WRITER_DATASERIES, "DataSeries", 0, writer::DataSeries::Instantiate },
 #endif
@@ -75,7 +85,7 @@ struct Manager::WriterInfo {
 	double interval;
 	Func* postprocessor;
 	WriterFrontend* writer;
-	WriterBackend::WriterInfo info;
+	WriterBackend::WriterInfo* info;
 	};
 
 struct Manager::Stream {
@@ -118,6 +128,7 @@ Manager::Stream::~Stream()
 
 		Unref(winfo->type);
 		delete winfo->writer;
+		delete winfo->info;
 		delete winfo;
 		}
 
@@ -193,7 +204,6 @@ WriterBackend* Manager::CreateBackend(WriterFrontend* frontend, bro_int_t type)
 
 	assert(ld->factory);
 
-	frontend->ty_name = ld->name;
 	WriterBackend* backend = (*ld->factory)(frontend);
 	assert(backend);
 
@@ -476,18 +486,17 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
 			return false;
 			}
 
-		threading::Field* field = new threading::Field();
+		TypeTag st = TYPE_VOID;
-		field->name = new_path;
-		field->type = t->Tag();
-		field->optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
 
-		if ( field->type == TYPE_TABLE )
+		if ( t->Tag() == TYPE_TABLE )
-			field->subtype = t->AsSetType()->Indices()->PureType()->Tag();
+			st = t->AsSetType()->Indices()->PureType()->Tag();
 
-		else if ( field->type == TYPE_VECTOR )
+		else if ( t->Tag() == TYPE_VECTOR )
-			field->subtype = t->AsVectorType()->YieldType()->Tag();
+			st = t->AsVectorType()->YieldType()->Tag();
 
-		filter->fields[filter->num_fields - 1] = field;
+		bool optional = rt->FieldDecl(i)->FindAttr(ATTR_OPTIONAL);
+
+		filter->fields[filter->num_fields - 1] = new threading::Field(new_path.c_str(), 0, t->Tag(), st, optional);
 		}
 
 	return true;
@@ -594,7 +603,7 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
 		{
 		threading::Field* field = filter->fields[i];
 		DBG_LOG(DBG_LOGGING, "   field %10s: %s",
-			field->name.c_str(), type_name(field->type));
+			field->name, type_name(field->type));
 		}
 #endif
 
@@ -769,8 +778,9 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 			for ( int j = 0; j < filter->num_fields; ++j )
 				arg_fields[j] = new threading::Field(*filter->fields[j]);
 
-			WriterBackend::WriterInfo info;
+			WriterBackend::WriterInfo* info = new WriterBackend::WriterInfo;
-			info.path = path;
+			info->path = copy_string(path.c_str());
+			info->network_time = network_time;
 
 			HashKey* k;
 			IterCookie* c = filter->config->AsTable()->InitForIteration();
@@ -781,7 +791,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 				ListVal* index = filter->config->RecoverIndex(k);
 				string key = index->Index(0)->AsString()->CheckString();
 				string value = v->Value()->AsString()->CheckString();
-				info.config.insert(std::make_pair(key, value));
+				info->config.insert(std::make_pair(copy_string(key.c_str()), copy_string(value.c_str())));
 				Unref(index);
 				delete k;
 				}
@@ -843,11 +853,16 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 			val->Type()->AsEnumType()->Lookup(val->InternalInt());
 
 		if ( s )
-			lval->val.string_val = new string(s);
+			{
+			lval->val.string_val.data = copy_string(s);
+			lval->val.string_val.length = strlen(s);
+			}
+
 		else
 			{
 			val->Type()->Error("enum type does not contain value", val);
-			lval->val.string_val = new string();
+			lval->val.string_val.data = copy_string("");
+			lval->val.string_val.length = 0;
 			}
 		break;
 		}
@@ -879,15 +894,20 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 	case TYPE_STRING:
 		{
 		const BroString* s = val->AsString();
-		lval->val.string_val =
+		char* buf = new char[s->Len()];
-			new string((const char*) s->Bytes(), s->Len());
+		memcpy(buf, s->Bytes(), s->Len());
+
+		lval->val.string_val.data = buf;
+		lval->val.string_val.length = s->Len();
 		break;
 		}
 
 	case TYPE_FILE:
 		{
 		const BroFile* f = val->AsFile();
-		lval->val.string_val = new string(f->Name());
+		string s = f->Name();
+		lval->val.string_val.data = copy_string(s.c_str());
+		lval->val.string_val.length = s.size();
 		break;
 		}
 
@@ -896,7 +916,9 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 		ODesc d;
 		const Func* f = val->AsFunc();
 		f->Describe(&d);
-		lval->val.string_val = new string(d.Description());
+		const char* s = d.Description();
+		lval->val.string_val.data = copy_string(s);
+		lval->val.string_val.length = strlen(s);
 		break;
 		}
 
@@ -976,7 +998,7 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 	return vals;
 	}
 
-WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
+WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
 				int num_fields, const threading::Field* const*  fields, bool local, bool remote)
 	{
 	Stream* stream = FindStream(id);
@@ -986,7 +1008,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 		return 0;
 
 	Stream::WriterMap::iterator w =
-		stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info.path));
+		stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info->path));
 
 	if ( w != stream->writers.end() )
 		// If we already have a writer for this. That's fine, we just
@@ -1012,7 +1034,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 		{
 		Filter* f = *it;
 		if ( f->writer->AsEnum() == writer->AsEnum() &&
-		     f->path == info.path )
+		     f->path == info->path )
 			{
 			found_filter_match = true;
 			winfo->interval = f->interval;
@@ -1029,7 +1051,7 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 		}
 
 	stream->writers.insert(
-		Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info.path),
+		Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), info->path),
 		winfo));
 
 	// Still need to set the WriterInfo's rotation parameters, which we
@@ -1037,11 +1059,11 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, const Writer
 	const char* base_time = log_rotate_base_time ?
 		log_rotate_base_time->AsString()->CheckString() : 0;
 
-	winfo->info.rotation_interval = winfo->interval;
+	winfo->info->rotation_interval = winfo->interval;
-	winfo->info.rotation_base = parse_rotate_base_time(base_time);
+	winfo->info->rotation_base = parse_rotate_base_time(base_time);
 
-	winfo->writer = new WriterFrontend(id, writer, local, remote);
+	winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote);
-	winfo->writer->Init(winfo->info, num_fields, fields);
+	winfo->writer->Init(num_fields, fields);
 
 	InstallRotationTimer(winfo);
 
@@ -1123,7 +1145,7 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer)
 			EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer);
 			remote_serializer->SendLogCreateWriter(peer, (*s)->id,
 							       &writer_val,
-							       i->second->info,
+							       *i->second->info,
 							       writer->NumFields(),
 							       writer->Fields());
 			}
@@ -1156,7 +1178,7 @@ bool Manager::Flush(EnumVal* id)
 
 	for ( Stream::WriterMap::iterator i = stream->writers.begin();
 	      i != stream->writers.end(); i++ )
-		i->second->writer->Flush();
+		i->second->writer->Flush(network_time);
 
 	RemoveDisabledWriters(stream);
 
@@ -1259,14 +1281,14 @@ void Manager::InstallRotationTimer(WriterInfo* winfo)
 		timer_mgr->Add(winfo->rotation_timer);
 
 		DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f",
-			winfo->writer->Name().c_str(), winfo->rotation_timer->Time());
+			winfo->writer->Name(), winfo->rotation_timer->Time());
 		}
 	}
 
 void Manager::Rotate(WriterInfo* winfo)
 	{
 	DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f",
-		winfo->writer->Name().c_str(), network_time);
+		winfo->writer->Name(), network_time);
 
 	// Build a temporary path for the writer to move the file to.
 	struct tm tm;
@@ -1277,15 +1299,14 @@ void Manager::Rotate(WriterInfo* winfo)
 	localtime_r(&teatime, &tm);
 	strftime(buf, sizeof(buf), date_fmt, &tm);
 
-	string tmp = string(fmt("%s-%s", winfo->writer->Info().path.c_str(), buf));
-
 	// Trigger the rotation.
+	const char* tmp = fmt("%s-%s", winfo->writer->Info().path, buf);
 	winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
 
 	++rotations_pending;
 	}
 
-bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
+bool Manager::FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
 		      double open, double close, bool terminating)
 	{
 	--rotations_pending;
@@ -1295,7 +1316,7 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
 		return true;
 
 	DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
-		writer->Name().c_str(), network_time, new_name.c_str());
+		writer->Name(), network_time, new_name);
 
 	WriterInfo* winfo = FindWriter(writer);
 	if ( ! winfo )
@@ -1304,8 +1325,8 @@ bool Manager::FinishedRotation(WriterFrontend* writer, string new_name, string o
 	// Create the RotationInfo record.
 	RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
 	info->Assign(0, winfo->type->Ref());
-	info->Assign(1, new StringVal(new_name.c_str()));
+	info->Assign(1, new StringVal(new_name));
-	info->Assign(2, new StringVal(winfo->writer->Info().path.c_str()));
+	info->Assign(2, new StringVal(winfo->writer->Info().path));
 	info->Assign(3, new Val(open, TYPE_TIME));
 	info->Assign(4, new Val(close, TYPE_TIME));
 	info->Assign(5, new Val(terminating, TYPE_BOOL));

src/logging/Manager.h

@@ -162,8 +162,8 @@ protected:
 
 	//// Function also used by the RemoteSerializer.
 
-	// Takes ownership of fields.
+	// Takes ownership of fields and info.
-	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, const WriterBackend::WriterInfo& info,
+	WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info,
 				int num_fields, const threading::Field* const* fields,
 				bool local, bool remote);
 
@@ -175,7 +175,7 @@ protected:
 	void SendAllWritersTo(RemoteSerializer::PeerID peer);
 
 	// Signals that a file has been rotated.
-	bool FinishedRotation(WriterFrontend* writer, string new_name, string old_name,
+	bool FinishedRotation(WriterFrontend* writer, const char* new_name, const char* old_name,
 			      double open, double close, bool terminating);
 
 	// Deletes the values as passed into Write().

src/logging/WriterBackend.cc

@@ -18,20 +18,26 @@ namespace logging  {
 class RotationFinishedMessage : public threading::OutputMessage<WriterFrontend>
 {
 public:
-        RotationFinishedMessage(WriterFrontend* writer, string new_name, string old_name,
+	RotationFinishedMessage(WriterFrontend* writer, const char* new_name, const char* old_name,
 				double open, double close, bool terminating)
 		: threading::OutputMessage<WriterFrontend>("RotationFinished", writer),
-		new_name(new_name), old_name(old_name), open(open),
+		new_name(copy_string(new_name)), old_name(copy_string(old_name)), open(open),
 		close(close), terminating(terminating)	{ }
 
+	virtual ~RotationFinishedMessage()
+		{
+		delete [] new_name;
+		delete [] old_name;
+		}
+
 	virtual bool Process()
 		{
 		return log_mgr->FinishedRotation(Object(), new_name, old_name, open, close, terminating);
 		}
 
 private:
-        string new_name;
+        const char* new_name;
-        string old_name;
+        const char* old_name;
         double open;
         double close;
         bool terminating;
@@ -65,12 +71,16 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
 	{
 	int size;
 
-	if ( ! (fmt->Read(&path, "path") &&
+	string tmp_path;
+
+	if ( ! (fmt->Read(&tmp_path, "path") &&
 		fmt->Read(&rotation_base, "rotation_base") &&
 		fmt->Read(&rotation_interval, "rotation_interval") &&
 		fmt->Read(&size, "config_size")) )
 		return false;
 
+	path = copy_string(tmp_path.c_str());
+
 	config.clear();
 
 	while ( size )
@@ -81,7 +91,7 @@ bool WriterBackend::WriterInfo::Read(SerializationFormat* fmt)
 		if ( ! (fmt->Read(&value, "config-value") && fmt->Read(&value, "config-key")) )
 			return false;
 
-		config.insert(std::make_pair(value, key));
+		config.insert(std::make_pair(copy_string(value.c_str()), copy_string(key.c_str())));
 		}
 
 	return true;
@@ -113,8 +123,7 @@ WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread()
 	fields = 0;
 	buffering = true;
 	frontend = arg_frontend;
-
+	info = new WriterInfo(frontend->Info());
-	info.path = "<path not yet set>";
 
 	SetName(frontend->Name());
 	}
@@ -128,6 +137,8 @@ WriterBackend::~WriterBackend()
 
 		delete [] fields;
 		}
+
+	delete info;
 	}
 
 void WriterBackend::DeleteVals(int num_writes, Value*** vals)
@@ -144,7 +155,7 @@ void WriterBackend::DeleteVals(int num_writes, Value*** vals)
 	delete [] vals;
 	}
 
-bool WriterBackend::FinishedRotation(string new_name, string old_name,
+bool WriterBackend::FinishedRotation(const char* new_name, const char* old_name,
 				     double open, double close, bool terminating)
 	{
 	SendOut(new RotationFinishedMessage(frontend, new_name, old_name, open, close, terminating));
@@ -156,17 +167,12 @@ void WriterBackend::DisableFrontend()
 	SendOut(new DisableMessage(frontend));
 	}
 
-bool WriterBackend::Init(const WriterInfo& arg_info, int arg_num_fields, const Field* const* arg_fields, const string& frontend_name)
+bool WriterBackend::Init(int arg_num_fields, const Field* const* arg_fields)
 	{
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
-	string name = Fmt("%s/%s", info.path.c_str(), frontend_name.c_str());
+	if ( ! DoInit(*info, arg_num_fields, arg_fields) )
-
-	SetName(name);
-
-	if ( ! DoInit(arg_info, arg_num_fields, arg_fields) )
 		{
 		DisableFrontend();
 		return false;
@@ -248,7 +254,7 @@ bool WriterBackend::SetBuf(bool enabled)
 	return true;
 	}
 
-bool WriterBackend::Rotate(string rotated_path, double open,
+bool WriterBackend::Rotate(const char* rotated_path, double open,
 			   double close, bool terminating)
 	{
 	if ( ! DoRotate(rotated_path, open, close, terminating) )
@@ -260,9 +266,9 @@ bool WriterBackend::Rotate(string rotated_path, double open,
 	return true;
 	}
 
-bool WriterBackend::Flush()
+bool WriterBackend::Flush(double network_time)
 	{
-	if ( ! DoFlush() )
+	if ( ! DoFlush(network_time) )
 		{
 		DisableFrontend();
 		return false;
@@ -271,13 +277,15 @@ bool WriterBackend::Flush()
 	return true;
 	}
 
-bool WriterBackend::DoHeartbeat(double network_time, double current_time)
+bool WriterBackend::OnFinish(double network_time)
 	{
-	MsgThread::DoHeartbeat(network_time, current_time);
+	return DoFinish(network_time);
+	}
 
+bool WriterBackend::OnHeartbeat(double network_time, double current_time)
+	{
 	SendOut(new FlushWriteBufferMessage(frontend));
-
+	return DoHeartbeat(network_time, current_time);
-	return true;
 	}
 
 string WriterBackend::Render(const threading::Value::addr_t& addr) const

src/logging/WriterBackend.h

@@ -48,14 +48,17 @@ public:
 	 */
 	struct WriterInfo
 		{
-		typedef std::map<string, string> config_map;
+		// Structure takes ownership of these strings.
+		typedef std::map<const char*, const char*> config_map;
 
 		/**
 		 * A string left to the interpretation of the writer
 		 * implementation; it corresponds to the 'path' value configured
 		 * on the script-level for the logging filter.
+		 *
+		 * Structure takes ownership of string.
 		 */
-		string path;
+		const char* path;
 
 		/**
 		 * The rotation interval as configured for this writer.
@@ -67,13 +70,47 @@ public:
 		 */
 		double rotation_base;
 
+		/**
+		 * The network time when the writer is created.
+		 */
+		double network_time;
+
 		/**
 		 * A map of key/value pairs corresponding to the relevant
 		 * filter's "config" table.
 		 */
-		std::map<string, string> config;
+		config_map config;
+
+		WriterInfo()
+			{
+			path = 0;
+			}
+
+		WriterInfo(const WriterInfo& other)
+			{
+			path = other.path ? copy_string(other.path) : 0;
+			rotation_interval = other.rotation_interval;
+			rotation_base = other.rotation_base;
+			network_time = other.network_time;
+
+			for ( config_map::const_iterator i = other.config.begin(); i != other.config.end(); i++ )
+				config.insert(std::make_pair(copy_string(i->first), copy_string(i->second)));
+			}
+
+		~WriterInfo()
+			{
+			delete [] path;
+
+			for ( config_map::iterator i = config.begin(); i != config.end(); i++ )
+				{
+				delete [] i->first;
+				delete [] i->second;
+				}
+			}
 
 		private:
+		const WriterInfo& operator=(const WriterInfo& other); // Disable.
+
 		friend class ::RemoteSerializer;
 
 		// Note, these need to be adapted when changing the struct's
@@ -85,7 +122,6 @@ public:
 	/**
 	 * One-time initialization of the writer to define the logged fields.
 	 *
-	 * @param info Meta information for the writer.
 	 * @param num_fields
 	 *
 	 * @param fields An array of size \a num_fields with the log fields.
@@ -95,7 +131,7 @@ public:
 	 *
 	 * @return False if an error occured.
 	 */
-	bool Init(const WriterInfo& info, int num_fields, const threading::Field* const* fields, const string& frontend_name);
+	bool Init(int num_fields, const threading::Field* const* fields);
 
 	/**
 	 * Writes one log entry.
@@ -129,9 +165,11 @@ public:
 	 * Flushes any currently buffered output, assuming the writer
 	 * supports that. (If not, it will be ignored).
 	 *
+	 * @param network_time The network time when the flush was triggered.
+	 *
 	 * @return False if an error occured.
 	 */
-	bool Flush();
+	bool Flush(double network_time);
 
 	/**
 	 * Triggers rotation, if the writer supports that. (If not, it will
@@ -139,7 +177,7 @@ public:
 	 *
 	 * @return False if an error occured.
 	 */
-	bool Rotate(string rotated_path, double open, double close, bool terminating);
+	bool Rotate(const char* rotated_path, double open, double close, bool terminating);
 
 	/**
 	 * Disables the frontend that has instantiated this backend. Once
@@ -150,7 +188,7 @@ public:
 	/**
 	 * Returns the additional writer information passed into the constructor.
 	 */
-	const WriterInfo& Info() const	{ return info; }
+	const WriterInfo& Info() const	{ return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -186,7 +224,7 @@ public:
 	 * @param terminating: True if the original rotation request occured
 	 * due to the main Bro process shutting down.
 	 */
-	bool FinishedRotation(string new_name, string old_name,
+	bool FinishedRotation(const char* new_name, const char* old_name,
 			      double open, double close, bool terminating);
 
 	/** Helper method to render an IP address as a string.
@@ -213,6 +251,10 @@ public:
 	  */
 	string Render(double d) const;
 
+	// Overridden from MsgThread.
+	virtual bool OnHeartbeat(double network_time, double current_time);
+	virtual bool OnFinish(double network_time);
+
 protected:
 	friend class FinishMessage;
 
@@ -272,8 +314,10 @@ protected:
 	 * will then be disabled and eventually deleted. When returning
 	 * false, an implementation should also call Error() to indicate what
 	 * happened.
+	 *
+	 * @param network_time The network time when the flush was triggered.
 	 */
-	virtual bool DoFlush() = 0;
+	virtual bool DoFlush(double network_time) = 0;
 
 	/**
 	 * Writer-specific method implementing log rotation.  Most directly
@@ -309,25 +353,24 @@ protected:
 	 * due the main Bro prcoess terminating (and not because we've
 	 * reached a regularly scheduled time for rotation).
 	 */
-	virtual bool DoRotate(string rotated_path, double open, double close,
+	virtual bool DoRotate(const char* rotated_path, double open, double close,
 			      bool terminating) = 0;
 
 	/**
 	 * Writer-specific method called just before the threading system is
-	 * going to shutdown.
+	 * going to shutdown. It is assumed that once this messages returns,
+	 * the thread can be safely terminated.
 	 *
-	 * This method can be overridden but one must call
+	 * @param network_time The network time when the finish is triggered.
-	 * WriterBackend::DoFinish().
 	 */
-	virtual bool DoFinish() { return MsgThread::DoFinish(); }
+	virtual bool DoFinish(double network_time) = 0;
-
 	/**
 	 * Triggered by regular heartbeat messages from the main thread.
 	 *
-	 * This method can be overridden but one must call
+	 * This method can be overridden. Default implementation does
-	 * WriterBackend::DoHeartbeat().
+	 * nothing.
 	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
+	virtual bool DoHeartbeat(double network_time, double current_time) = 0;
 
 private:
 	/**
@@ -339,7 +382,7 @@ private:
 	// this class, it's running in a different thread!
 	WriterFrontend* frontend;
 
-	WriterInfo info;	// Meta information as passed to Init().
+	const WriterInfo* info;	// Meta information.
 	int num_fields;	// Number of log fields.
 	const threading::Field* const*  fields;	// Log fields.
 	bool buffering;	// True if buffering is enabled.

src/logging/WriterFrontend.cc

@@ -16,35 +16,36 @@ namespace logging  {
 class InitMessage : public threading::InputMessage<WriterBackend>
 {
 public:
-	InitMessage(WriterBackend* backend, const WriterBackend::WriterInfo& info, const int num_fields, const Field* const* fields, const string& frontend_name)
+	InitMessage(WriterBackend* backend, const int num_fields, const Field* const* fields)
 		: threading::InputMessage<WriterBackend>("Init", backend),
-		info(info), num_fields(num_fields), fields(fields),
+		num_fields(num_fields), fields(fields)
-		frontend_name(frontend_name) { }
+			{}
 
-	virtual bool Process() { return Object()->Init(info, num_fields, fields, frontend_name); }
+
+	virtual bool Process() { return Object()->Init(num_fields, fields); }
 
 private:
-	WriterBackend::WriterInfo info;
 	const int num_fields;
 	const Field * const* fields;
-	const string frontend_name;
 };
 
 class RotateMessage : public threading::InputMessage<WriterBackend>
 {
 public:
-	RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const string rotated_path, const double open,
+	RotateMessage(WriterBackend* backend, WriterFrontend* frontend, const char* rotated_path, const double open,
 		      const double close, const bool terminating)
 		: threading::InputMessage<WriterBackend>("Rotate", backend),
 		frontend(frontend),
-		rotated_path(rotated_path), open(open),
+		rotated_path(copy_string(rotated_path)), open(open),
 		close(close), terminating(terminating) { }
 
+	virtual ~RotateMessage()	{ delete [] rotated_path; }
+
 	virtual bool Process() { return Object()->Rotate(rotated_path, open, close, terminating); }
 
 private:
 	WriterFrontend* frontend;
-	const string rotated_path;
+	const char* rotated_path;
 	const double open;
 	const double close;
 	const bool terminating;
@@ -81,19 +82,13 @@ private:
 class FlushMessage : public threading::InputMessage<WriterBackend>
 {
 public:
-	FlushMessage(WriterBackend* backend)
+	FlushMessage(WriterBackend* backend, double network_time)
-		: threading::InputMessage<WriterBackend>("Flush", backend)	{}
+		: threading::InputMessage<WriterBackend>("Flush", backend),
+		network_time(network_time) {}
 
-	virtual bool Process() { return Object()->Flush(); }
+	virtual bool Process() { return Object()->Flush(network_time); }
-};
+private:
-
+	double network_time;
-class FinishMessage : public threading::InputMessage<WriterBackend>
-{
-public:
-	FinishMessage(WriterBackend* backend)
-		: threading::InputMessage<WriterBackend>("Finish", backend)	{}
-
-	virtual bool Process() { return Object()->DoFinish(); }
 };
 
 }
@@ -102,7 +97,7 @@ public:
 
 using namespace logging;
 
-WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
+WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote)
 	{
 	stream = arg_stream;
 	writer = arg_writer;
@@ -115,7 +110,10 @@ WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool ar
 	remote = arg_remote;
 	write_buffer = 0;
 	write_buffer_pos = 0;
-	ty_name = "<not set>";
+	info = new WriterBackend::WriterInfo(arg_info);
+
+	const char* w = arg_writer->Type()->AsEnumType()->Lookup(arg_stream->InternalInt());
+	name = copy_string(fmt("%s/%s", arg_info.path, w));
 
 	if ( local )
 		{
@@ -133,26 +131,16 @@ WriterFrontend::~WriterFrontend()
 	{
 	Unref(stream);
 	Unref(writer);
-	}
+	delete info;
-
-string WriterFrontend::Name() const
-	{
-	if ( ! info.path.size() )
-		return ty_name;
-
-	return ty_name + "/" + info.path;
 	}
 
 void WriterFrontend::Stop()
 	{
 	FlushWriteBuffer();
 	SetDisable();
-
-	if ( backend )
-		backend->Stop();
 	}
 
-void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num_fields, const Field* const * arg_fields)
+void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields)
 	{
 	if ( disabled )
 		return;
@@ -160,19 +148,18 @@ void WriterFrontend::Init(const WriterBackend::WriterInfo& arg_info, int arg_num
 	if ( initialized )
 		reporter->InternalError("writer initialize twice");
 
-	info = arg_info;
 	num_fields = arg_num_fields;
 	fields = arg_fields;
 
 	initialized = true;
 
 	if ( backend )
-		backend->SendIn(new InitMessage(backend, arg_info, arg_num_fields, arg_fields, Name()));
+		backend->SendIn(new InitMessage(backend, arg_num_fields, arg_fields));
 
 	if ( remote )
 		remote_serializer->SendLogCreateWriter(stream,
 						       writer,
-						       arg_info,
+						       *info,
 						       arg_num_fields,
 						       arg_fields);
 
@@ -186,7 +173,7 @@ void WriterFrontend::Write(int num_fields, Value** vals)
 	if ( remote )
 		remote_serializer->SendLogWrite(stream,
 						writer,
-						info.path,
+						info->path,
 						num_fields,
 						vals);
 
@@ -240,7 +227,7 @@ void WriterFrontend::SetBuf(bool enabled)
 		FlushWriteBuffer();
 	}
 
-void WriterFrontend::Flush()
+void WriterFrontend::Flush(double network_time)
 	{
 	if ( disabled )
 		return;
@@ -248,10 +235,10 @@ void WriterFrontend::Flush()
 	FlushWriteBuffer();
 
 	if ( backend )
-		backend->SendIn(new FlushMessage(backend));
+		backend->SendIn(new FlushMessage(backend, network_time));
 	}
 
-void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating)
+void WriterFrontend::Rotate(const char* rotated_path, double open, double close, bool terminating)
 	{
 	if ( disabled )
 		return;
@@ -266,17 +253,6 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool
 		log_mgr->FinishedRotation(0, "", rotated_path, open, close, terminating);
 	}
 
-void WriterFrontend::Finish()
-	{
-	if ( disabled )
-		return;
-
-	FlushWriteBuffer();
-
-	if ( backend )
-		backend->SendIn(new FinishMessage(backend));
-	}
-
 void WriterFrontend::DeleteVals(Value** vals)
 	{
 	// Note this code is duplicated in Manager::DeleteVals().

src/logging/WriterFrontend.h

@@ -32,6 +32,10 @@ public:
 	 * frontend will internally instantiate a WriterBackend of the
 	 * corresponding type.
 	 *
+	 * info: The meta information struct for the writer.
+	 *
+	 * writer_name: A descriptive name for the writer's type.
+	 *
 	 * local: If true, the writer will instantiate a local backend.
 	 *
 	 * remote: If true, the writer will forward all data to remote
@@ -39,7 +43,7 @@ public:
 	 *
 	 * Frontends must only be instantiated by the main thread.
 	 */
-	WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote);
+	WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote);
 
 	/**
 	 * Destructor.
@@ -50,7 +54,7 @@ public:
 
 	/**
 	 * Stops all output to this writer. Calling this methods disables all
-	 * message forwarding to the backend and stops the backend thread.
+	 * message forwarding to the backend.
 	 *
 	 * This method must only be called from the main thread.
 	 */
@@ -68,7 +72,7 @@ public:
 	 *
 	 * This method must only be called from the main thread.
 	 */
-	void Init(const WriterBackend::WriterInfo& info, int num_fields, const threading::Field* const*  fields);
+	void Init(int num_fields, const threading::Field* const*  fields);
 
 	/**
 	 * Write out a record.
@@ -114,8 +118,10 @@ public:
 	 * message back that will asynchronously call Disable().
 	 *
 	 * This method must only be called from the main thread.
+	 *
+	 * @param network_time The network time when the flush was triggered.
 	 */
-	void Flush();
+	void Flush(double network_time);
 
 	/**
 	 * Triggers log rotation.
@@ -128,7 +134,7 @@ public:
 	 *
 	 * This method must only be called from the main thread.
 	 */
-	void Rotate(string rotated_path, double open, double close, bool terminating);
+	void Rotate(const char* rotated_path, double open, double close, bool terminating);
 
 	/**
 	 * Finalizes writing to this tream.
@@ -138,8 +144,10 @@ public:
 	 * sends a message back that will asynchronously call Disable().
 	 *
 	 * This method must only be called from the main thread.
+	 *
+	 * @param network_time The network time when the finish was triggered.
 	 */
-	void Finish();
+	void Finish(double network_time);
 
 	/**
 	 * Explicitly triggers a transfer of all potentially buffered Write()
@@ -171,7 +179,7 @@ public:
 	/**
 	 * Returns the additional writer information as passed into the constructor.
 	 */
-	const WriterBackend::WriterInfo& Info() const	{ return info; }
+	const WriterBackend::WriterInfo& Info() const	{ return *info; }
 
 	/**
 	 * Returns the number of log fields as passed into the constructor.
@@ -184,7 +192,7 @@ public:
 	 *
 	 * This method is safe to call from any thread.
 	 */
-	string Name() const;
+	const char* Name() const	{ return name; }
 
 	/**
 	 * Returns the log fields as passed into the constructor.
@@ -206,8 +214,8 @@ protected:
 	bool local;	// True if logging locally.
 	bool remote;	// True if loggin remotely.
 
-	string ty_name;	// Name of the backend type. Set by the manager.
+	const char* name;	// Descriptive name of the
-	WriterBackend::WriterInfo info;	// The writer information.
+	WriterBackend::WriterInfo* info;	// The writer information.
 	int num_fields;	// The number of log fields.
 	const threading::Field* const*  fields;	// The log fields.
 

src/logging/writers/Ascii.cc

@@ -2,6 +2,7 @@
 
 #include <string>
 #include <errno.h>
+#include <fcntl.h>
 
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
@@ -15,10 +16,11 @@ using threading::Field;
 
 Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	{
-	file = 0;
+	fd = 0;
+	ascii_done = false;
 
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
-	include_header = BifConst::LogAscii::include_header;
+	include_meta = BifConst::LogAscii::include_meta;
 
 	separator_len = BifConst::LogAscii::separator->Len();
 	separator = new char[separator_len];
@@ -40,10 +42,10 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(),
 	       unset_field_len);
 
-	header_prefix_len = BifConst::LogAscii::header_prefix->Len();
+	meta_prefix_len = BifConst::LogAscii::meta_prefix->Len();
-	header_prefix = new char[header_prefix_len];
+	meta_prefix = new char[meta_prefix_len];
-	memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(),
+	memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(),
-	       header_prefix_len);
+	       meta_prefix_len);
 
 	desc.EnableEscaping();
 	desc.AddEscapeSequence(separator, separator_len);
@@ -51,26 +53,46 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 
 Ascii::~Ascii()
 	{
-	if ( file )
+	if ( ! ascii_done )
-		fclose(file);
+		{
+		fprintf(stderr, "internal error: finish missing\n");
+		abort();
+		}
 
 	delete [] separator;
 	delete [] set_separator;
 	delete [] empty_field;
 	delete [] unset_field;
-	delete [] header_prefix;
+	delete [] meta_prefix;
 	}
 
 bool Ascii::WriteHeaderField(const string& key, const string& val)
 	{
-	string str = string(header_prefix, header_prefix_len) +
+	string str = string(meta_prefix, meta_prefix_len) +
 		key + string(separator, separator_len) + val + "\n";
 
-	return (fwrite(str.c_str(), str.length(), 1, file) == 1);
+	return safe_write(fd, str.c_str(), str.length());
+	}
+
+void Ascii::CloseFile(double t)
+	{
+	if ( ! fd )
+		return;
+
+	if ( include_meta )
+		{
+		string ts = t ? Timestamp(t) : string("<abnormal termination>");
+		WriteHeaderField("end", ts);
+		}
+
+	close(fd);
+	fd = 0;
 	}
 
 bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const * fields)
 	{
+    assert(! fd);
+
 	string path = info.path;
 
 	if ( output_to_stdout )
@@ -78,34 +100,39 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 
 	fname = IsSpecial(path) ? path : path + "." + LogExt();
 
-	if ( ! (file = fopen(fname.c_str(), "w")) )
+	fd = open(fname.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0777);
+
+	if ( fd < 0 )
 		{
 		Error(Fmt("cannot open %s: %s", fname.c_str(),
-			  strerror(errno)));
+			  Strerror(errno)));
-
+		fd = 0;
 		return false;
 		}
 
-	if ( include_header )
+	if ( include_meta )
 		{
 		string names;
 		string types;
 
-		string str = string(header_prefix, header_prefix_len)
+		string str = string(meta_prefix, meta_prefix_len)
 			+ "separator " // Always use space as separator here.
 			+ get_escaped_string(string(separator, separator_len), false)
 			+ "\n";
 
-		if( fwrite(str.c_str(), str.length(), 1, file) != 1 )
+		if ( ! safe_write(fd, str.c_str(), str.length()) )
 			goto write_error;
 
+		string ts = Timestamp(info.network_time);
+
 		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
 		            string(set_separator, set_separator_len), false)) &&
 		        WriteHeaderField("empty_field", get_escaped_string(
 		            string(empty_field, empty_field_len), false)) &&
 		        WriteHeaderField("unset_field", get_escaped_string(
 		            string(unset_field, unset_field_len), false)) &&
-		        WriteHeaderField("path", get_escaped_string(path, false))) )
+		        WriteHeaderField("path", get_escaped_string(path, false)) &&
+		        WriteHeaderField("start", ts)) )
 			goto write_error;
 
 		for ( int i = 0; i < num_fields; ++i )
@@ -116,8 +143,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 				types += string(separator, separator_len);
 				}
 
-			names += fields[i]->name;
+			names += string(fields[i]->name);
-			types += fields[i]->TypeName();
+			types += fields[i]->TypeName().c_str();
 			}
 
 		if ( ! (WriteHeaderField("fields", names)
@@ -128,21 +155,32 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 	return true;
 
 write_error:
-	Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
+	Error(Fmt("error writing to %s: %s", fname.c_str(), Strerror(errno)));
 	return false;
 	}
 
-bool Ascii::DoFlush()
+bool Ascii::DoFlush(double network_time)
 	{
-	fflush(file);
+	fsync(fd);
 	return true;
 	}
 
-bool Ascii::DoFinish()
+bool Ascii::DoFinish(double network_time)
 	{
-	return WriterBackend::DoFinish();
+	if ( ascii_done )
+		{
+		fprintf(stderr, "internal error: duplicate finish\n");
+		abort();
 		}
 
+	ascii_done = true;
+
+	CloseFile(network_time);
+
+	return true;
+	}
+
+
 bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 	{
 	if ( ! val->present )
@@ -198,8 +236,8 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 	case TYPE_FILE:
 	case TYPE_FUNC:
 		{
-		int size = val->val.string_val->size();
+		int size = val->val.string_val.length;
-		const char* data = val->val.string_val->data();
+		const char* data = val->val.string_val.data;
 
 		if ( ! size )
 			{
@@ -280,8 +318,7 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 		}
 
 	default:
-		Error(Fmt("unsupported field format %d for %s", val->type,
+		Error(Fmt("unsupported field format %d for %s", val->type, field->name));
-			  field->name.c_str()));
 		return false;
 	}
 
@@ -291,7 +328,7 @@ bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
 bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
-	if ( ! file )
+	if ( ! fd )
 		DoInit(Info(), NumFields(), Fields());
 
 	desc.Clear();
@@ -307,31 +344,47 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 
 	desc.AddRaw("\n", 1);
 
-	if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 )
+	const char* bytes = (const char*)desc.Bytes();
+	int len = desc.Len();
+
+	if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 )
 		{
-		Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
+		// It would so escape the first character.
+		char buf[16];
+		snprintf(buf, sizeof(buf), "\\x%02x", bytes[0]);
+
+		if ( ! safe_write(fd, buf, strlen(buf)) )
+			goto write_error;
+
+		++bytes;
+		--len;
+		}
+
+	if ( ! safe_write(fd, bytes, len) )
+		goto write_error;
+
+        if ( IsBuf() )
+		fsync(fd);
+
+	return true;
+
+write_error:
+	Error(Fmt("error writing to %s: %s", fname.c_str(), Strerror(errno)));
 	return false;
 	}
 
-	if ( IsBuf() )
+bool Ascii::DoRotate(const char* rotated_path, double open, double close, bool terminating)
-		fflush(file);
-
-	return true;
-	}
-
-bool Ascii::DoRotate(string rotated_path, double open, double close, bool terminating)
 	{
 	// Don't rotate special files or if there's not one currently open.
-	if ( ! file || IsSpecial(Info().path) )
+	if ( ! fd || IsSpecial(Info().path) )
 		return true;
 
-	fclose(file);
+	CloseFile(close);
-	file = 0;
 
-	string nname = rotated_path + "." + LogExt();
+	string nname = string(rotated_path) + "." + LogExt();
 	rename(fname.c_str(), nname.c_str());
 
-	if ( ! FinishedRotation(nname, fname, open, close, terminating) )
+	if ( ! FinishedRotation(nname.c_str(), fname.c_str(), open, close, terminating) )
 		{
 		Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
 		return false;
@@ -346,9 +399,40 @@ bool Ascii::DoSetBuf(bool enabled)
 	return true;
 	}
 
+bool Ascii::DoHeartbeat(double network_time, double current_time)
+	{
+	// Nothing to do.
+	return true;
+	}
+
 string Ascii::LogExt()
 	{
 	const char* ext = getenv("BRO_LOG_SUFFIX");
-	if ( ! ext ) ext = "log";
+	if ( ! ext )
+		ext = "log";
+
 	return ext;
 	}
+
+string Ascii::Timestamp(double t)
+	{
+#if 1
+	return "2012-01-01-00-00-00";
+#else
+    // Using the version below leads to occasional crashes at least on Mac OS.
+    // Not sure why, all the function should be thread-safe ...
+
+	time_t teatime = time_t(t);
+
+	struct tm tmbuf;
+	struct tm* tm = localtime_r(&teatime, &tmbuf);
+
+        char tmp[128];
+	const char* const date_fmt = "%Y-%m-%d-%H-%M-%S";
+	strftime(tmp, sizeof(tmp), date_fmt, tm);
+
+	return tmp;
+#endif
+	}
+
+

src/logging/writers/Ascii.h

@@ -24,23 +24,27 @@ protected:
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, double open,
+	virtual bool DoRotate(const char* rotated_path, double open,
 			      double close, bool terminating);
-	virtual bool DoFlush();
+	virtual bool DoFlush(double network_time);
-	virtual bool DoFinish();
+	virtual bool DoFinish(double network_time);
+	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
 	bool IsSpecial(string path) 	{ return path.find("/dev/") == 0; }
 	bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
 	bool WriteHeaderField(const string& key, const string& value);
+	void CloseFile(double t);
+	string Timestamp(double t);
 
-	FILE* file;
+	int fd;
 	string fname;
 	ODesc desc;
+	bool ascii_done;
 
 	// Options set from the script-level.
 	bool output_to_stdout;
-	bool include_header;
+	bool include_meta;
 
 	char* separator;
 	int separator_len;
@@ -54,8 +58,8 @@ private:
 	char* unset_field;
 	int unset_field_len;
 
-	char* header_prefix;
+	char* meta_prefix;
-	int header_prefix_len;
+	int meta_prefix_len;
 };
 
 }

src/logging/writers/DataSeries.cc

@@ -78,10 +78,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
 	case TYPE_STRING:
 	case TYPE_FILE:
 	case TYPE_FUNC:
-		if ( ! val->val.string_val->size() )
+		if ( ! val->val.string_val.length )
 			return "";
 
-		return string(val->val.string_val->data(), val->val.string_val->size());
+		return string(val->val.string_val.data, val->val.string_val.length);
 
 	case TYPE_TABLE:
 		{
@@ -302,7 +302,8 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
 
 	if( ds_dump_schema )
 		{
-		FILE* pFile = fopen ( string(info.path + ".ds.xml").c_str() , "wb" );
+		string name = string(info.path) + ".ds.xml";
+		FILE* pFile = fopen(name.c_str(), "wb" );
 
 		if( pFile )
 			{
@@ -311,7 +312,7 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
 			}
 
 		else
-			Error(Fmt("cannot dump schema: %s", strerror(errno)));
+			Error(Fmt("cannot dump schema: %s", Strerror(errno)));
 		}
 
 	compress_type = Extent::compress_all;
@@ -343,7 +344,7 @@ bool DataSeries::DoInit(const WriterInfo& info, int num_fields, const threading:
 	return OpenLog(info.path);
 	}
 
-bool DataSeries::DoFlush()
+bool DataSeries::DoFlush(double network_time)
 {
 	// Flushing is handled by DataSeries automatically, so this function
 	// doesn't do anything.
@@ -366,11 +367,10 @@ void DataSeries::CloseLog()
 	log_file = 0;
 	}
 
-bool DataSeries::DoFinish()
+bool DataSeries::DoFinish(double network_time)
 {
 	CloseLog();
-
+	return true;
-	return WriterBackend::DoFinish();
 }
 
 bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
@@ -395,17 +395,17 @@ bool DataSeries::DoWrite(int num_fields, const threading::Field* const * fields,
 	return true;
 }
 
-bool DataSeries::DoRotate(string rotated_path, double open, double close, bool terminating)
+bool DataSeries::DoRotate(const char* rotated_path, double open, double close, bool terminating)
 {
 	// Note that if DS files are rotated too often, the aggregate log
 	// size will be (much) larger.
 	CloseLog();
 
-	string dsname = Info().path + ".ds";
+	string dsname = string(Info().path) + ".ds";
-	string nname = rotated_path + ".ds";
+	string nname = string(rotated_path) + ".ds";
 	rename(dsname.c_str(), nname.c_str());
 
-	if ( ! FinishedRotation(nname, dsname, open, close, terminating) )
+	if ( ! FinishedRotation(nname.c_str(), dsname.c_str(), open, close, terminating) )
 		{
 		Error(Fmt("error rotating %s to %s", dsname.c_str(), nname.c_str()));
 		return false;
@@ -420,4 +420,9 @@ bool DataSeries::DoSetBuf(bool enabled)
 	return true;
 }
 
+bool DataSeries::DoHeartbeat(double network_time, double current_time)
+{
+	return true;
+}
+
 #endif /* USE_DATASERIES */

src/logging/writers/DataSeries.h

@@ -32,10 +32,11 @@ protected:
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, double open,
+	virtual bool DoRotate(const char* rotated_path, double open,
 			      double close, bool terminating);
-	virtual bool DoFlush();
+	virtual bool DoFlush(double network_time);
-	virtual bool DoFinish();
+	virtual bool DoFinish(double network_time);
+	virtual bool DoHeartbeat(double network_time, double current_time);
 
 private:
 	static const size_t ROW_MIN = 2048;			// Minimum extent size.

src/logging/writers/ElasticSearch.cc

@@ -0,0 +1,415 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// This is experimental code that is not yet ready for production usage.
+//
+
+#include "config.h"
+
+#ifdef USE_ELASTICSEARCH
+
+#include <string>
+#include <errno.h>
+
+#include "util.h"
+#include "BroString.h"
+
+#include "NetVar.h"
+#include "threading/SerialTypes.h"
+
+#include <curl/curl.h>
+#include <curl/easy.h>
+
+#include "ElasticSearch.h"
+
+using namespace logging;
+using namespace writer;
+using threading::Value;
+using threading::Field;
+
+ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
+	{
+	cluster_name_len = BifConst::LogElasticSearch::cluster_name->Len();
+	cluster_name = new char[cluster_name_len + 1];
+	memcpy(cluster_name, BifConst::LogElasticSearch::cluster_name->Bytes(), cluster_name_len);
+	cluster_name[cluster_name_len] = 0;
+
+	index_prefix = string((const char*) BifConst::LogElasticSearch::index_prefix->Bytes(), BifConst::LogElasticSearch::index_prefix->Len());
+
+	es_server = string(Fmt("http://%s:%d", BifConst::LogElasticSearch::server_host->Bytes(),
+	                                       (int) BifConst::LogElasticSearch::server_port));
+	bulk_url = string(Fmt("%s/_bulk", es_server.c_str()));
+
+	http_headers = curl_slist_append(NULL, "Content-Type: text/json; charset=utf-8");
+	buffer.Clear();
+	counter = 0;
+	current_index = string();
+	prev_index = string();
+	last_send = current_time();
+	failing = false;
+
+	transfer_timeout = BifConst::LogElasticSearch::transfer_timeout * 1000;
+
+	curl_handle = HTTPSetup();
+}
+
+ElasticSearch::~ElasticSearch()
+	{
+	delete [] cluster_name;
+	}
+
+bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
+	{
+	return true;
+	}
+
+bool ElasticSearch::DoFlush(double network_time)
+	{
+	BatchIndex();
+	return true;
+	}
+
+bool ElasticSearch::DoFinish(double network_time)
+	{
+	BatchIndex();
+	curl_slist_free_all(http_headers);
+	curl_easy_cleanup(curl_handle);
+	return true;
+	}
+
+bool ElasticSearch::BatchIndex()
+	{
+	curl_easy_reset(curl_handle);
+	curl_easy_setopt(curl_handle, CURLOPT_URL, bulk_url.c_str());
+	curl_easy_setopt(curl_handle, CURLOPT_POST, 1);
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t)buffer.Len());
+	curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, buffer.Bytes());
+	failing = ! HTTPSend(curl_handle);
+
+	// We are currently throwing the data out regardless of if the send failed.  Fire and forget!
+	buffer.Clear();
+	counter = 0;
+	last_send = current_time();
+
+	return true;
+	}
+
+bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val)
+	{
+	switch ( val->type )
+		{
+		// ES treats 0 as false and any other value as true so bool types go here.
+		case TYPE_BOOL:
+		case TYPE_INT:
+			b->Add(val->val.int_val);
+			break;
+
+		case TYPE_COUNT:
+		case TYPE_COUNTER:
+			{
+			// ElasticSearch doesn't seem to support unsigned 64bit ints.
+			if ( val->val.uint_val >= INT64_MAX )
+				{
+				Error(Fmt("count value too large: %" PRIu64, val->val.uint_val));
+				b->AddRaw("null", 4);
+				}
+			else
+				b->Add(val->val.uint_val);
+			break;
+			}
+
+		case TYPE_PORT:
+			b->Add(val->val.port_val.port);
+			break;
+
+		case TYPE_SUBNET:
+			b->AddRaw("\"", 1);
+			b->Add(Render(val->val.subnet_val));
+			b->AddRaw("\"", 1);
+			break;
+
+		case TYPE_ADDR:
+			b->AddRaw("\"", 1);
+			b->Add(Render(val->val.addr_val));
+			b->AddRaw("\"", 1);
+			break;
+
+		case TYPE_DOUBLE:
+		case TYPE_INTERVAL:
+			b->Add(val->val.double_val);
+			break;
+
+		case TYPE_TIME:
+			{
+			// ElasticSearch uses milliseconds for timestamps and json only
+			// supports signed ints (uints can be too large).
+			uint64_t ts = (uint64_t) (val->val.double_val * 1000);
+			if ( ts >= INT64_MAX )
+				{
+				Error(Fmt("time value too large: %" PRIu64, ts));
+				b->AddRaw("null", 4);
+				}
+			else
+				b->Add(ts);
+			break;
+			}
+
+		case TYPE_ENUM:
+		case TYPE_STRING:
+		case TYPE_FILE:
+		case TYPE_FUNC:
+			{
+			b->AddRaw("\"", 1);
+			for ( int i = 0; i < val->val.string_val.length; ++i )
+				{
+				char c = val->val.string_val.data[i];
+				// 2byte Unicode escape special characters.
+				if ( c < 32 || c > 126 || c == '\n' || c == '"' || c == '\'' || c == '\\' || c == '&' )
+					{
+					static const char hex_chars[] = "0123456789abcdef";
+					b->AddRaw("\\u00", 4);
+					b->AddRaw(&hex_chars[(c & 0xf0) >> 4], 1);
+					b->AddRaw(&hex_chars[c & 0x0f], 1);
+					}
+				else
+					b->AddRaw(&c, 1);
+				}
+			b->AddRaw("\"", 1);
+			break;
+			}
+
+		case TYPE_TABLE:
+			{
+			b->AddRaw("[", 1);
+			for ( int j = 0; j < val->val.set_val.size; j++ )
+				{
+				if ( j > 0 )
+					b->AddRaw(",", 1);
+				AddValueToBuffer(b, val->val.set_val.vals[j]);
+				}
+			b->AddRaw("]", 1);
+			break;
+			}
+
+		case TYPE_VECTOR:
+			{
+			b->AddRaw("[", 1);
+			for ( int j = 0; j < val->val.vector_val.size; j++ )
+				{
+				if ( j > 0 )
+					b->AddRaw(",", 1);
+				AddValueToBuffer(b, val->val.vector_val.vals[j]);
+				}
+			b->AddRaw("]", 1);
+			break;
+			}
+
+		default:
+			return false;
+		}
+	return true;
+	}
+
+bool ElasticSearch::AddFieldToBuffer(ODesc *b, Value* val, const Field* field)
+	{
+	if ( ! val->present )
+		return false;
+
+	b->AddRaw("\"", 1);
+	b->Add(field->name);
+	b->AddRaw("\":", 2);
+	AddValueToBuffer(b, val);
+	return true;
+	}
+
+bool ElasticSearch::DoWrite(int num_fields, const Field* const * fields,
+			     Value** vals)
+	{
+	if ( current_index.empty() )
+		UpdateIndex(network_time, Info().rotation_interval, Info().rotation_base);
+
+	// Our action line looks like:
+	buffer.AddRaw("{\"index\":{\"_index\":\"", 20);
+	buffer.Add(current_index);
+	buffer.AddRaw("\",\"_type\":\"", 11);
+	buffer.Add(Info().path);
+	buffer.AddRaw("\"}}\n", 4);
+
+	buffer.AddRaw("{", 1);
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( i > 0 && buffer.Bytes()[buffer.Len()] != ',' && vals[i]->present )
+			buffer.AddRaw(",", 1);
+		AddFieldToBuffer(&buffer, vals[i], fields[i]);
+		}
+	buffer.AddRaw("}\n", 2);
+
+	counter++;
+	if ( counter >= BifConst::LogElasticSearch::max_batch_size ||
+	     uint(buffer.Len()) >= BifConst::LogElasticSearch::max_byte_size )
+		BatchIndex();
+
+	return true;
+	}
+
+bool ElasticSearch::UpdateIndex(double now, double rinterval, double rbase)
+	{
+	if ( rinterval == 0 )
+		{
+		// if logs aren't being rotated, don't use a rotation oriented index name.
+		current_index = index_prefix;
+		}
+	else
+		{
+		double nr = calc_next_rotate(now, rinterval, rbase);
+		double interval_beginning = now - (rinterval - nr);
+
+		struct tm tm;
+		char buf[128];
+		time_t teatime = (time_t)interval_beginning;
+		localtime_r(&teatime, &tm);
+		strftime(buf, sizeof(buf), "%Y%m%d%H%M", &tm);
+
+		prev_index = current_index;
+		current_index = index_prefix + "-" + buf;
+
+		// Send some metadata about this index.
+		buffer.AddRaw("{\"index\":{\"_index\":\"@", 21);
+		buffer.Add(index_prefix);
+		buffer.AddRaw("-meta\",\"_type\":\"index\",\"_id\":\"", 30);
+		buffer.Add(current_index);
+		buffer.AddRaw("-", 1);
+		buffer.Add(Info().rotation_base);
+		buffer.AddRaw("-", 1);
+		buffer.Add(Info().rotation_interval);
+		buffer.AddRaw("\"}}\n{\"name\":\"", 13);
+		buffer.Add(current_index);
+		buffer.AddRaw("\",\"start\":", 10);
+		buffer.Add(interval_beginning);
+		buffer.AddRaw(",\"end\":", 7);
+		buffer.Add(interval_beginning+rinterval);
+		buffer.AddRaw("}\n", 2);
+		}
+
+	//printf("%s - prev:%s current:%s\n", Info().path.c_str(), prev_index.c_str(), current_index.c_str());
+	return true;
+	}
+
+
+bool ElasticSearch::DoRotate(const char* rotated_path, double open, double close, bool terminating)
+	{
+	// Update the currently used index to the new rotation interval.
+	UpdateIndex(close, Info().rotation_interval, Info().rotation_base);
+
+	// Only do this stuff if there was a previous index.
+	if ( ! prev_index.empty() )
+		{
+		// FIXME: I think this section is taking too long and causing the thread to die.
+
+		// Compress the previous index
+		//curl_easy_reset(curl_handle);
+		//curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_settings", es_server.c_str(), prev_index.c_str()));
+		//curl_easy_setopt(curl_handle, CURLOPT_CUSTOMREQUEST, "PUT");
+		//curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, "{\"index\":{\"store.compress.stored\":\"true\"}}");
+		//curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDSIZE_LARGE, (curl_off_t) 42);
+		//HTTPSend(curl_handle);
+
+		// Optimize the previous index.
+		// TODO: make this into variables.
+		//curl_easy_reset(curl_handle);
+		//curl_easy_setopt(curl_handle, CURLOPT_URL, Fmt("%s/%s/_optimize?max_num_segments=1&wait_for_merge=false", es_server.c_str(), prev_index.c_str()));
+		//HTTPSend(curl_handle);
+		}
+
+	if ( ! FinishedRotation(current_index.c_str(), prev_index.c_str(), open, close, terminating) )
+		{
+		Error(Fmt("error rotating %s to %s", prev_index.c_str(), current_index.c_str()));
+		}
+
+	return true;
+	}
+
+bool ElasticSearch::DoSetBuf(bool enabled)
+	{
+	// Nothing to do.
+	return true;
+	}
+
+bool ElasticSearch::DoHeartbeat(double network_time, double current_time)
+	{
+	if ( last_send > 0 && buffer.Len() > 0 &&
+	     current_time-last_send > BifConst::LogElasticSearch::max_batch_interval )
+		{
+		BatchIndex();
+		}
+
+	return true;
+	}
+
+
+CURL* ElasticSearch::HTTPSetup()
+	{
+	CURL* handle = curl_easy_init();
+	if ( ! handle )
+		{
+		Error("cURL did not initialize correctly.");
+		return 0;
+		}
+
+	return handle;
+	}
+
+bool ElasticSearch::HTTPReceive(void* ptr, int size, int nmemb, void* userdata)
+	{
+	//TODO: Do some verification on the result?
+	return true;
+	}
+
+bool ElasticSearch::HTTPSend(CURL *handle)
+	{
+	curl_easy_setopt(handle, CURLOPT_HTTPHEADER, http_headers);
+	curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, &logging::writer::ElasticSearch::HTTPReceive); // This gets called with the result.
+	// HTTP 1.1 likes to use chunked encoded transfers, which aren't good for speed.
+	// The best (only?) way to disable that is to just use HTTP 1.0
+	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
+
+	//curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
+
+	CURLcode return_code = curl_easy_perform(handle);
+
+	switch ( return_code )
+		{
+		case CURLE_COULDNT_CONNECT:
+		case CURLE_COULDNT_RESOLVE_HOST:
+		case CURLE_WRITE_ERROR:
+		case CURLE_RECV_ERROR:
+			{
+			if ( ! failing )
+				Error(Fmt("ElasticSearch server may not be accessible."));
+			}
+
+		case CURLE_OPERATION_TIMEDOUT:
+			{
+			if ( ! failing )
+				Warning(Fmt("HTTP operation with elasticsearch server timed out at %" PRIu64 " msecs.", transfer_timeout));
+			}
+
+		case CURLE_OK:
+			{
+			uint http_code = 0;
+			curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
+			if ( http_code == 200 )
+				// Hopefully everything goes through here.
+				return true;
+			else if ( ! failing )
+				Error(Fmt("Received a non-successful status code back from ElasticSearch server, check the elasticsearch server log."));
+			}
+
+		default:
+			{
+			}
+		}
+		// The "successful" return happens above
+		return false;
+	}
+
+#endif

src/logging/writers/ElasticSearch.h

@@ -0,0 +1,81 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Log writer for writing to an ElasticSearch database
+//
+// This is experimental code that is not yet ready for production usage.
+//
+
+#ifndef LOGGING_WRITER_ELASTICSEARCH_H
+#define LOGGING_WRITER_ELASTICSEARCH_H
+
+#include <curl/curl.h>
+#include "../WriterBackend.h"
+
+namespace logging { namespace writer {
+
+class ElasticSearch : public WriterBackend {
+public:
+	ElasticSearch(WriterFrontend* frontend);
+	~ElasticSearch();
+
+	static WriterBackend* Instantiate(WriterFrontend* frontend)
+		{ return new ElasticSearch(frontend); }
+	static string LogExt();
+
+protected:
+	// Overidden from WriterBackend.
+
+	virtual bool DoInit(const WriterInfo& info, int num_fields,
+			    const threading::Field* const* fields);
+
+	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
+			     threading::Value** vals);
+	virtual bool DoSetBuf(bool enabled);
+	virtual bool DoRotate(const char* rotated_path, double open,
+			      double close, bool terminating);
+	virtual bool DoFlush(double network_time);
+	virtual bool DoFinish(double network_time);
+	virtual bool DoHeartbeat(double network_time, double current_time);
+
+private:
+	bool AddFieldToBuffer(ODesc *b, threading::Value* val, const threading::Field* field);
+	bool AddValueToBuffer(ODesc *b, threading::Value* val);
+	bool BatchIndex();
+	bool SendMappings();
+	bool UpdateIndex(double now, double rinterval, double rbase);
+
+	CURL* HTTPSetup();
+	bool HTTPReceive(void* ptr, int size, int nmemb, void* userdata);
+	bool HTTPSend(CURL *handle);
+
+	// Buffers, etc.
+	ODesc buffer;
+	uint64 counter;
+	double last_send;
+	string current_index;
+	string prev_index;
+
+	CURL* curl_handle;
+
+	// From scripts
+	char* cluster_name;
+	int cluster_name_len;
+
+	string es_server;
+	string bulk_url;
+
+	struct curl_slist *http_headers;
+
+	string path;
+	string index_prefix;
+	uint64 transfer_timeout;
+	bool failing;
+
+	uint64 batch_size;
+};
+
+}
+}
+
+
+#endif

src/logging/writers/None.cc

@@ -1,4 +1,6 @@
 
+#include <algorithm>
+
 #include "None.h"
 #include "NetVar.h"
 
@@ -15,8 +17,17 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
 		std::cout << "  rotation_interval=" << info.rotation_interval << std::endl;
 		std::cout << "  rotation_base=" << info.rotation_base << std::endl;
 
-		for ( std::map<string,string>::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
+		// Output the config sorted by keys.
-			std::cout << "  config[" << i->first << "] = " << i->second << std::endl;
+
+		std::vector<std::pair<string, string> > keys;
+
+		for ( WriterInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ )
+			keys.push_back(std::make_pair(i->first, i->second));
+
+		std::sort(keys.begin(), keys.end());
+
+		for ( std::vector<std::pair<string,string> >::const_iterator i = keys.begin(); i != keys.end(); i++ )
+			std::cout << "  config[" << (*i).first << "] = " << (*i).second << std::endl;
 
 		for ( int i = 0; i < num_fields; i++ )
 			{
@@ -31,11 +42,11 @@ bool None::DoInit(const WriterInfo& info, int num_fields,
 	return true;
 	}
 
-bool None::DoRotate(string rotated_path, double open, double close, bool terminating)
+bool None::DoRotate(const char* rotated_path, double open, double close, bool terminating)
 	{
-	if ( ! FinishedRotation(string("/dev/null"), Info().path, open, close, terminating))
+	if ( ! FinishedRotation("/dev/null", Info().path, open, close, terminating))
 		{
-		Error(Fmt("error rotating %s", Info().path.c_str()));
+		Error(Fmt("error rotating %s", Info().path));
 		return false;
 		}
 

src/logging/writers/None.h

@@ -24,10 +24,11 @@ protected:
 	virtual bool DoWrite(int num_fields, const threading::Field* const* fields,
 			     threading::Value** vals)	{ return true; }
 	virtual bool DoSetBuf(bool enabled)	{ return true; }
-	virtual bool DoRotate(string rotated_path, double open,
+	virtual bool DoRotate(const char* rotated_path, double open,
 			      double close, bool terminating);
-	virtual bool DoFlush()	{ return true; }
+	virtual bool DoFlush(double network_time)	{ return true; }
-	virtual bool DoFinish()	{ WriterBackend::DoFinish(); return true; }
+	virtual bool DoFinish(double network_time)	{ return true; }
+	virtual bool DoHeartbeat(double network_time, double current_time)	{ return true; }
 };
 
 }

src/main.cc

@@ -12,6 +12,10 @@
 #include <getopt.h>
 #endif
 
+#ifdef USE_CURL
+#include <curl/curl.h>
+#endif
+
 #ifdef USE_IDMEF
 extern "C" {
 #include <libidmef/idmefxml.h>
@@ -361,12 +365,6 @@ RETSIGTYPE sig_handler(int signo)
 	set_processing_status("TERMINATING", "sig_handler");
 	signal_val = signo;
 
-	if ( thread_mgr->Terminating() && (signal_val == SIGTERM || signal_val == SIGINT) )
-		// If the thread manager is already terminating (i.e.,
-		// waiting for child threads to exit), another term signal
-		// will send the threads a kill.
-		thread_mgr->KillThreads();
-
 	return RETSIGVAL;
 	}
 
@@ -718,6 +716,10 @@ int main(int argc, char** argv)
 	SSL_library_init();
 	SSL_load_error_strings();
 
+#ifdef USE_CURL
+	curl_global_init(CURL_GLOBAL_ALL);
+#endif
+
 	// FIXME: On systems that don't provide /dev/urandom, OpenSSL doesn't
 	// seed the PRNG. We should do this here (but at least Linux, FreeBSD
 	// and Solaris provide /dev/urandom).
@@ -1068,6 +1070,10 @@ int main(int argc, char** argv)
 		done_with_network();
 		net_delete();
 
+#ifdef USE_CURL
+		curl_global_cleanup();
+#endif
+
 		terminate_bro();
 
 		// Close files after net_delete(), because net_delete()

src/ssl-analyzer.pac

@@ -93,6 +93,7 @@ function version_ok(vers : uint16) : bool
 	case SSLv30:
 	case TLSv10:
 	case TLSv11:
+	case TLSv12:
 		return true;
 
 	default:

src/ssl-defs.pac

@@ -22,5 +22,6 @@ enum SSLVersions {
 	SSLv20		= 0x0002,
 	SSLv30		= 0x0300,
 	TLSv10		= 0x0301,
-	TLSv11		= 0x0302
+	TLSv11		= 0x0302,
+	TLSv12		= 0x0303
 };

src/threading/BasicThread.cc

@@ -12,18 +12,23 @@
 
 using namespace threading;
 
+static const int STD_FMT_BUF_LEN = 2048;
+
 uint64_t BasicThread::thread_counter = 0;
 
 BasicThread::BasicThread()
 	{
 	started = false;
 	terminating = false;
+	killed = false;
 	pthread = 0;
 
-	buf_len = 2048;
+	buf_len = STD_FMT_BUF_LEN;
 	buf = (char*) malloc(buf_len);
 
-	name = Fmt("thread-%d", ++thread_counter);
+	strerr_buffer = 0;
+
+	name = copy_string(fmt("thread-%" PRIu64, ++thread_counter));
 
 	thread_mgr->AddThread(this);
 	}
@@ -32,31 +37,42 @@ BasicThread::~BasicThread()
 	{
 	if ( buf )
 		free(buf);
+
+	delete [] name;
+	delete [] strerr_buffer;
 	}
 
-void BasicThread::SetName(const string& arg_name)
+void BasicThread::SetName(const char* arg_name)
 	{
-	// Slight race condition here with reader threads, but shouldn't matter.
+	delete [] name;
-	name = arg_name;
+	name = copy_string(arg_name);
 	}
 
-void BasicThread::SetOSName(const string& name)
+void BasicThread::SetOSName(const char* arg_name)
 	{
+
 #ifdef HAVE_LINUX
-	prctl(PR_SET_NAME, name.c_str(), 0, 0, 0);
+	prctl(PR_SET_NAME, arg_name, 0, 0, 0);
 #endif
 
 #ifdef __APPLE__
-	pthread_setname_np(name.c_str());
+	pthread_setname_np(arg_name);
 #endif
 
 #ifdef FREEBSD
-	pthread_set_name_np(pthread_self(), name, name.c_str());
+	pthread_set_name_np(pthread_self(), arg_name, arg_name);
 #endif
 	}
 
 const char* BasicThread::Fmt(const char* format, ...)
 	{
+	if ( buf_len > 10 * STD_FMT_BUF_LEN )
+		{
+		// Shrink back to normal.
+		buf = (char*) safe_realloc(buf, STD_FMT_BUF_LEN);
+		buf_len = STD_FMT_BUF_LEN;
+		}
+
 	va_list al;
 	va_start(al, format);
 	int n = safe_vsnprintf(buf, buf_len, format, al);
@@ -64,46 +80,56 @@ const char* BasicThread::Fmt(const char* format, ...)
 
 	if ( (unsigned int) n >= buf_len )
 		{ // Not enough room, grow the buffer.
-		int tmp_len = n + 32;
+		buf_len = n + 32;
-		char* tmp = (char*) malloc(tmp_len);
+		buf = (char*) safe_realloc(buf, buf_len);
 
 		// Is it portable to restart?
 		va_start(al, format);
-		n = safe_vsnprintf(tmp, tmp_len, format, al);
+		n = safe_vsnprintf(buf, buf_len, format, al);
 		va_end(al);
-
-		free(tmp);
 		}
 
 	return buf;
 	}
 
+const char* BasicThread::Strerror(int err)
+	{
+	if ( ! strerr_buffer )
+		strerr_buffer = new char[256];
+
+	strerror_r(err, strerr_buffer, 256);
+	return strerr_buffer;
+	}
+
 void BasicThread::Start()
 	{
-
 	if ( started )
 		return;
 
-	int err = pthread_mutex_init(&terminate, 0);
-	if ( err != 0  )
-		reporter->FatalError("Cannot create terminate mutex for thread %s: %s", name.c_str(), strerror(err));
-
-	// We use this like a binary semaphore and acquire it immediately.
-	err = pthread_mutex_lock(&terminate);
-	if ( err != 0 )
-		reporter->FatalError("Cannot aquire terminate mutex for thread %s: %s", name.c_str(), strerror(err));
-
-	err = pthread_create(&pthread, 0, BasicThread::launcher, this);
-	if ( err != 0 )
-		reporter->FatalError("Cannot create thread %s:%s", name.c_str(), strerror(err));
-
-	DBG_LOG(DBG_THREADING, "Started thread %s", name.c_str());
-
 	started = true;
 
+	int err = pthread_create(&pthread, 0, BasicThread::launcher, this);
+	if ( err != 0 )
+		reporter->FatalError("Cannot create thread %s: %s", name, Strerror(err));
+
+	DBG_LOG(DBG_THREADING, "Started thread %s", name);
+
 	OnStart();
 	}
 
+void BasicThread::PrepareStop()
+	{
+	if ( ! started )
+		return;
+
+	if ( terminating )
+		return;
+
+	DBG_LOG(DBG_THREADING, "Preparing thread %s to terminate ...", name);
+
+	OnPrepareStop();
+	}
+
 void BasicThread::Stop()
 	{
 	if ( ! started )
@@ -112,17 +138,11 @@ void BasicThread::Stop()
 	if ( terminating )
 		return;
 
-	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name.c_str());
+	DBG_LOG(DBG_THREADING, "Signaling thread %s to terminate ...", name);
-
-	// Signal that it's ok for the thread to exit now by unlocking the
-	// mutex.
-	int err = pthread_mutex_unlock(&terminate);
-	if ( err != 0 )
-		reporter->FatalError("Failure flagging terminate condition for thread %s: %s", name.c_str(), strerror(err));
-
-	terminating = true;
 
 	OnStop();
+
+	terminating = true;
 	}
 
 void BasicThread::Join()
@@ -130,30 +150,34 @@ void BasicThread::Join()
 	if ( ! started )
 		return;
 
-	if ( ! terminating )
+	assert(terminating);
-		Stop();
 
-	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name.c_str());
+	DBG_LOG(DBG_THREADING, "Joining thread %s ...", name);
 
-	if ( pthread_join(pthread, 0) != 0  )
+	if ( pthread && pthread_join(pthread, 0) != 0  )
-		reporter->FatalError("Failure joining thread %s", name.c_str());
+		reporter->FatalError("Failure joining thread %s", name);
 
-	pthread_mutex_destroy(&terminate);
+	DBG_LOG(DBG_THREADING, "Joined with thread %s", name);
-
-	DBG_LOG(DBG_THREADING, "Done with thread %s", name.c_str());
 
 	pthread = 0;
 	}
 
 void BasicThread::Kill()
 	{
-	if ( ! (started && pthread) )
+	// We don't *really* kill the thread here because that leads to race
-		return;
+	// conditions. Instead we set a flag that parts of the the code need
+	// to check and get out of any loops they might be in.
+	terminating = true;
+	killed = true;
+	OnKill();
+	}
 
-	// I believe this is safe to call from a signal handler ... Not error
+void BasicThread::Done()
-	// checking so that killing doesn't bail out if we have already
+	{
-	// terminated.
+	DBG_LOG(DBG_THREADING, "Thread %s has finished", name);
-	pthread_kill(pthread, SIGKILL);
+
+	terminating = true;
+	killed = true;
 	}
 
 void* BasicThread::launcher(void *arg)
@@ -173,15 +197,12 @@ void* BasicThread::launcher(void *arg)
 	sigdelset(&mask_set, SIGSEGV);
 	sigdelset(&mask_set, SIGBUS);
 	int res = pthread_sigmask(SIG_BLOCK, &mask_set, 0);
-	assert(res == 0);  //
+	assert(res == 0);
 
 	// Run thread's main function.
 	thread->Run();
 
-	// Wait until somebody actually wants us to terminate.
+	thread->Done();
-	if ( pthread_mutex_lock(&thread->terminate) != 0 )
-		reporter->FatalError("Failure acquiring terminate mutex at end of thread %s", thread->Name().c_str());
 
 	return 0;
 	}
-

src/threading/BasicThread.h

@@ -5,7 +5,6 @@
 #include <pthread.h>
 #include <semaphore.h>
 
-#include "Queue.h"
 #include "util.h"
 
 using namespace std;
@@ -42,22 +41,25 @@ public:
 	 *
 	 * This method is safe to call from any thread.
 	 */
-	const string& Name() const { return name; }
+	const char* Name() const { return name; }
 
 	/**
 	* Sets a descriptive name for the thread. This should be a string
 	* that's useful in output presented to the user and uniquely
 	* identifies the thread.
 	*
-	* This method must be called only from the thread itself.
+	* This method must be called only from main thread at initialization
+	* time.
 	*/
-	void SetName(const string& name);
+	void SetName(const char* name);
 
 	/**
 	 * Set the name shown by the OS as the thread's description. Not
 	 * supported on all OSs.
+	 *
+	 * Must be called only from the child thread.
 	 */
-	void SetOSName(const string& name);
+	void SetOSName(const char* name);
 
 	/**
 	 * Starts the thread. Calling this methods will spawn a new OS thread
@@ -68,6 +70,18 @@ public:
 	 */
 	void Start();
 
+	/**
+	 * Signals the thread to prepare for stopping. This must be called
+	 * before Stop() and allows the thread to trigger shutting down
+	 * without yet blocking for doing so.
+	 *
+	 * Calling this method has no effect if Start() hasn't been executed
+	 * yet.
+	 *
+	 * Only Bro's main thread must call this method.
+	 */
+	void PrepareStop();
+
 	/**
 	 * Signals the thread to stop. The method lets Terminating() now
 	 * return true. It does however not force the thread to terminate.
@@ -88,6 +102,13 @@ public:
 	 */
 	bool Terminating()  const { return terminating; }
 
+	/**
+	 * Returns true if Kill() has been called.
+	 *
+	 * This method is safe to call from any thread.
+	 */
+	bool Killed()  const { return killed; }
+
 	/**
 	 * A version of fmt() that the thread can safely use.
 	 *
@@ -96,6 +117,14 @@ public:
 	 */
 	const char* Fmt(const char* format, ...);
 
+	/**
+	 * A version of strerror() that the thread can safely use. This is
+	 * essentially a wrapper around strerror_r(). Note that it keeps a
+	 * single buffer per thread internally so the result remains valid
+	 * only until the next call.
+	 */
+	const char* Strerror(int err);
+
 protected:
 	friend class Manager;
 
@@ -116,12 +145,24 @@ protected:
 	virtual void OnStart()	{}
 
 	/**
-	 * Executed with Stop(). This is a hook into stopping the thread. It
+	 * Executed with PrepareStop() (and before OnStop()). This is a hook
-	 * will be called from Bro's main thread after the thread has been
+	 * into preparing the thread for stopping. It will be called from
-	 * signaled to stop.
+	 * Bro's main thread before the thread has been signaled to stop.
+	 */
+	virtual void OnPrepareStop()	{}
+
+	/**
+	 * Executed with Stop() (and after OnPrepareStop()). This is a hook
+	 * into stopping the thread. It will be called from Bro's main thread
+	 * after the thread has been signaled to stop.
 	 */
 	virtual void OnStop()	{}
 
+	/**
+	 * Executed with Kill(). This is a hook into killing the thread.
+	 */
+	virtual void OnKill()	{}
+
 	/**
 	 * Destructor. This will be called by the manager.
 	 *
@@ -145,14 +186,18 @@ protected:
 	 */
 	void Kill();
 
+	/** Called by child thread's launcher when it's done processing. */
+	void Done();
+
 private:
 	// pthread entry function.
 	static void* launcher(void *arg);
 
-	string name;
+	const char* name;
 	pthread_t pthread;
 	bool started; 		// Set to to true once running.
 	bool terminating;	// Set to to true to signal termination.
+	bool killed;	// Set to true once forcefully killed.
 
 	// Used as a semaphore to tell the pthread thread when it may
 	// terminate.
@@ -162,6 +207,9 @@ private:
 	char* buf;
 	unsigned int buf_len;
 
+	// For implementating Strerror().
+	char* strerr_buffer;
+
 	static uint64_t thread_counter;
 };
 

src/threading/Manager.cc

@@ -30,6 +30,10 @@ void Manager::Terminate()
 	do Process(); while ( did_process );
 
 	// Signal all to stop.
+
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		(*i)->PrepareStop();
+
 	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
 		(*i)->Stop();
 
@@ -48,24 +52,16 @@ void Manager::Terminate()
 	terminating = false;
 	}
 
-void Manager::KillThreads()
-	{
-	DBG_LOG(DBG_THREADING, "Killing threads ...");
-
-	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
-		(*i)->Kill();
-	}
-
 void Manager::AddThread(BasicThread* thread)
 	{
-	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str());
+	DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name());
 	all_threads.push_back(thread);
 	idle = false;
 	}
 
 void Manager::AddMsgThread(MsgThread* thread)
 	{
-	DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name().c_str());
+	DBG_LOG(DBG_THREADING, "%s is a MsgThread ...", thread->Name());
 	msg_threads.push_back(thread);
 	}
 
@@ -91,6 +87,14 @@ double Manager::NextTimestamp(double* network_time)
 	return -1.0;
 	}
 
+void Manager::KillThreads()
+	{
+	DBG_LOG(DBG_THREADING, "Killing threads ...");
+
+	for ( all_thread_list::iterator i = all_threads.begin(); i != all_threads.end(); i++ )
+		(*i)->Kill();
+        }
+
 void Manager::Process()
 	{
 	bool do_beat = false;
@@ -114,6 +118,12 @@ void Manager::Process()
 			{
 			Message* msg = t->RetrieveOut();
 
+			if ( ! msg )
+				{
+				assert(t->Killed());
+				break;
+				}
+
 			if ( msg->Process() )
 				{
 				if ( network_time )
@@ -122,8 +132,7 @@ void Manager::Process()
 
 			else
 				{
-				string s = msg->Name() + " failed, terminating thread";
+				reporter->Error("%s failed, terminating thread", msg->Name());
-				reporter->Error("%s", s.c_str());
 				t->Stop();
 				}
 

src/threading/Manager.h

@@ -49,15 +49,6 @@ public:
 	 */
 	bool Terminating() const	{ return terminating; }
 
-	/**
-	 * Immediately kills all child threads. It does however not yet join
-	 * them, one still needs to call Terminate() for that.
-	 *
-	 * This method is safe to call from a signal handler, and can in fact
-	 * be called while Terminate() is already in progress.
-	 */
-	void KillThreads();
-
 	typedef std::list<std::pair<string, MsgThread::Stats> > msg_stats_list;
 
 	/**
@@ -115,6 +106,13 @@ protected:
 	 */
 	virtual double NextTimestamp(double* network_time);
 
+	/**
+	 * Kills all thread immediately. Note that this may cause race conditions
+	 * if a child thread currently holds a lock that might block somebody
+	 * else.
+	 */
+	virtual void KillThreads();
+
 	/**
 	 * Part of the IOSource interface.
 	 */

src/threading/MsgThread.cc

@@ -5,6 +5,7 @@
 #include "Manager.h"
 
 #include <unistd.h>
+#include <signal.h>
 
 using namespace threading;
 
@@ -16,19 +17,17 @@ namespace threading  {
 class FinishMessage : public InputMessage<MsgThread>
 {
 public:
-	FinishMessage(MsgThread* thread) : InputMessage<MsgThread>("Finish", thread)	{ }
+	FinishMessage(MsgThread* thread, double network_time) : InputMessage<MsgThread>("Finish", thread),
+		network_time(network_time) { }
 
-	virtual bool Process()	{ return Object()->DoFinish(); }
+	virtual bool Process()	{
-};
+		bool result = Object()->OnFinish(network_time);
+		Object()->Finished();
+		return result;
+	}
 
-// A dummy message that's only purpose is unblock the current read operation
+private:
-// so that the child's Run() methods can check the termination status.
+	double network_time;
-class UnblockMessage : public InputMessage<MsgThread>
-{
-public:
-	UnblockMessage(MsgThread* thread) : InputMessage<MsgThread>("Unblock", thread)	{ }
-
-	virtual bool Process()	{ return true; }
 };
 
 /// Sends a heartbeat to the child thread.
@@ -39,7 +38,10 @@ public:
 		: InputMessage<MsgThread>("Heartbeat", thread)
 		{ network_time = arg_network_time; current_time = arg_current_time; }
 
-	virtual bool Process()	{ return Object()->DoHeartbeat(network_time, current_time); }
+	virtual bool Process()	{
+		Object()->HeartbeatInChild();
+		return Object()->OnHeartbeat(network_time, current_time);
+	}
 
 private:
 	double network_time;
@@ -55,14 +57,16 @@ public:
 		INTERNAL_WARNING, INTERNAL_ERROR
 	};
 
-	ReporterMessage(Type arg_type, MsgThread* thread, const string& arg_msg)
+	ReporterMessage(Type arg_type, MsgThread* thread, const char* arg_msg)
 		: OutputMessage<MsgThread>("ReporterMessage", thread)
-		{ type = arg_type; msg = arg_msg; }
+		{ type = arg_type; msg = copy_string(arg_msg); }
+
+	~ReporterMessage() 	 { delete [] msg; }
 
 	virtual bool Process();
 
 private:
-	string msg;
+	const char* msg;
 	Type type;
 };
 
@@ -71,18 +75,19 @@ private:
 class DebugMessage : public OutputMessage<MsgThread>
 {
 public:
-	DebugMessage(DebugStream arg_stream, MsgThread* thread, const string& arg_msg)
+	DebugMessage(DebugStream arg_stream, MsgThread* thread, const char* arg_msg)
 		: OutputMessage<MsgThread>("DebugMessage", thread)
-		{ stream = arg_stream; msg = arg_msg; }
+		{ stream = arg_stream; msg = copy_string(arg_msg); }
+
+	virtual ~DebugMessage()	{ delete [] msg; }
 
 	virtual bool Process()
 		{
-		string s = Object()->Name() + ": " + msg;
+		debug_logger.Log(stream, "%s: %s", Object()->Name(), msg);
-		debug_logger.Log(stream, "%s", s.c_str());
 		return true;
 		}
 private:
-	string msg;
+	const char* msg;
 	DebugStream stream;
 };
 #endif
@@ -93,41 +98,39 @@ private:
 
 Message::~Message()
 	{
+	delete [] name;
 	}
 
 bool ReporterMessage::Process()
 	{
-	string s = Object()->Name() + ": " + msg;
-	const char* cmsg = s.c_str();
-
 	switch ( type ) {
 
 	case INFO:
-		reporter->Info("%s", cmsg);
+		reporter->Info("%s: %s", Object()->Name(), msg);
 		break;
 
 	case WARNING:
-		reporter->Warning("%s", cmsg);
+		reporter->Warning("%s: %s", Object()->Name(), msg);
 		break;
 
 	case ERROR:
-		reporter->Error("%s", cmsg);
+		reporter->Error("%s: %s", Object()->Name(), msg);
 		break;
 
 	case FATAL_ERROR:
-		reporter->FatalError("%s", cmsg);
+		reporter->FatalError("%s: %s", Object()->Name(), msg);
 		break;
 
 	case FATAL_ERROR_WITH_CORE:
-		reporter->FatalErrorWithCore("%s", cmsg);
+		reporter->FatalErrorWithCore("%s: %s", Object()->Name(), msg);
 		break;
 
 	case INTERNAL_WARNING:
-		reporter->InternalWarning("%s", cmsg);
+		reporter->InternalWarning("%s: %s", Object()->Name(), msg);
 		break;
 
 	case INTERNAL_ERROR :
-		reporter->InternalError("%s", cmsg);
+		reporter->InternalError("%s: %s", Object()->Name(), msg);
 		break;
 
 	default:
@@ -137,32 +140,74 @@ bool ReporterMessage::Process()
 	return true;
 	}
 
-MsgThread::MsgThread() : BasicThread()
+MsgThread::MsgThread() : BasicThread(), queue_in(this, 0), queue_out(0, this)
 	{
 	cnt_sent_in = cnt_sent_out = 0;
 	finished = false;
 	thread_mgr->AddMsgThread(this);
 	}
 
+// Set by Bro's main signal handler.
+extern int signal_val;
+
+void MsgThread::OnPrepareStop()
+	{
+	if ( finished || Killed() )
+		return;
+
+	// Signal thread to terminate and wait until it has acknowledged.
+	SendIn(new FinishMessage(this, network_time), true);
+	}
+
 void MsgThread::OnStop()
 	{
-	// Signal thread to terminate and wait until it has acknowledged.
+	int signal_count = 0;
-	SendIn(new FinishMessage(this), true);
+	int old_signal_val = signal_val;
+	signal_val = 0;
 
 	int cnt = 0;
-	while ( ! finished )
+	uint64_t last_size = 0;
+	uint64_t cur_size = 0;
+
+	while ( ! (finished || Killed() ) )
 		{
-		if ( ++cnt > 1000 ) // Insurance against broken threads ...
+		// Terminate if we get another kill signal.
+		if ( signal_val == SIGTERM || signal_val == SIGINT )
 			{
-			reporter->Warning("thread %s didn't finish in time", Name().c_str());
+			++signal_count;
-			break;
+
+			if ( signal_count == 1 )
+				{
+				// Abort all threads here so that we won't hang next
+				// on another one.
+				fprintf(stderr, "received signal while waiting for thread %s, aborting all ...\n", Name());
+				thread_mgr->KillThreads();
 				}
+			else
+				{
+				// More than one signal. Abort processing
+				// right away. on another one.
+				fprintf(stderr, "received another signal while waiting for thread %s, aborting processing\n", Name());
+				exit(1);
+				}
+
+			signal_val = 0;
+			}
+
+		queue_in.WakeUp();
 
 		usleep(1000);
 		}
 
-	// One more message to make sure the current queue read operation unblocks.
+	signal_val = old_signal_val;
-	SendIn(new UnblockMessage(this), true);
+	}
+
+void MsgThread::OnKill()
+	{
+	// Send a message to unblock the reader if its currently waiting for
+	// input. This is just an optimization to make it terminate more
+	// quickly, even without the message it will eventually time out.
+	queue_in.WakeUp();
 	}
 
 void MsgThread::Heartbeat()
@@ -170,25 +215,20 @@ void MsgThread::Heartbeat()
 	SendIn(new HeartbeatMessage(this, network_time, current_time()));
 	}
 
-bool MsgThread::DoHeartbeat(double network_time, double current_time)
+void MsgThread::HeartbeatInChild()
 	{
-	string n = Name();
+	string n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", Name(),
-
-	n = Fmt("bro: %s (%" PRIu64 "/%" PRIu64 ")", n.c_str(),
 		cnt_sent_in - queue_in.Size(),
 		cnt_sent_out - queue_out.Size());
 
 	SetOSName(n.c_str());
-
-	return true;
 	}
 
-bool MsgThread::DoFinish()
+void MsgThread::Finished()
 	{
 	// This is thread-safe "enough", we're the only one ever writing
 	// there.
 	finished = true;
-	return true;
 	}
 
 void MsgThread::Info(const char* msg)
@@ -245,7 +285,7 @@ void MsgThread::SendIn(BasicInputMessage* msg, bool force)
 		return;
 		}
 
-	DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name().c_str(), Name().c_str());
+	DBG_LOG(DBG_THREADING, "Sending '%s' to %s ...", msg->Name(), Name());
 
 	queue_in.Put(msg);
 	++cnt_sent_in;
@@ -268,9 +308,10 @@ void MsgThread::SendOut(BasicOutputMessage* msg, bool force)
 BasicOutputMessage* MsgThread::RetrieveOut()
 	{
 	BasicOutputMessage* msg = queue_out.Get();
-	assert(msg);
+	if ( ! msg )
+		return 0;
 
-	DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s",  msg->Name().c_str(), Name().c_str());
+	DBG_LOG(DBG_THREADING, "Retrieved '%s' from %s",  msg->Name(), Name());
 
 	return msg;
 	}
@@ -278,10 +319,12 @@ BasicOutputMessage* MsgThread::RetrieveOut()
 BasicInputMessage* MsgThread::RetrieveIn()
 	{
 	BasicInputMessage* msg = queue_in.Get();
-	assert(msg);
+
+	if ( ! msg )
+		return 0;
 
 #ifdef DEBUG
-	string s = Fmt("Retrieved '%s' in %s",  msg->Name().c_str(), Name().c_str());
+	string s = Fmt("Retrieved '%s' in %s",  msg->Name(), Name());
 	Debug(DBG_THREADING, s.c_str());
 #endif
 
@@ -290,27 +333,33 @@ BasicInputMessage* MsgThread::RetrieveIn()
 
 void MsgThread::Run()
 	{
-	while ( true )
+	while ( ! (finished || Killed() ) )
 		{
-		// When requested to terminate, we only do so when
-		// all input has been processed.
-		if ( Terminating() && ! queue_in.Ready() )
-			break;
-
 		BasicInputMessage* msg = RetrieveIn();
 
+		if ( ! msg )
+			continue;
+
 		bool result = msg->Process();
 
 		if ( ! result )
 			{
-			string s = msg->Name() + " failed, terminating thread (MsgThread)";
+			string s = Fmt("%s failed, terminating thread (MsgThread)", Name());
 			Error(s.c_str());
-			Stop();
 			break;
 			}
 
 		delete msg;
 		}
+
+	// In case we haven't send the finish method yet, do it now. Reading
+	// global network_time here should be fine, it isn't changing
+	// anymore.
+	if ( ! finished )
+		{
+		OnFinish(network_time);
+		Finished();
+		}
 	}
 
 void MsgThread::GetStats(Stats* stats)

src/threading/MsgThread.h

@@ -189,39 +189,42 @@ protected:
 	 *
 	 * This is method is called regularly by the threading::Manager.
 	 *
-	 * Can be overriden in derived classed to hook into the heart beat,
+	 * Can be overriden in derived classed to hook into the heart beat
-	 * but must call the parent implementation. Note that this method is
+	 * sending, but must call the parent implementation. Note that this
-	 * always called by the main thread and must not access data of the
+	 * method is always called by the main thread and must not access
-	 * child thread directly. See DoHeartbeat() if you want to do
+	 * data of the child thread directly. Implement OnHeartbeat() if you
-	 * something on the child-side.
+	 * want to do something on the child-side.
 	 */
 	virtual void Heartbeat();
 
-	/**
+	/** Internal heartbeat processing. Called from child.
-	 * Overriden from BasicThread.
-	 *
 	 */
-	virtual void Run();
+	void HeartbeatInChild();
-	virtual void OnStop();
 
 	/**
 	 * Regulatly triggered for execution in the child thread.
 	 *
-	 * When overriding, one must call the parent class' implementation.
-	 *
 	 * network_time: The network_time when the heartbeat was trigger by
 	 * the main thread.
 	 *
 	 * current_time: Wall clock when the heartbeat was trigger by the
 	 * main thread.
 	 */
-	virtual bool DoHeartbeat(double network_time, double current_time);
+	virtual bool OnHeartbeat(double network_time, double current_time) = 0;
 
 	/** Triggered for execution in the child thread just before shutting threads down.
-	 *  The child thread should finish its operations and then *must*
+	 *  The child thread should finish its operations.
-	 *  call this class' implementation.
 	 */
-	virtual bool DoFinish();
+	virtual bool OnFinish(double network_time) = 0;
+
+	/**
+	 * Overriden from BasicThread.
+	 *
+	 */
+	virtual void Run();
+	virtual void OnStop();
+	virtual void OnPrepareStop();
+	virtual void OnKill();
 
 private:
 	/**
@@ -280,6 +283,10 @@ private:
 	 */
 	bool MightHaveOut() { return queue_out.MaybeReady(); }
 
+	/** Flags that the child process has finished processing. Called from child.
+	 */
+	void Finished();
+
 	Queue<BasicInputMessage *> queue_in;
 	Queue<BasicOutputMessage *> queue_out;
 
@@ -305,7 +312,7 @@ public:
 	 * what's passed into the constructor and used mainly for debugging
 	 * purposes.
 	 */
-	const string& Name() const { return name; }
+	const char* Name() const { return name; }
 
 	/**
 	 * Callback that must be overriden for processing a message.
@@ -319,10 +326,11 @@ protected:
 	 * @param arg_name A descriptive name for the type of message. Used
 	 * mainly for debugging purposes.
 	 */
-	Message(const string& arg_name)	{ name = arg_name; }
+	Message(const char* arg_name)
+		{ name = copy_string(arg_name); }
 
 private:
-	string name;
+	const char* name;
 };
 
 /**
@@ -337,7 +345,7 @@ protected:
 	 * @param name A descriptive name for the type of message. Used
 	 * mainly for debugging purposes.
 	 */
-	BasicInputMessage(const string& name) : Message(name)	{}
+	BasicInputMessage(const char* name) : Message(name)	{}
 };
 
 /**
@@ -352,7 +360,7 @@ protected:
 	 * @param name A descriptive name for the type of message. Used
 	 * mainly for debugging purposes.
 	 */
-	BasicOutputMessage(const string& name) : Message(name)	{}
+	BasicOutputMessage(const char* name) : Message(name)	{}
 };
 
 /**
@@ -377,7 +385,7 @@ protected:
 	 *
 	 * @param arg_object: An object to store with the message. 
 	 */
-	InputMessage(const string& name, O* arg_object) : BasicInputMessage(name)
+	InputMessage(const char* name, O* arg_object) : BasicInputMessage(name)
 		{ object = arg_object; }
 
 private:
@@ -406,7 +414,7 @@ protected:
 	 *
 	 * @param arg_object An object to store with the message. 
 	 */
-	OutputMessage(const string& name, O* arg_object) : BasicOutputMessage(name)
+	OutputMessage(const char* name, O* arg_object) : BasicOutputMessage(name)
 		{ object = arg_object; }
 
 private:

src/threading/Queue.h

@@ -1,4 +1,3 @@
-
 #ifndef THREADING_QUEUE_H
 #define THREADING_QUEUE_H
 
@@ -6,8 +5,10 @@
 #include <queue>
 #include <deque>
 #include <stdint.h>
+#include <sys/time.h>
 
 #include "Reporter.h"
+#include "BasicThread.h"
 
 #undef Queue // Defined elsewhere unfortunately.
 
@@ -30,8 +31,12 @@ class Queue
 public:
 	/**
 	 * Constructor.
+	 *
+	 * reader, writer: The corresponding threads. This is for checking
+	 * whether they have terminated so that we can abort I/O opeations.
+	 * Can be left null for the main thread.
 	 */
-	Queue();
+	Queue(BasicThread* arg_reader, BasicThread* arg_writer);
 
 	/**
 	 * Destructor.
@@ -39,7 +44,9 @@ public:
 	~Queue();
 
 	/**
-	 * Retrieves one elment.
+	 * Retrieves one element. This may block for a little while of no
+	 * input is available and eventually return with a null element if
+	 * nothing shows up.
 	 */
 	T Get();
 
@@ -60,6 +67,11 @@ public:
 	 */
 	bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); }
 
+	/** Wake up the reader if it's currently blocked for input. This is
+	 primarily to give it a chance to check termination quickly.
+	**/
+	void WakeUp();
+
 	/**
 	 * Returns the number of queued items not yet retrieved.
 	 */
@@ -91,6 +103,9 @@ private:
 	int read_ptr;	// Where the next operation will read from
 	int write_ptr;	// Where the next operation will write to
 
+	BasicThread* reader;
+	BasicThread* writer;
+
 	// Statistics.
 	uint64_t num_reads;
 	uint64_t num_writes;
@@ -109,18 +124,20 @@ inline static void safe_unlock(pthread_mutex_t* mutex)
 	}
 
 template<typename T>
-inline Queue<T>::Queue()
+inline Queue<T>::Queue(BasicThread* arg_reader, BasicThread* arg_writer)
 	{
 	read_ptr = 0;
 	write_ptr = 0;
 	num_reads = num_writes = 0;
+	reader = arg_reader;
+	writer = arg_writer;
 
 	for( int i = 0; i < NUM_QUEUES; ++i )
 		{
-		if ( pthread_cond_init(&has_data[i], NULL) != 0 )
+		if ( pthread_cond_init(&has_data[i], 0) != 0 )
 			reporter->FatalError("cannot init queue condition variable");
 
-		if ( pthread_mutex_init(&mutex[i], NULL) != 0 )
+		if ( pthread_mutex_init(&mutex[i], 0) != 0 )
 			reporter->FatalError("cannot init queue mutex");
 		}
 	}
@@ -138,12 +155,23 @@ inline Queue<T>::~Queue()
 template<typename T>
 inline T Queue<T>::Get()
 	{
+	if ( (reader && reader->Killed()) || (writer && writer->Killed()) )
+		return 0;
+
 	safe_lock(&mutex[read_ptr]);
 
 	int old_read_ptr = read_ptr;
 
 	if ( messages[read_ptr].empty() )
-		pthread_cond_wait(&has_data[read_ptr], &mutex[read_ptr]);
+		{
+		struct timespec ts;
+		ts.tv_sec = time(0) + 5;
+		ts.tv_nsec = 0;
+
+		pthread_cond_timedwait(&has_data[read_ptr], &mutex[read_ptr], &ts);
+		safe_unlock(&mutex[read_ptr]);
+		return 0;
+		}
 
 	T data = messages[read_ptr].front();
 	messages[read_ptr].pop();
@@ -222,6 +250,17 @@ inline void Queue<T>::GetStats(Stats* stats)
 		safe_unlock(&mutex[i]);
 	}
 
+template<typename T>
+inline void Queue<T>::WakeUp()
+	{
+	for ( int i = 0; i < NUM_QUEUES; i++ )
+		{
+		safe_lock(&mutex[i]);
+		pthread_cond_signal(&has_data[i]);
+		safe_unlock(&mutex[i]);
+		}
+	}
+
 }
 
 

src/threading/SerialTypes.cc

@@ -11,23 +11,54 @@ bool Field::Read(SerializationFormat* fmt)
 	{
 	int t;
 	int st;
+	string tmp_name;
+	bool have_2nd;
 
-	bool success = (fmt->Read(&name, "name")
+	if ( ! fmt->Read(&have_2nd, "have_2nd") )
-			&& fmt->Read(&secondary_name, "secondary_name")
+		return false;
+
+	if ( have_2nd )
+		{
+		string tmp_secondary_name;
+		if ( ! fmt->Read(&tmp_secondary_name, "secondary_name") )
+			return false;
+
+		secondary_name = copy_string(tmp_secondary_name.c_str());
+		}
+	else
+		secondary_name = 0;
+
+	bool success = (fmt->Read(&tmp_name, "name")
 			&& fmt->Read(&t, "type")
 			&& fmt->Read(&st, "subtype")
 			&& fmt->Read(&optional, "optional"));
 
+	if ( ! success )
+		return false;
+
+	name = copy_string(tmp_name.c_str());
+
 	type = (TypeTag) t;
 	subtype = (TypeTag) st;
 
-	return success;
+	return true;
 	}
 
 bool Field::Write(SerializationFormat* fmt) const
 	{
+	assert(name);
+
+	if ( secondary_name )
+		{
+		if ( ! (fmt->Write(true, "have_2nd")
+			&& fmt->Write(secondary_name, "secondary_name")) )
+			return false;
+		}
+	else
+		if ( ! fmt->Write(false, "have_2nd") )
+			return false;
+
 	return (fmt->Write(name, "name")
-		&& fmt->Write(secondary_name, "secondary_name")
 		&& fmt->Write((int)type, "type")
 		&& fmt->Write((int)subtype, "subtype"),
 		fmt->Write(optional, "optional"));
@@ -51,7 +82,7 @@ Value::~Value()
 	{
 	if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC)
 	     && present )
-		delete val.string_val;
+		delete [] val.string_val.data;
 
 	if ( type == TYPE_TABLE && present )
 		{
@@ -224,10 +255,7 @@ bool Value::Read(SerializationFormat* fmt)
 	case TYPE_STRING:
 	case TYPE_FILE:
 	case TYPE_FUNC:
-		{
+		return fmt->Read(&val.string_val.data, &val.string_val.length, "string");
-		val.string_val = new string;
-		return fmt->Read(val.string_val, "string");
-		}
 
 	case TYPE_TABLE:
 		{
@@ -339,7 +367,7 @@ bool Value::Write(SerializationFormat* fmt) const
 	case TYPE_STRING:
 	case TYPE_FILE:
 	case TYPE_FUNC:
-		return fmt->Write(*val.string_val, "string");
+		return fmt->Write(val.string_val.data, val.string_val.length, "string");
 
 	case TYPE_TABLE:
 		{

src/threading/SerialTypes.h

@@ -12,6 +12,7 @@
 using namespace std;
 
 class SerializationFormat;
+class RemoteSerializer;
 
 namespace threading {
 
@@ -19,10 +20,10 @@ namespace threading {
  * Definition of a log file, i.e., one column of a log stream.
  */
 struct Field {
-	string name;	//! Name of the field.
+	const char* name;	//! Name of the field.
 	//! Needed by input framework. Port fields have two names (one for the
 	//! port, one for the type), and this specifies the secondary name.
-	string secondary_name;
+	const char* secondary_name;
 	TypeTag type;	//! Type of the field.
 	TypeTag subtype;	//! Inner type for sets.
 	bool optional;	//! True if field is optional.
@@ -30,13 +31,24 @@ struct Field {
 	/**
 	 * Constructor.
 	 */
-	Field() 	{ subtype = TYPE_VOID; optional = false; }
+	Field(const char* name, const char* secondary_name, TypeTag type, TypeTag subtype, bool optional)
+		: name(name ? copy_string(name) : 0),
+		  secondary_name(secondary_name ? copy_string(secondary_name) : 0),
+		  type(type), subtype(subtype), optional(optional)	{ }
 
 	/**
 	 * Copy constructor.
 	 */
 	Field(const Field& other)
-		: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) {  }
+		: name(other.name ? copy_string(other.name) : 0),
+		  secondary_name(other.secondary_name ? copy_string(other.secondary_name) : 0),
+		  type(other.type), subtype(other.subtype), optional(other.optional)	{ }
+
+	~Field()
+		{
+		delete [] name;
+		delete [] secondary_name;
+		}
 
 	/**
 	 * Unserializes a field.
@@ -63,6 +75,12 @@ struct Field {
 	 * thread-safe.
 	 */
 	string TypeName() const;
+
+private:
+	friend class ::RemoteSerializer;
+
+	// Force usage of constructor above.
+	Field()	{};
 };
 
 /**
@@ -102,7 +120,11 @@ struct Value {
 		vec_t vector_val;
 		addr_t addr_val;
 		subnet_t subnet_val;
-		string* string_val;
+
+		struct {
+			char* data;
+			int length;
+		} string_val;
 	} val;
 
 	/**

src/types.bif

@@ -163,6 +163,7 @@ enum Writer %{
 	WRITER_NONE,
 	WRITER_ASCII,
 	WRITER_DATASERIES,
+	WRITER_ELASTICSEARCH,
 %}
 
 enum ID %{

src/util.cc

@@ -1290,6 +1290,30 @@ uint64 calculate_unique_id(size_t pool)
 	return HashKey::HashBytes(&(uid_pool[pool].key), sizeof(uid_pool[pool].key));
 	}
 
+bool safe_write(int fd, const char* data, int len)
+	{
+	while ( len > 0 )
+		{
+		int n = write(fd, data, len);
+
+		if ( n < 0 )
+			{
+			if ( errno == EINTR )
+				continue;
+
+			fprintf(stderr, "safe_write error: %d\n", errno);
+			abort();
+
+			return false;
+			}
+
+		data += n;
+		len -= n;
+		}
+
+	return true;
+	}
+
 void out_of_memory(const char* where)
 	{
 	reporter->FatalError("out of memory in %s.\n", where);

src/util.h

@@ -13,6 +13,7 @@
 // Expose C99 functionality from inttypes.h, which would otherwise not be
 // available in C++.
 #define __STDC_FORMAT_MACROS
+#define __STDC_LIMIT_MACROS
 #include <inttypes.h>
 
 #if __STDC__
@@ -289,6 +290,11 @@ inline size_t pad_size(size_t size)
 
 #define padded_sizeof(x) (pad_size(sizeof(x)))
 
+// Like write() but handles interrupted system calls by restarting. Returns
+// true if the write was successful, otherwise sets errno. This function is
+// thread-safe as long as no two threads write to the same descriptor.
+extern bool safe_write(int fd, const char* data, int len);
+
 extern void out_of_memory(const char* where);
 
 inline void* safe_realloc(void* ptr, size_t size)
@@ -338,4 +344,5 @@ inline int safe_vsnprintf(char* str, size_t size, const char* format, va_list al
 // handed out by malloc.
 extern void get_memory_usage(unsigned int* total,
 			     unsigned int* malloced);
+
 #endif

testing/btest/Baseline/bifs.to_double/out

@@ -3,4 +3,4 @@
 -60.0
 3600.0
 86400.0
-1337982322.762159
+1342748947.655087

testing/btest/Baseline/core.checksums/bad.out

@@ -3,81 +3,101 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-03-01
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784981.078396	-	-	-	-	-	bad_IP_checksum	-	F	bro
+#end	2012-03-26-18-03-01
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-01-25
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784885.686428	UWkUyAuUGXf	127.0.0.1	30000	127.0.0.1	80	bad_TCP_checksum	-	F	bro
+#end	2012-03-26-18-01-25
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-02-13
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332784933.501023	UWkUyAuUGXf	127.0.0.1	30000	127.0.0.1	13000	bad_UDP_checksum	-	F	bro
+#end	2012-03-26-18-02-13
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-29-23
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075363.536871	UWkUyAuUGXf	192.168.1.100	8	192.168.1.101	0	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-29-23
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-06-50
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785210.013051	-	-	-	-	-	routing0_hdr	-	F	bro
 1332785210.013051	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	80	bad_TCP_checksum	-	F	bro
+#end	2012-03-26-18-06-50
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-17-23-00
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332782580.798420	-	-	-	-	-	routing0_hdr	-	F	bro
 1332782580.798420	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:78:1:32::2	13000	bad_UDP_checksum	-	F	bro
+#end	2012-03-26-17-23-00
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-25-11
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075111.800086	-	-	-	-	-	routing0_hdr	-	F	bro
 1334075111.800086	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:78:1:32::1	129	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-25-11
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-07-30
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785250.469132	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	80	bad_TCP_checksum	-	F	bro
+#end	2012-03-26-18-07-30
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-17-02-22
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332781342.923813	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	30000	2001:4f8:4:7:2e0:81ff:fe52:9a6b	13000	bad_UDP_checksum	-	F	bro
+#end	2012-03-26-17-02-22
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-22-19
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-22-19

testing/btest/Baseline/core.checksums/good.out

@@ -3,54 +3,68 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-22-19
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334074939.467194	UWkUyAuUGXf	2001:4f8:4:7:2e0:81ff:fe52:ffff	128	2001:4f8:4:7:2e0:81ff:fe52:9a6b	129	bad_ICMP_checksum	-	F	bro
+#end	2012-04-10-16-22-19
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-18-05-25
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332785125.596793	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-03-26-18-05-25
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-03-26-17-21-48
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1332782508.592037	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-03-26-17-21-48
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-16-23-47
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334075027.053380	-	-	-	-	-	routing0_hdr	-	F	bro
+#end	2012-04-10-16-23-47

testing/btest/Baseline/core.disable-mobile-ipv6/weird.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-05-21-56-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1333663011.602839	-	-	-	-	-	unknown_protocol_135	-	F	bro
+#end	2012-04-05-21-56-51

testing/btest/Baseline/core.expr-exception/reporter.log

@@ -3,14 +3,16 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
+#start	2011-03-18-19-06-08
 #fields	ts	level	message	location
 #types	time	enum	string	string
-1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.915940	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.915940	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.916118	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.916118	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.918295	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.918295	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.952193	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.952193	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.952228	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.952228	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
-1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/home/jsiwek/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/da/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 10
+#end	2011-03-18-19-06-13

testing/btest/Baseline/core.ipv6-frag/dns.log

@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dns
+#start	2012-03-07-01-37-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs
 #types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]
 1331084278.438444	UWkUyAuUGXf	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51850	2607:f740:b::f93	53	udp	3903	txtpadding_323.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
 1331084293.592245	arKYeMETxOg	2001:470:1f11:81f:d138:5f55:6d4:1fe2	51851	2607:f740:b::f93	53	udp	40849	txtpadding_3230.n1.netalyzr.icsi.berkeley.edu	1	C_INTERNET	16	TXT	0	NOERROR	T	F	T	F	0	This TXT record should be ignored	1.000000
+#end	2012-03-07-01-38-18

testing/btest/Baseline/core.print-bpf-filters/conn.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2005-10-07-23-23-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
+#end	2005-10-07-23-23-57

testing/btest/Baseline/core.print-bpf-filters/output

@@ -3,30 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229717.179155	-	ip or not ip	T	T
+1342748953.570646	-	ip or not ip	T	T
+#end	<abnormal termination>
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229717.462355	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+1342748953.898675	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (tcp port 1080)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)	T	T
+#end	<abnormal termination>
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229717.733007	-	port 42	T	T
+1342748954.278211	-	port 42	T	T
+#end	<abnormal termination>
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	packet_filter
+#start	1970-01-01-00-00-00
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1340229718.001009	-	port 56730	T	T
+1342748954.883780	-	port 56730	T	T
+#end	2005-10-07-23-23-57

testing/btest/Baseline/core.reporter-error-in-handler/output

@@ -1,2 +1,2 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
 1st error printed on script level

testing/btest/Baseline/core.reporter-fmt-strings/output

@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-fmt-strings/reporter-fmt-strings.bro, line 9: not an event (dont_interpret_this(%s))

testing/btest/Baseline/core.reporter-parse-error/output

@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-parse-error/reporter-parse-error.bro, line 7: unknown identifier TESTFAILURE, at or near "TESTFAILURE"

testing/btest/Baseline/core.reporter-runtime-error/output

@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])

testing/btest/Baseline/core.reporter-type-mismatch/output

@@ -1,3 +1,3 @@
-error in string and /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
+error in string and /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: arithmetic mixed with non-arithmetic (string and 42)
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11 and string: type mismatch (42 and string)
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-type-mismatch/reporter-type-mismatch.bro, line 11: argument type mismatch in event invocation (foo(42))

testing/btest/Baseline/core.reporter/logger-test.log

@@ -1,6 +1,6 @@
-reporter_info|init test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000

testing/btest/Baseline/core.reporter/output

@@ -1,3 +1,3 @@
-/Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /Users/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error

testing/btest/Baseline/core.truncation/output

@@ -3,30 +3,38 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-11-16-01-35
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334160095.895421	-	-	-	-	-	truncated_IP	-	F	bro
+#end	2012-04-11-16-01-35
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-11-14-57-21
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334156241.519125	-	-	-	-	-	truncated_IP	-	F	bro
+#end	2012-04-11-14-57-21
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-04-10-21-50-48
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1334094648.590126	-	-	-	-	-	truncated_IP	-	F	bro
+#end	2012-04-10-21-50-48
 #separator \x09
 #set_separator	,
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-05-29-22-02-34
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1338328954.078361	-	-	-	-	-	internally_truncated_header	-	F	bro
+#end	2012-05-29-22-02-34

testing/btest/Baseline/core.tunnels.ayiya/conn.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2009-11-08-04-41-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1257655301.595604	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	tcp	http	2.101052	2981	4665	S1	-	0	ShADad	10	3605	11	5329	k6kgXLOoSKl
@@ -13,3 +14,4 @@
 1257655296.585188	TEfuqmmG4bh	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff00:2	130	icmp	-	0.919988	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
 1257655296.585151	j4u32Pc5bif	fe80::216:cbff:fe9a:4cb9	131	ff02::2:f901:d225	130	icmp	-	0.719947	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
 1257655296.585034	nQcgTWjvg4c	fe80::216:cbff:fe9a:4cb9	131	ff02::1:ff9a:4cb9	130	icmp	-	4.922880	32	0	OTH	-	0	-	2	144	0	0	k6kgXLOoSKl
+#end	2009-11-08-04-41-57

testing/btest/Baseline/core.tunnels.ayiya/http.log

@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2009-11-08-04-41-41
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1257655301.652206	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	10102	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1257655302.514424	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	2	GET	ipv6.google.com	/csi?v=3&s=webhp&action=&tran=undefined&e=17259,19771,21517,21766,21887,22212&ei=BUz2Su7PMJTglQfz3NzCAw&rt=prt.77,xjs.565,ol.645	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
 1257655303.603569	5OKnoww6xl4	2001:4978:f:4c::2	53382	2001:4860:b002::68	80	3	GET	ipv6.google.com	/gen_204?atyp=i&ct=fade&cad=1254&ei=BUz2Su7PMJTglQfz3NzCAw&zx=1257655303600	http://ipv6.google.com/	Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.5; en; rv:1.9.0.15pre) Gecko/2009091516 Camino/2.0b4 (like Firefox/3.0.15pre)	0	0	204	No Content	-	-	-	(empty)	-	-	-	-	-	-
+#end	2009-11-08-04-41-57

testing/btest/Baseline/core.tunnels.ayiya/tunnel.log

@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2009-11-08-04-41-33
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1257655293.629048	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::DISCOVER
 1257655296.585034	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::DISCOVER
 1257655317.464035	k6kgXLOoSKl	192.168.3.101	53859	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::CLOSE
 1257655317.464035	UWkUyAuUGXf	192.168.3.101	53796	216.14.98.22	5072	Tunnel::AYIYA	Tunnel::CLOSE
+#end	2009-11-08-04-41-57

testing/btest/Baseline/core.tunnels.false-teredo/dpd.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	dpd
+#start	2009-11-18-17-59-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	analyzer	failure_reason
 #types	time	string	addr	port	addr	port	enum	string	string
 1258567191.486869	UWkUyAuUGXf	192.168.1.105	57696	192.168.1.1	53	udp	TEREDO	Teredo payload length [c\x1d\x81\x80\x00\x01\x00\x02\x00\x02\x00\x00\x04amch\x0equestionmarket\x03com\x00\x00\x01\x00...]
@@ -11,3 +12,4 @@
 1258581768.898165	TEfuqmmG4bh	192.168.1.104	50798	192.168.1.1	53	udp	TEREDO	Teredo payload length [o\xe3\x81\x80\x00\x01\x00\x02\x00\x04\x00\x04\x03www\x0fnashuatelegraph\x03com\x00\x00\x01\x00...]
 1258584478.989528	FrJExwHcSal	192.168.1.104	64963	192.168.1.1	53	udp	TEREDO	Teredo payload length [e\xbd\x81\x80\x00\x01\x00\x08\x00\x06\x00\x06\x08wellness\x05blogs\x04time\x03com\x00\x00\x01\x00...]
 1258600683.934672	5OKnoww6xl4	192.168.1.103	59838	192.168.1.1	53	udp	TEREDO	Teredo payload length [h\xf0\x81\x80\x00\x01\x00\x01\x00\x02\x00\x00\x06update\x0csanasecurity\x03com\x00\x00\x01\x00...]
+#end	2009-11-19-03-18-03

testing/btest/Baseline/core.tunnels.false-teredo/weird.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2009-11-18-17-59-51
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1258567191.405770	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
@@ -11,3 +12,4 @@
 1258581768.568451	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
 1258584478.859853	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
 1258600683.934458	-	-	-	-	-	truncated_header_in_tunnel	-	F	bro
+#end	2009-11-19-03-18-03

testing/btest/Baseline/core.tunnels.teredo/conn.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2008-05-16-15-50-57
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1210953047.736921	arKYeMETxOg	192.168.2.16	1576	75.126.130.163	80	tcp	-	0.000357	0	0	SHR	-	0	fA	1	40	1	40	(empty)
@@ -26,3 +27,4 @@
 1210953052.324629	FrJExwHcSal	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	TEfuqmmG4bh
 1210953060.829303	qCaWGmzFtM5	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.463615	4	4	OTH	-	0	-	1	52	1	52	GSxOnSLghOa,nQcgTWjvg4c
 1210953052.202579	j4u32Pc5bif	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	nQcgTWjvg4c
+#end	2008-05-16-15-51-16

testing/btest/Baseline/core.tunnels.teredo/http.log

@@ -3,9 +3,11 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2008-05-16-15-50-58
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1210953057.917183	3PKsZ2Uye21	192.168.2.16	1578	75.126.203.78	80	1	POST	download913.avast.com	/cgi-bin/iavs4stats.cgi	-	Syncer/4.80 (av_pro-1169;f)	589	0	204	<empty>	-	-	-	(empty)	-	-	-	text/plain	-	-
 1210953061.585996	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1210953073.381474	70MGiRM1Qf4	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1210953074.674817	c4Zw9TmAE05	192.168.2.16	1580	67.228.110.120	80	1	GET	www.wireshark.org	/	http://ipv6.google.com/search?hl=en&q=Wireshark+%21&btnG=Google+Search	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	11845	200	OK	-	-	-	(empty)	-	-	-	text/xml	-	-
+#end	2008-05-16-15-51-16

testing/btest/Baseline/core.tunnels.teredo/tunnel.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2008-05-16-15-50-52
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1210953052.202579	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::DISCOVER
@@ -11,3 +12,4 @@
 1210953076.058333	nQcgTWjvg4c	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::CLOSE
 1210953076.058333	GSxOnSLghOa	192.168.2.16	3797	83.170.1.38	32900	Tunnel::TEREDO	Tunnel::CLOSE
 1210953076.058333	TEfuqmmG4bh	192.168.2.16	3797	65.55.158.81	3544	Tunnel::TEREDO	Tunnel::CLOSE
+#end	2008-05-16-15-51-16

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/conn.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 1340127577.354166	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	tcp	http	0.052829	1675	10467	S1	-	0	ShADad	10	2279	12	11191	j4u32Pc5bif
@@ -12,3 +13,4 @@
 1340127577.339015	nQcgTWjvg4c	fe80::8000:f227:bec8:61af	134	fe80::8000:ffff:ffff:fffd	133	icmp	-	-	-	-	OTH	-	0	-	1	88	0	0	k6kgXLOoSKl
 1340127577.343969	TEfuqmmG4bh	2001:0:4137:9e50:8000:f12a:b9c8:2815	128	2001:4860:0:2001::68	129	icmp	-	0.007778	4	4	OTH	-	0	-	1	52	1	52	UWkUyAuUGXf,j4u32Pc5bif
 1340127577.336558	arKYeMETxOg	fe80::8000:ffff:ffff:fffd	133	ff02::2	134	icmp	-	-	-	-	OTH	-	0	-	1	64	0	0	UWkUyAuUGXf
+#end	2012-06-19-17-39-37

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/http.log

@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
 1340127577.361683	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	1	GET	ipv6.google.com	/	-	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	6640	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
 1340127577.379360	FrJExwHcSal	2001:0:4137:9e50:8000:f12a:b9c8:2815	1286	2001:4860:0:2001::68	80	2	GET	ipv6.google.com	/search?hl=en&q=Wireshark+!&btnG=Google+Search	http://ipv6.google.com/	Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9b5) Gecko/2008032620 Firefox/3.0b5	0	25119	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-06-19-17-39-37

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/tunnel.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	tunnel
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
 #types	time	string	addr	port	addr	port	enum	enum
 1340127577.336558	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::DISCOVER
@@ -11,3 +12,4 @@
 1340127577.406995	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Tunnel::TEREDO	Tunnel::CLOSE
 1340127577.406995	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Tunnel::TEREDO	Tunnel::CLOSE
 1340127577.406995	k6kgXLOoSKl	192.168.2.16	3797	65.55.158.81	3544	Tunnel::TEREDO	Tunnel::CLOSE
+#end	2012-06-19-17-39-37

testing/btest/Baseline/core.tunnels.teredo_bubble_with_payload/weird.log

@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	weird
+#start	2012-06-19-17-39-37
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	name	addl	notice	peer
 #types	time	string	addr	port	addr	port	string	string	bool	string
 1340127577.346849	UWkUyAuUGXf	192.168.2.16	3797	65.55.158.80	3544	Teredo_bubble_with_payload	-	F	bro
 1340127577.349292	j4u32Pc5bif	192.168.2.16	3797	83.170.1.38	32900	Teredo_bubble_with_payload	-	F	bro
+#end	2012-06-19-17-39-37

testing/btest/Baseline/core.vlan-mpls/conn.log

@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	conn
+#start	2005-10-07-23-23-55
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
 952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.102560	26	0	SH	-	0	SADF	11	470	0	0	(empty)
 1128727435.450898	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.733303	98	9417	SF	-	0	ShADdFaf	12	730	10	9945	(empty)
 1278600802.069419	k6kgXLOoSKl	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.004152	9	3429	SF	-	0	ShADadfF	7	381	7	3801	(empty)
+#end	2010-07-08-14-53-22

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
+#start	2012-07-20-14-34-11
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -20,6 +21,7 @@ scripts/base/init-bare.bro
       scripts/base/frameworks/logging/./postprocessors/./sftp.bro
     scripts/base/frameworks/logging/./writers/ascii.bro
     scripts/base/frameworks/logging/./writers/dataseries.bro
+    scripts/base/frameworks/logging/./writers/elasticsearch.bro
     scripts/base/frameworks/logging/./writers/none.bro
   scripts/base/frameworks/input/__load__.bro
     scripts/base/frameworks/input/./main.bro
@@ -28,3 +30,4 @@ scripts/base/init-bare.bro
     scripts/base/frameworks/input/./readers/raw.bro
     scripts/base/frameworks/input/./readers/benchmark.bro
 scripts/policy/misc/loaded-scripts.bro
+#end	2012-07-20-14-34-11

testing/btest/Baseline/coverage.coverage-blacklist/output

@@ -1,5 +1,5 @@
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13	print cover me;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 13	print cover me;
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17	print always executed;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 17	print always executed;
-0	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26	print also impossible, but included in code coverage analysis;
+0	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 26	print also impossible, but included in code coverage analysis;
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29	print success;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 29	print success;
-1	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5	print first;
+1	/da/home/robin/bro/master/testing/btest/.tmp/coverage.coverage-blacklist/coverage-blacklist.bro, line 5	print first;

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,6 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
+#start	2012-07-20-14-34-40
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -20,6 +21,7 @@ scripts/base/init-bare.bro
       scripts/base/frameworks/logging/./postprocessors/./sftp.bro
     scripts/base/frameworks/logging/./writers/ascii.bro
     scripts/base/frameworks/logging/./writers/dataseries.bro
+    scripts/base/frameworks/logging/./writers/elasticsearch.bro
     scripts/base/frameworks/logging/./writers/none.bro
   scripts/base/frameworks/input/__load__.bro
     scripts/base/frameworks/input/./main.bro
@@ -108,3 +110,4 @@ scripts/base/init-default.bro
     scripts/base/protocols/syslog/./consts.bro
     scripts/base/protocols/syslog/./main.bro
 scripts/policy/misc/loaded-scripts.bro
+#end	2012-07-20-14-34-40

testing/btest/Baseline/istate.broccoli/bro.log

@@ -1,3 +1,3 @@
-ping received, seq 0, 1324314397.698781 at src, 1324314397.699240 at dest, 
+ping received, seq 0, 1342749173.594568 at src, 1342749173.637317 at dest, 
-ping received, seq 1, 1324314398.698905 at src, 1324314398.699094 at dest, 
+ping received, seq 1, 1342749174.594948 at src, 1342749174.596551 at dest, 
-ping received, seq 2, 1324314399.699012 at src, 1324314399.699231 at dest, 
+ping received, seq 2, 1342749175.595486 at src, 1342749175.596581 at dest, 

testing/btest/Baseline/istate.events-ssl/receiver.http.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336588614.060989	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749182.906082	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-04

testing/btest/Baseline/istate.events-ssl/sender.http.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-03
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336588614.060989	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749182.906082	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-04

testing/btest/Baseline/istate.events/receiver.http.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336587178.164598	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749191.765740	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-13

testing/btest/Baseline/istate.events/sender.http.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	http
+#start	2012-07-20-01-53-12
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	string	string	file
-1336587178.164598	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+1342749191.765740	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	(empty)	-	-	-	text/html	-	-
+#end	2012-07-20-01-53-13

testing/btest/Baseline/istate.pybroccoli/bro..stdout

@@ -1,7 +1,7 @@
 ==== atomic
 -10
 2
-1336411585.166009
+1342749196.619505
 2.0 mins
 F
 1.5

testing/btest/Baseline/istate.pybroccoli/python..stdout.filtered

@@ -1,7 +1,7 @@
 ==== atomic a 1 ====
 -4L -4
 42 42
-1336411585.1711
+1342749196.6624
 60.0
 True True
 3.14
@@ -14,7 +14,7 @@ True True
 ==== atomic a 2 ====
 -10L -10
 2 2
-1336411585.1660
+1342749196.6195
 120.0
 False False
 1.5
@@ -27,7 +27,7 @@ False False
 ==== atomic b 2 ====
 -10L -10
 <broccoli.count instance at > 2
-<broccoli.time instance at > 1336411585.1660
+<broccoli.time instance at > 1342749196.6195
 <broccoli.interval instance at > 120.0
 False False
 1.5

testing/btest/Baseline/language.wrong-delete-field/output

@@ -1 +1 @@
-error in /Users/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)
+error in /da/home/robin/bro/master/testing/btest/.tmp/language.wrong-delete-field/wrong-delete-field.bro, line 10: illegal delete statement (delete x$a)

testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log

@@ -3,20 +3,22 @@
 #empty_field	(empty)
 #unset_field	-
 #path	communication
+#start	2012-07-20-01-49-40
 #fields	ts	peer	src_name	connected_peer_desc	connected_peer_addr	connected_peer_port	level	message
 #types	time	string	string	string	addr	port	string	string
-1340904724.781527	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
+1342748980.737451	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
-1340904724.784954	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
+1342748980.747149	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
-1340904724.786168	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
+1342748980.748489	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
-1340904724.786168	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
+1342748980.748489	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
-1340904724.786168	bro	script	-	-	-	info	connection established
+1342748980.750749	bro	script	-	-	-	info	connection established
-1340904724.786168	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
+1342748980.750749	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
-1340904724.786168	bro	script	-	-	-	info	accepting state
+1342748980.750749	bro	script	-	-	-	info	accepting state
-1340904724.787645	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
+1342748980.752225	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
-1340904724.787645	bro	parent	-	-	-	info	warning: no events to request
+1342748980.752225	bro	parent	-	-	-	info	warning: no events to request
-1340904724.788857	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
+1342748980.753384	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
-1340904724.829480	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer supports keep-in-cache; using that
+1342748980.793108	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer supports keep-in-cache; using that
-1340904724.829480	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: running
+1342748980.793108	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: running
-1340904724.829480	bro	parent	-	-	-	info	terminating...
+1342748980.793108	bro	parent	-	-	-	info	terminating...
-1340904724.832952	bro	child	-	-	-	info	terminating
+1342748980.796454	bro	child	-	-	-	info	terminating
-1340904724.834082	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
+1342748980.797536	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
+#end	2012-07-20-01-49-40

testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log

@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	ssh-new-default
+#start	2012-07-20-01-49-19
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1324314313.140603	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342748959.430282	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1324314313.140603	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342748959.430282	1.2.3.4	1234	2.3.4.5	80	failure	US
+#end	2012-07-20-01-49-19

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log

@@ -3,8 +3,10 @@
 #empty_field|(empty)
 #unset_field|-
 #path|ssh
+#start|2012-07-20-01-49-19
 #fields|data|data2
 #types|string|string
 abc\x0a\xffdef|DATA2
 abc\x7c\xffdef|DATA2
 abc\xff\x7cdef|DATA2
+#end|2012-07-20-01-49-19

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh-filtered.log

@@ -0,0 +1,12 @@
+PREFIX<>separator |
+PREFIX<>set_separator|,
+PREFIX<>empty_field|EMPTY
+PREFIX<>unset_field|NOT-SET
+PREFIX<>path|ssh
+PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
+PREFIX<>types|time|addr|port|addr|port|string|string|bool
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
+1342748959.659721|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log

@@ -1,12 +0,0 @@
-PREFIX<>separator |
-PREFIX<>set_separator|,
-PREFIX<>empty_field|EMPTY
-PREFIX<>unset_field|NOT-SET
-PREFIX<>path|ssh
-PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
-PREFIX<>types|time|addr|port|addr|port|string|string|bool
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
-1324314313.345323|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log

@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#start	2012-07-20-01-49-19
 #fields	x	y	z
 #types	string	string	string
 \x2d	-	(empty)
+#end	2012-07-20-01-49-19