Merge remote branch 'origin/topic/jsiwek/log-escaping'

* origin/topic/jsiwek/log-escaping:
  Add missing ascii writer options to log header.
  Escape the ASCII log's set separator (addresses #712)
  Rewrite ODesc character escaping functionality. (addresses #681)

Closes #712.

src/Desc.cc

@@ -41,8 +41,7 @@ ODesc::ODesc(desc_type t, BroFile* arg_f)
 	do_flush = 1;
 	include_stats = 0;
 	indent_with_spaces = 0;
-	escape = 0;
+	escape = false;
-	escape_len = 0;
 	}
 
 ODesc::~ODesc()
@@ -56,10 +55,9 @@ ODesc::~ODesc()
 		free(base);
 	}
 
-void ODesc::SetEscape(const char* arg_escape, int len)
+void ODesc::EnableEscaping()
 	{
-	escape = arg_escape;
+	escape = true;
-	escape_len = len;
 	}
 
 void ODesc::PushIndent()
@@ -228,6 +226,25 @@ static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned
 	return 0;
 	}
 
+pair<const char*, size_t> ODesc::FirstEscapeLoc(const char* bytes, size_t n)
+	{
+	pair<const char*, size_t> p(find_first_unprintable(this, bytes, n), 1);
+
+	string str(bytes, n);
+	list<string>::const_iterator it;
+	for ( it = escape_sequences.begin(); it != escape_sequences.end(); ++it )
+		{
+		size_t pos = str.find(*it);
+		if ( pos != string::npos && (p.first == 0 || bytes + pos < p.first) )
+			{
+			p.first = bytes + pos;
+			p.second = it->size();
+			}
+		}
+
+	return p;
+	}
+
 void ODesc::AddBytes(const void* bytes, unsigned int n)
 	{
 	if ( ! escape )
@@ -241,45 +258,30 @@ void ODesc::AddBytes(const void* bytes, unsigned int n)
 
 	while ( s < e )
 		{
-		const char* t1 = (const char*) memchr(s, escape[0], e - s);
+		pair<const char*, size_t> p = FirstEscapeLoc(s, e - s);
-
+		if ( p.first )
-		if ( ! t1 )
+			{
-		    t1 = e;
+			AddBytesRaw(s, p.first - s);
-
+			if ( p.second == 1 )
-		const char* t2 = find_first_unprintable(this, s, t1 - s);
-
-		if ( t2 && t2 < t1 )
 				{
-			AddBytesRaw(s, t2 - s);
-
 				char hex[6] = "\\x00";
-			hex[2] = hex_chars[((*t2) & 0xf0) >> 4];
+				hex[2] = hex_chars[((*p.first) & 0xf0) >> 4];
-			hex[3] = hex_chars[(*t2) & 0x0f];
+				hex[3] = hex_chars[(*p.first) & 0x0f];
 				AddBytesRaw(hex, 4);
-
-			s = t2 + 1;
-			continue;
 				}
-
+			else
-		if ( memcmp(t1, escape, escape_len) != 0 )
-		    break;
-
-		AddBytesRaw(s, t1 - s);
-
-		for ( int i = 0; i < escape_len; ++i )
 				{
-			char hex[5] = "\\x00";
+				string esc_str = get_escaped_string(string(p.first, p.second));
-			hex[2] = hex_chars[((*t1) & 0xf0) >> 4];
+				AddBytesRaw(esc_str.c_str(), esc_str.size());
-			hex[3] = hex_chars[(*t1) & 0x0f];
-			AddBytesRaw(hex, 4);
-			++t1;
 				}
-
+			s = p.first + p.second;
-		s = t1;
 			}
-
+		else
-	if ( s < e )
+			{
 			AddBytesRaw(s, e - s);
+			break;
+			}
+		}
 	}
 
 void ODesc::AddBytesRaw(const void* bytes, unsigned int n)

src/Desc.h

@@ -4,6 +4,8 @@
 #define descriptor_h
 
 #include <stdio.h>
+#include <list>
+#include <utility>
 #include "BroString.h"
 
 typedef enum {
@@ -48,8 +50,13 @@ public:
 
 	void SetFlush(int arg_do_flush)	{ do_flush = arg_do_flush; }
 
-	// The string passed in must remain valid as long as this object lives.
+	void EnableEscaping();
-	void SetEscape(const char* escape, int len);
+	void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); }
+	void AddEscapeSequence(const char* s, size_t n)
+	    { escape_sequences.push_back(string(s, n)); }
+	void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); }
+	void RemoveEscapeSequence(const char* s, size_t n)
+	    { escape_sequences.remove(string(s, n)); }
 
 	void PushIndent();
 	void PopIndent();
@@ -133,6 +140,19 @@ protected:
 
 	void OutOfMemory();
 
+	/**
+	 * Returns the location of the first place in the bytes to be hex-escaped.
+	 *
+	 * @param bytes the starting memory address to start searching for
+	 *        escapable character.
+	 * @param n the maximum number of bytes to search.
+	 * @return a pair whose first element represents a starting memory address
+	 *         to be escaped up to the number of characters indicated by the
+	 *         second element.  The first element may be 0 if nothing is
+	 *         to be escaped.
+	 */
+	pair<const char*, size_t> FirstEscapeLoc(const char* bytes, size_t n);
+
 	desc_type type;
 	desc_style style;
 
@@ -140,8 +160,8 @@ protected:
 	unsigned int offset;	// where we are in the buffer
 	unsigned int size;	// size of buffer in bytes
 
-	int escape_len;	// number of bytes in to escape sequence
+	bool escape;	// escape unprintable characters in output?
-	const char* escape;	// bytes to escape on output
+	list<string> escape_sequences; // additional sequences of chars to escape
 
 	BroFile* f;	// or the file we're using.
 

src/LogWriterAscii.cc

@@ -6,27 +6,6 @@
 #include "LogWriterAscii.h"
 #include "NetVar.h"
 
-/**
- * Takes a string, escapes each character into its equivalent hex code (\x##), and
- * returns a string containing all escaped values.
- *
- * @param str string to escape
- * @return A std::string containing a list of escaped hex values of the form \x##
- */
-static string get_escaped_string(const std::string& str)
-{
-	char tbuf[16];
-	string esc = "";
-
-	for ( size_t i = 0; i < str.length(); ++i )
-		{
-		snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]);
-		esc += tbuf;
-		}
-
-	return esc;
-}
-
 LogWriterAscii::LogWriterAscii()
 	{
 	file = 0;
@@ -59,7 +38,8 @@ LogWriterAscii::LogWriterAscii()
 	memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(),
 	       header_prefix_len);
 
-	desc.SetEscape(separator, separator_len);
+	desc.EnableEscaping();
+	desc.AddEscapeSequence(separator, separator_len);
 	}
 
 LogWriterAscii::~LogWriterAscii()
@@ -108,7 +88,13 @@ bool LogWriterAscii::DoInit(string path, int num_fields,
 		if( fwrite(str.c_str(), str.length(), 1, file) != 1 )
 			goto write_error;
 
-		if ( ! WriteHeaderField("path", path) )
+		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
+		            string(set_separator, set_separator_len))) &&
+		        WriteHeaderField("empty_field", get_escaped_string(
+		            string(empty_field, empty_field_len))) &&
+		        WriteHeaderField("unset_field", get_escaped_string(
+		            string(unset_field, unset_field_len))) &&
+		        WriteHeaderField("path", path)) )
 			goto write_error;
 
 		string names;
@@ -238,14 +224,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
 			break;
 			}
 
+		desc->AddEscapeSequence(set_separator, set_separator_len);
 		for ( int j = 0; j < val->val.set_val.size; j++ )
 			{
 			if ( j > 0 )
-				desc->AddN(set_separator, set_separator_len);
+				desc->AddRaw(set_separator, set_separator_len);
 
 			if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) )
+				{
+				desc->RemoveEscapeSequence(set_separator, set_separator_len);
 				return false;
 				}
+			}
+		desc->RemoveEscapeSequence(set_separator, set_separator_len);
 
 		break;
 		}
@@ -258,14 +249,19 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
 			break;
 			}
 
+		desc->AddEscapeSequence(set_separator, set_separator_len);
 		for ( int j = 0; j < val->val.vector_val.size; j++ )
 			{
 			if ( j > 0 )
-				desc->AddN(set_separator, set_separator_len);
+				desc->AddRaw(set_separator, set_separator_len);
 
 			if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) )
+				{
+				desc->RemoveEscapeSequence(set_separator, set_separator_len);
 				return false;
 				}
+			}
+		desc->RemoveEscapeSequence(set_separator, set_separator_len);
 
 		break;
 		}

src/util.cc

@@ -41,6 +41,27 @@
 #include "Net.h"
 #include "Reporter.h"
 
+/**
+ * Takes a string, escapes each character into its equivalent hex code (\x##), and
+ * returns a string containing all escaped values.
+ *
+ * @param str string to escape
+ * @return A std::string containing a list of escaped hex values of the form \x##
+ */
+std::string get_escaped_string(const std::string& str)
+{
+    char tbuf[16];
+    string esc = "";
+
+    for ( size_t i = 0; i < str.length(); ++i )
+        {
+        snprintf(tbuf, sizeof(tbuf), "\\x%02x", str[i]);
+        esc += tbuf;
+        }
+
+    return esc;
+}
+
 char* copy_string(const char* s)
 	{
 	char* c = new char[strlen(s)+1];

src/util.h

@@ -89,6 +89,8 @@ void delete_each(T* t)
 		delete *it;
 	}
 
+std::string get_escaped_string(const std::string& str);
+
 extern char* copy_string(const char* s);
 extern int streq(const char* s1, const char* s2);
 

testing/btest/Baseline/core.expr-exception/reporter.log

@@ -1,13 +1,16 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	reporter
 #fields	ts	level	message	location
 #types	time	enum	string	string
-1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.783842	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.915940	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.915940	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.916118	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.916118	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.918295	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.918295	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.952193	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.952193	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.952228	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.952228	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.954761	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475168.962628	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
-1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/home/robin/bro/master/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8
+1300475169.780331	Reporter::ERROR	field value missing [c$ftp]	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.expr-exception/expr-exception.bro, line 8

testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	conn
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count

testing/btest/Baseline/core.print-bpf-filters-ipv4/output

@@ -1,20 +1,32 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1320367155.152502	-	not ip6	T	T
+1323275491.966719	-	not ip6	T	T
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1320367155.379066	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6)	T	T
+1323275492.165829	-	(((((((((((((((((((((((((port 53) or (tcp port 989)) or (tcp port 443)) or (port 6669)) or (udp and port 5353)) or (port 6668)) or (udp and port 5355)) or (tcp port 22)) or (tcp port 995)) or (port 21)) or (tcp port 25 or tcp port 587)) or (port 6667)) or (tcp port 614)) or (tcp port 990)) or (udp port 137)) or (tcp port 993)) or (tcp port 5223)) or (port 514)) or (tcp port 585)) or (tcp port 992)) or (tcp port 563)) or (tcp port 994)) or (tcp port 636)) or (tcp and port (80 or 81 or 631 or 1080 or 3138 or 8000 or 8080 or 8888))) or (port 6666)) and (not ip6)	T	T
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1320367155.601980	-	port 42	T	T
+1323275492.362403	-	port 42	T	T
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	packet_filter
 #fields	ts	node	filter	init	success
 #types	time	string	string	bool	bool
-1320367155.826539	-	port 56730	T	T
+1323275492.563649	-	port 56730	T	T

testing/btest/Baseline/core.vlan-mpls/conn.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	conn
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
 #types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	loaded_scripts
 #fields	name
 #types	string

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	loaded_scripts
 #fields	name
 #types	string

testing/btest/Baseline/istate.events-ssl/receiver.http.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file
-1319568535.914761	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-
+1323276411.786237	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/istate.events-ssl/sender.http.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file
-1319568535.914761	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-
+1323276411.786237	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/istate.events/receiver.http.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file
-1319568558.542142	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-
+1323276438.655853	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/istate.events/sender.http.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file
-1319568558.542142	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-
+1323276438.655853	arKYeMETxOg	141.42.64.125	56730	125.190.109.199	80	1	GET	www.icir.org	/	-	Wget/1.10	0	9130	200	OK	-	-	-	-	-	-	-	text/html	-	-

testing/btest/Baseline/scripts.base.frameworks.communication.communication_log_baseline/send.log

@@ -1,16 +1,19 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	communication
 #fields	ts	peer	src_name	connected_peer_desc	connected_peer_addr	connected_peer_port	level	message
 #types	time	string	string	string	addr	port	string	string
-1322788789.351248	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
+1323275566.293849	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] added peer
-1322788789.354851	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
+1323275566.300180	bro	child	-	-	-	info	[#1/127.0.0.1:47757] connected
-1322788789.354956	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
+1323275566.300467	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer connected
-1322788789.354956	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
+1323275566.300467	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: version
-1322788789.355429	bro	script	-	-	-	info	connection established
+1323275566.300936	bro	script	-	-	-	info	connection established
-1322788789.355429	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
+1323275566.300936	bro	script	-	-	-	info	requesting events matching /^?(NOTHING)$?/
-1322788789.355429	bro	script	-	-	-	info	accepting state
+1323275566.300936	bro	script	-	-	-	info	accepting state
-1322788789.355967	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
+1323275566.302043	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] phase: handshake
-1322788789.355967	bro	parent	-	-	-	info	warning: no events to request
+1323275566.302043	bro	parent	-	-	-	info	warning: no events to request
-1322788789.355967	bro	parent	-	-	-	info	terminating...
+1323275566.302043	bro	parent	-	-	-	info	terminating...
-1322788789.355967	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
+1323275566.302043	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] peer_description is bro
-1322788789.355967	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection
+1323275566.302043	bro	parent	-	-	-	info	[#1/127.0.0.1:47757] closing connection

testing/btest/Baseline/scripts.base.frameworks.logging.adapt-filter/ssh-new-default.log

@@ -1,6 +1,9 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh-new-default
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167052.603186	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1323275589.577486	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1315167052.603186	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323275589.577486	1.2.3.4	1234	2.3.4.5	80	failure	US

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log

@@ -1,4 +1,7 @@
 #separator \x7c
+#set_separator|\x2c
+#empty_field|\x2d
+#unset_field|\x2d
 #path|ssh
 #fields|data|data2
 #types|string|string

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-empty/ssh.log

@@ -1,9 +1,12 @@
 PREFIX<>separator \x7c
+PREFIX<>set_separator|\x2c
+PREFIX<>empty_field|\x45\x4d\x50\x54\x59
+PREFIX<>unset_field|\x4e\x4f\x54\x2d\x53\x45\x54
 PREFIX<>path|ssh
 PREFIX<>fields|t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
 PREFIX<>types|time|addr|port|addr|port|string|string|bool
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
+1323275635.348361|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
+1323275635.348361|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
+1323275635.348361|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
+1323275635.348361|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
-1315167052.828457|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T
+1323275635.348361|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-notset-str/test.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	x	y	z
 #types	string	string	string

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-set-separator/test.log

@@ -0,0 +1,8 @@
+#separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
+#path	test
+#fields	ss
+#types	table
+CC,AA,\x2c,\x2c\x2c

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape/ssh.log

@@ -1,9 +1,12 @@
 #separator \x7c\x7c
+#set_separator||\x2c
+#empty_field||\x2d
+#unset_field||\x2d
 #path||ssh
 #fields||t||id.orig_h||id.orig_p||id.resp_h||id.resp_p||status||country
 #types||time||addr||port||addr||port||string||string
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||success||unknown
+1323275761.036351||1.2.3.4||1234||2.3.4.5||80||success||unknown
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||US
+1323275761.036351||1.2.3.4||1234||2.3.4.5||80||failure||US
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
+1323275761.036351||1.2.3.4||1234||2.3.4.5||80||fa\x7c\x7cure||UK
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
+1323275761.036351||1.2.3.4||1234||2.3.4.5||80||su\x7c\x7cess||BR
-1315802040.006123||1.2.3.4||1234||2.3.4.5||80||failure||MX
+1323275761.036351||1.2.3.4||1234||2.3.4.5||80||failure||MX

testing/btest/Baseline/scripts.base.frameworks.logging.ascii-timestamps/test.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	data
 #types	time

testing/btest/Baseline/scripts.base.frameworks.logging.attr-extend/ssh.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	status	country	a1	b1	b2
 #types	string	string	count	count	count

testing/btest/Baseline/scripts.base.frameworks.logging.attr/ssh.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	status	country
 #types	string	string

testing/btest/Baseline/scripts.base.frameworks.logging.empty-event/ssh.log

@@ -1,9 +1,12 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.369918	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1323275824.696040	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1315167053.369918	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323275824.696040	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167053.369918	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323275824.696040	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1315167053.369918	1.2.3.4	1234	2.3.4.5	80	success	BR
+1323275824.696040	1.2.3.4	1234	2.3.4.5	80	success	BR
-1315167053.369918	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1323275824.696040	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/scripts.base.frameworks.logging.exclude/ssh.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	port	addr	port	string	string

testing/btest/Baseline/scripts.base.frameworks.logging.file/ssh.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	t	f
 #types	time	file
-1315167053.585834	Foo.log
+1323275842.508479	Foo.log

testing/btest/Baseline/scripts.base.frameworks.logging.include/ssh.log

@@ -1,9 +1,12 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	t	id.orig_h
 #types	time	addr
-1315167053.694473	1.2.3.4
+1323275846.507507	1.2.3.4
-1315167053.694473	1.2.3.4
+1323275846.507507	1.2.3.4
-1315167053.694473	1.2.3.4
+1323275846.507507	1.2.3.4
-1315167053.694473	1.2.3.4
+1323275846.507507	1.2.3.4
-1315167053.694473	1.2.3.4
+1323275846.507507	1.2.3.4

testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/local.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	local
 #fields	ts	id.orig_h
 #types	time	addr

testing/btest/Baseline/scripts.base.frameworks.logging.path-func-column-demote/remote.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	remote
 #fields	ts	id.orig_h
 #types	time	addr

testing/btest/Baseline/scripts.base.frameworks.logging.path-func/output

@@ -6,37 +6,58 @@ static-prefix-1-US.log
 static-prefix-2-MX2.log
 static-prefix-2-UK.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-0-BR
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	success	BR
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	success	BR
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-0-MX3
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	failure	MX3
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	failure	MX3
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-0-unknown
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	success	unknown
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-1-MX
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	failure	MX
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-1-US
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	failure	US
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-2-MX2
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	failure	MX2
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	failure	MX2
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	static-prefix-2-UK
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.803346	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323275860.153895	1.2.3.4	1234	2.3.4.5	80	failure	UK

testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.failure.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test.failure
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.923545	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323275882.725518	1.2.3.4	1234	2.3.4.5	80	failure	US

testing/btest/Baseline/scripts.base.frameworks.logging.pred/test.success.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test.success
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167053.923545	1.2.3.4	1234	2.3.4.5	80	success	-
+1323275882.725518	1.2.3.4	1234	2.3.4.5	80	success	-

testing/btest/Baseline/scripts.base.frameworks.logging.remote-types/receiver.test.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x45\x4d\x50\x54\x59
+#unset_field	\x2d
 #path	test
 #fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve
 #types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector
-T	-42	Test::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315167054.320958	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY
+T	-42	Test::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1323275900.286451	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY

testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.failure.log

@@ -1,7 +1,10 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test.failure
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.log

@@ -1,9 +1,12 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	success	-
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	success	-
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	success	BR
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	success	BR
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/scripts.base.frameworks.logging.remote/sender.test.success.log

@@ -1,6 +1,9 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test.success
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	success	-
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	success	-
-1315167059.502670	1.2.3.4	1234	2.3.4.5	80	success	BR
+1323276013.684540	1.2.3.4	1234	2.3.4.5	80	success	BR

testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.failure.log

@@ -1,6 +1,9 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh.failure
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167066.575996	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323276050.103643	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167066.575996	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323276050.103643	1.2.3.4	1234	2.3.4.5	80	failure	UK

testing/btest/Baseline/scripts.base.frameworks.logging.remove/ssh.log

@@ -1,7 +1,10 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167066.575996	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323276050.103643	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167066.575996	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323276050.103643	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1315167066.575996	1.2.3.4	1234	2.3.4.5	80	failure	BR
+1323276050.103643	1.2.3.4	1234	2.3.4.5	80	failure	BR

testing/btest/Baseline/scripts.base.frameworks.logging.rotate-custom/out

@@ -18,11 +18,14 @@ custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, pat
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
 custom rotate, [writer=Log::WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
+#empty_field	\x2d
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #path	test
 #path	test2
 #separator \x09
+#set_separator	\x2c
 #types	time	addr	port	addr	port
+#unset_field	\x2d
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
 1299470395.000000	10.0.0.2	20	10.0.0.3	0
 1299470405.000000	10.0.0.1	20	10.0.0.2	1025

testing/btest/Baseline/scripts.base.frameworks.logging.rotate/out

@@ -10,6 +10,9 @@ test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
 test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 > test.2011-03-07-03-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -17,6 +20,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299470395.000000	10.0.0.2	20	10.0.0.3	0
 > test.2011-03-07-04-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -24,6 +30,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299473995.000000	10.0.0.2	20	10.0.0.3	1
 > test.2011-03-07-05-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -31,6 +40,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299477595.000000	10.0.0.2	20	10.0.0.3	2
 > test.2011-03-07-06-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -38,6 +50,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299481195.000000	10.0.0.2	20	10.0.0.3	3
 > test.2011-03-07-07-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -45,6 +60,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299484795.000000	10.0.0.2	20	10.0.0.3	4
 > test.2011-03-07-08-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -52,6 +70,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299488395.000000	10.0.0.2	20	10.0.0.3	5
 > test.2011-03-07-09-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -59,6 +80,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299491995.000000	10.0.0.2	20	10.0.0.3	6
 > test.2011-03-07-10-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -66,6 +90,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299495595.000000	10.0.0.2	20	10.0.0.3	7
 > test.2011-03-07-11-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port
@@ -73,6 +100,9 @@ test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
 1299499195.000000	10.0.0.2	20	10.0.0.3	8
 > test.2011-03-07-12-00-05.log
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	test
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 #types	time	addr	port	addr	port

testing/btest/Baseline/scripts.base.frameworks.logging.stdout/output

@@ -1,9 +1,12 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	/dev/stdout
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167067.393739	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1323276116.980214	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1315167067.393739	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323276116.980214	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167067.393739	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323276116.980214	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1315167067.393739	1.2.3.4	1234	2.3.4.5	80	success	BR
+1323276116.980214	1.2.3.4	1234	2.3.4.5	80	success	BR
-1315167067.393739	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1323276116.980214	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/scripts.base.frameworks.logging.test-logging/ssh.log

@@ -1,9 +1,12 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1315167067.507542	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1323276164.251500	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1315167067.507542	1.2.3.4	1234	2.3.4.5	80	failure	US
+1323276164.251500	1.2.3.4	1234	2.3.4.5	80	failure	US
-1315167067.507542	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1323276164.251500	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1315167067.507542	1.2.3.4	1234	2.3.4.5	80	success	BR
+1323276164.251500	1.2.3.4	1234	2.3.4.5	80	success	BR
-1315167067.507542	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1323276164.251500	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x45\x4d\x50\x54\x59
+#unset_field	\x2d
 #path	ssh
 #fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
 #types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
-T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1323276169.782634	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}

testing/btest/Baseline/scripts.base.frameworks.logging.unset-record/testing.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	testing
 #fields	a.val1	a.val2	b
 #types	count	count	count

testing/btest/Baseline/scripts.base.frameworks.logging.vec/ssh.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	ssh
 #fields	vec
 #types	vector

testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log

@@ -1,7 +1,10 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	metrics
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1317950616.401733	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1323276206.622034	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1317950616.401733	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+1323276206.622034	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
-1317950616.401733	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1323276206.622034	TEST_METRIC	foo-bar	7.2.1.5	-	-	2

testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log

@@ -1,7 +1,10 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	metrics
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1315167083.455574	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
+1323276222.644659	TEST_METRIC	foo-bar	6.5.4.3	-	-	2
-1315167083.455574	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
+1323276222.644659	TEST_METRIC	foo-bar	1.2.3.4	-	-	3
-1315167083.455574	TEST_METRIC	foo-bar	7.2.1.5	-	-	1
+1323276222.644659	TEST_METRIC	foo-bar	7.2.1.5	-	-	1

testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	notice
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
-1316952194.679491	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	1.2.3.4	-	-
+1323276259.751377	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	1.2.3.4	-	-

testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log

@@ -1,6 +1,9 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	notice
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
-1316952223.891502	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	1.2.3.4	-	-
+1323276275.255136	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	1.2.3.4	-	-
-1316952223.891502	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	6.5.4.3	-	-
+1323276275.255136	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	6.5.4.3	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	notice
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
-1316952264.931290	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	-	-	-
+1323276288.745044	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	-	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	notice
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
 #types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
-1316950574.408256	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	-	-	-
+1323276310.879512	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-	-	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log

@@ -1,5 +1,8 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	notice
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
 #types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double
-1316950497.513136	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-
+1323276329.733314	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_LOG	6	3600.000000	-	-	-	-	-	-

testing/btest/Baseline/scripts.base.protocols.http.100-continue/http.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file

testing/btest/Baseline/scripts.base.protocols.http.http-extract-files/http.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file

testing/btest/Baseline/scripts.base.protocols.http.http-mime-and-md5/http.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	mime_type	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	string	file

testing/btest/Baseline/scripts.base.protocols.http.http-pipelining/http.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	http
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	md5	extraction_file
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table	string	string	table	string	file

testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	irc
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	channels	command	value	addl	tags	dcc_file_name	dcc_file_size	extraction_file
 #types	time	string	addr	port	addr	port	string	string	table	string	string	string	table	string	count	file

testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	irc
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	channels	command	value	addl	tags	dcc_file_name	dcc_file_size	dcc_mime_type	extraction_file
 #types	time	string	addr	port	addr	port	string	string	table	string	string	string	table	string	count	string	file

testing/btest/Baseline/scripts.base.protocols.smtp.basic/smtp.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	smtp
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
 #types	time	string	addr	port	addr	port	count	string	string	table	string	string	table	string	string	string	string	addr	string	string	string	vector	string

testing/btest/Baseline/scripts.base.protocols.smtp.mime-extract/smtp_entities.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	smtp_entities
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	file	string

testing/btest/Baseline/scripts.base.protocols.smtp.mime/smtp_entities.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	smtp_entities
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	filename	content_len	mime_type	md5	extraction_file	excerpt
 #types	time	string	addr	port	addr	port	count	string	count	string	string	file	string

testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-all.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	known_hosts
 #fields	ts	host
 #types	time	addr

testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-local.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	known_hosts
 #fields	ts	host
 #types	time	addr

testing/btest/Baseline/scripts.policy.protocols.conn.known-hosts/knownhosts-remote.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	known_hosts
 #fields	ts	host
 #types	time	addr

testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-all.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	known_services
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table

testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-local.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	known_services
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table

testing/btest/Baseline/scripts.policy.protocols.conn.known-services/knownservices-remote.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	known_services
 #fields	ts	host	port_num	port_proto	service
 #types	time	addr	port	enum	table

testing/btest/Baseline/scripts.policy.protocols.dns.event-priority/dns.log

@@ -1,4 +1,7 @@
 #separator \x09
+#set_separator	\x2c
+#empty_field	\x2d
+#unset_field	\x2d
 #path	dns
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	QR	AA	TC	RD	RA	Z	answers	TTLs	auth	addl
 #types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	bool	count	vector	vector	table	table

testing/btest/scripts/base/frameworks/logging/ascii-escape-set-separator.bro

@@ -0,0 +1,21 @@
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff test.log
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		ss: set[string];
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Log]);
+
+
+	Log::write(Test::LOG, [$ss=set("AA", ",", ",,", "CC")]);
+}
+