Merge branch 'topic/bernhard/input-logging-commmon-functions' into topic/bernhard/sqlite

scripts/base/frameworks/input/main.bro

@@ -11,6 +11,24 @@ export {
 	## The default reader mode used. Defaults to `MANUAL`.
 	const default_mode = MANUAL &redef;
 
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	## Can be overwritten by individual writers.	
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;
+
 	## Flag that controls if the input framework accepts records
 	## that contain types that are not supported (at the moment
 	## file and function). If true, the input framework will

scripts/base/frameworks/input/readers/ascii.bro

@@ -7,15 +7,15 @@ module InputAscii;
 export {
 	## Separator between fields.
 	## Please note that the separator has to be exactly one character long
-	const separator = "\t" &redef;
+	const separator = Input::separator &redef;
 
 	## Separator between set elements.
 	## Please note that the separator has to be exactly one character long
-	const set_separator = "," &redef;
+	const set_separator = Input::set_separator &redef;
 
 	## String to use for empty fields.
-	const empty_field = "(empty)" &redef;
+	const empty_field = Input::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Input::unset_field &redef;
 }

scripts/base/frameworks/logging/main.bro

@@ -17,6 +17,23 @@ export {
 	## anything else.
 	const default_writer = WRITER_ASCII &redef;
 
+	## Default separator between fields for logwriters.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious. 
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;	
+
 	## Type defining the content of a logging stream.
 	type Stream: record {
 		## A record type defining the log's columns.

scripts/base/frameworks/logging/writers/ascii.bro

@@ -25,17 +25,17 @@ export {
 	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
-	const separator = "\t" &redef;
+	const separator = Log::separator &redef;
 
 	## Separator between set elements.
-	const set_separator = "," &redef;
+	const set_separator = Log::set_separator &redef;
 
 	## String to use for empty fields. This should be different from
         ## *unset_field* to make the output non-ambigious. 
-	const empty_field = "(empty)" &redef;
+	const empty_field = Log::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Log::unset_field &redef;
 }
 
 # Default function to postprocess a rotated ASCII log file. It moves the rotated

src/AsciiInputOutput.cc

@@ -0,0 +1,485 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "config.h"
+
+#include <sstream>
+#include <errno.h>
+#include "AsciiInputOutput.h"
+#include "bro_inet_ntop.h"
+
+AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) 
+	{
+	thread = t;
+	}
+
+AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, 
+				const string & empty_field, const string & unset_field) 
+	{
+	thread = t;
+	this->separator = separator;
+	this->set_separator = set_separator;
+	this->empty_field = empty_field;
+	this->unset_field = unset_field;
+	}
+
+
+AsciiInputOutput::~AsciiInputOutput()
+	{
+	}
+
+bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const
+	{
+	if ( ! val->present )
+		{
+		desc->Add(unset_field);
+		return true;
+		}
+
+	switch ( val->type ) {
+
+	case TYPE_BOOL:
+		desc->Add(val->val.int_val ? "T" : "F");
+		break;
+
+	case TYPE_INT:
+		desc->Add(val->val.int_val);
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		desc->Add(val->val.uint_val);
+		break;
+
+	case TYPE_PORT:
+		desc->Add(val->val.port_val.port);
+		break;
+
+	case TYPE_SUBNET:
+		desc->Add(Render(val->val.subnet_val));
+		break;
+
+	case TYPE_ADDR:
+		desc->Add(Render(val->val.addr_val));
+		break;
+
+	case TYPE_DOUBLE:
+		// Rendering via Add() truncates trailing 0s after the
+		// decimal point. The difference with TIME/INTERVAL is mainly
+		// to keep the log format consistent.
+		desc->Add(val->val.double_val);
+		break;
+
+	case TYPE_INTERVAL:
+	case TYPE_TIME:
+		// Rendering via Render() keeps trailing 0s after the decimal
+		// point. The difference with DOUBLEis mainly to keep the log
+		// format consistent.
+		desc->Add(Render(val->val.double_val));
+		break;
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		{
+		int size = val->val.string_val.length;
+		const char* data = val->val.string_val.data;
+
+		if ( ! size )
+			{
+			desc->Add(empty_field);
+			break;
+			}
+
+		if ( size == unset_field.size() && memcmp(data, unset_field.data(), size) == 0 )
+			{
+			// The value we'd write out would match exactly the
+			// place-holder we use for unset optional fields. We
+			// escape the first character so that the output
+			// won't be ambigious.
+			static const char hex_chars[] = "0123456789abcdef";
+			char hex[6] = "\\x00";
+			hex[2] = hex_chars[((*data) & 0xf0) >> 4];
+			hex[3] = hex_chars[(*data) & 0x0f];
+			desc->AddRaw(hex, 4);
+
+			++data;
+			--size;
+			}
+
+		if ( size )
+			desc->AddN(data, size);
+
+		break;
+		}
+
+	case TYPE_TABLE:
+		{
+		if ( ! val->val.set_val.size )
+			{
+			desc->Add(empty_field);
+			break;
+			}
+
+		desc->AddEscapeSequence(set_separator);
+		for ( int j = 0; j < val->val.set_val.size; j++ )
+			{
+			if ( j > 0 )
+				desc->AddRaw(set_separator);
+
+			if ( ! ValToODesc(desc, val->val.set_val.vals[j], field) )
+				{
+				desc->RemoveEscapeSequence(set_separator);
+				return false;
+				}
+			}
+		desc->RemoveEscapeSequence(set_separator);
+
+		break;
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( ! val->val.vector_val.size )
+			{
+			desc->Add(empty_field);
+			break;
+			}
+
+		desc->AddEscapeSequence(set_separator);
+		for ( int j = 0; j < val->val.vector_val.size; j++ )
+			{
+			if ( j > 0 )
+				desc->AddRaw(set_separator);
+
+			if ( ! ValToODesc(desc, val->val.vector_val.vals[j], field) )
+				{
+				desc->RemoveEscapeSequence(set_separator);
+				return false;
+				}
+			}
+		desc->RemoveEscapeSequence(set_separator);
+
+		break;
+		}
+
+	default:
+		thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, field->name));
+		return false;
+	}
+
+	return true;
+	}
+
+
+threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag type, TypeTag subtype) const
+	{
+	if ( s.compare(unset_field) == 0 )  // field is not set...
+		return new threading::Value(type, false);
+
+	threading::Value* val = new threading::Value(type, true);
+	char* end = 0;
+	errno = 0;
+
+	switch ( type ) {
+	case TYPE_ENUM:
+	case TYPE_STRING:
+		s = get_unescaped_string(s);
+		val->val.string_val.length = s.size();
+		val->val.string_val.data = copy_string(s.c_str());
+		break;
+
+	case TYPE_BOOL:
+		if ( s == "T" )
+			val->val.int_val = 1;
+		else if ( s == "F" )
+			val->val.int_val = 0;
+		else
+			{
+			thread->Error(thread->Fmt("Field: %s Invalid value for boolean: %s",
+				  name.c_str(), s.c_str()));
+			return 0;
+			}
+		break;
+
+	case TYPE_INT:
+		val->val.int_val = strtoll(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		val->val.double_val = strtod(s.c_str(), &end);
+		if ( CheckNumberError(s, end) )
+			return 0;
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+		break;
+
+	case TYPE_PORT:
+		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		break;
+
+	case TYPE_SUBNET:
+		{
+		s = get_unescaped_string(s);
+		size_t pos = s.find("/");
+		if ( pos == s.npos )
+			{
+			thread->Error(thread->Fmt("Invalid value for subnet: %s", s.c_str()));
+			return 0;
+			}
+
+		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
+
+		if ( CheckNumberError(s, end) )
+			return 0;
+
+		string addr = s.substr(0, pos);
+
+		val->val.subnet_val.prefix = StringToAddr(addr);
+		val->val.subnet_val.length = width;
+		break;
+		}
+
+	case TYPE_ADDR:
+		s = get_unescaped_string(s);
+		val->val.addr_val = StringToAddr(s);
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		// First - common initialization
+		// Then - initialization for table.
+		// Then - initialization for vector.
+		// Then - common stuff
+		{
+		// how many entries do we have...
+		unsigned int length = 1;
+		for ( unsigned int i = 0; i < s.size(); i++ )
+			{
+			if ( s[i] == set_separator[0] )
+				length++;
+			}
+
+		unsigned int pos = 0;
+
+		if ( s.compare(empty_field) == 0 )
+			length = 0;
+
+		threading::Value** lvals = new threading::Value* [length];
+
+		if ( type == TYPE_TABLE )
+			{
+			val->val.set_val.vals = lvals;
+			val->val.set_val.size = length;
+			}
+
+		else if ( type == TYPE_VECTOR )
+			{
+			val->val.vector_val.vals = lvals;
+			val->val.vector_val.size = length;
+			}
+
+		else
+			assert(false);
+
+		if ( length == 0 )
+			break; //empty
+
+		istringstream splitstream(s);
+		while ( splitstream )
+			{
+			string element;
+
+			if ( ! getline(splitstream, element, set_separator[0]) )
+				break;
+
+			if ( pos >= length )
+				{
+				thread->Error(thread->Fmt("Internal error while parsing set. pos %d >= length %d."
+				          " Element: %s", pos, length, element.c_str()));
+				break;
+				}
+
+			threading::Value* newval = EntryToVal(element, name, subtype);
+			if ( newval == 0 )
+				{
+				thread->Error("Error while reading set");
+				return 0;
+				}
+
+			lvals[pos] = newval;
+
+			pos++;
+			}
+
+		// Test if the string ends with a set_separator... or if the
+		// complete string is empty. In either of these cases we have
+		// to push an empty val on top of it.
+		if ( s.empty() || *s.rbegin() == set_separator[0] )
+			{
+			lvals[pos] = EntryToVal("", name, subtype);
+			if ( lvals[pos] == 0 )
+				{
+				thread->Error("Error while trying to add empty set element");
+				return 0;
+				}
+
+			pos++;
+			}
+
+		if ( pos != length )
+			{
+			thread->Error(thread->Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
+			return 0;
+			}
+
+		break;
+		}
+
+	default:
+		thread->Error(thread->Fmt("unsupported field format %d for %s", type,
+		name.c_str()));
+		return 0;
+	}
+
+	return val;
+	}
+
+bool AsciiInputOutput::CheckNumberError(const string& s, const char * end) const
+	{
+	// Do this check first, before executing s.c_str() or similar.
+	// otherwise the value to which *end is pointing at the moment might
+	// be gone ...
+	bool endnotnull =  (*end != '\0');
+
+	if ( s.length() == 0 )
+		{
+		thread->Error("Got empty string for number field");
+		return true;
+		}
+
+	if ( end == s.c_str() ) {
+		thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str()));
+		return true;
+	}
+
+	if ( endnotnull )
+		thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+
+	if ( errno == EINVAL )
+		{
+		thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str()));
+		return true;
+		}
+
+	else if ( errno == ERANGE )
+		{
+		thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str()));
+		return true;
+		}
+
+	return false;
+	}
+
+string AsciiInputOutput::Render(const threading::Value::addr_t& addr) 
+	{
+	if ( addr.family == IPv4 )
+		{
+		char s[INET_ADDRSTRLEN];
+
+		if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
+			return "<bad IPv4 address conversion>";
+		else
+			return s;
+		}
+	else
+		{
+		char s[INET6_ADDRSTRLEN];
+
+		if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
+			return "<bad IPv6 address conversion>";
+		else
+			return s;
+		}
+	}
+
+TransportProto AsciiInputOutput::StringToProto(const string &proto) const
+	{
+	if ( proto == "unknown" )
+		return TRANSPORT_UNKNOWN;
+	else if ( proto == "tcp" )
+		return TRANSPORT_TCP;
+	else if ( proto == "udp" )
+		return TRANSPORT_UDP;
+	else if ( proto == "icmp" )
+		return TRANSPORT_ICMP;
+
+	thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
+
+	return TRANSPORT_UNKNOWN;
+	}
+
+
+// More or less verbose copy from IPAddr.cc -- which uses reporter.
+threading::Value::addr_t AsciiInputOutput::StringToAddr(const string &s) const
+	{
+		threading::Value::addr_t val;
+
+	if ( s.find(':') == std::string::npos ) // IPv4.
+		{
+		val.family = IPv4;
+
+		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
+			{
+			thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
+			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
+			}
+		}
+
+	else
+		{
+		val.family = IPv6;
+		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
+			{
+			thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
+			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
+			}
+		}
+
+	return val;
+	}
+
+string AsciiInputOutput::Render(const threading::Value::subnet_t& subnet) 
+	{
+	char l[16];
+
+	if ( subnet.prefix.family == IPv4 )
+		modp_uitoa10(subnet.length - 96, l);
+	else
+		modp_uitoa10(subnet.length, l);
+
+	string s = Render(subnet.prefix) + "/" + l;
+
+	return s;
+	}
+
+string AsciiInputOutput::Render(double d) 
+	{
+	char buf[256];
+	modp_dtoa(d, buf, 6);
+	return buf;
+	}
+

src/AsciiInputOutput.h

@@ -0,0 +1,81 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef AsciiInputOutput_h
+#define AsciiInputOutput_h
+
+#include "Desc.h"
+#include "threading/MsgThread.h"
+
+class AsciiInputOutput {
+	public: 
+		// Constructor that leaves separators, etc empty.
+		// Use if you just need functionality like StringToAddr, etc.
+		AsciiInputOutput(threading::MsgThread*);
+
+		// Constructor that defines all separators, etc.
+		// Use if you need either ValToODesc or EntryToVal.
+		AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, 
+				const string & empty_field, const string & unset_field);
+		~AsciiInputOutput();
+
+
+		// converts a threading value to the corresponding ascii representation
+		// returns false & logs an error with reporter in case an error occurs
+		bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const;
+
+		// convert the ascii representation of a field into a Value
+		threading::Value* EntryToVal(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
+
+		/** Helper method to render an IP address as a string.
+		  *
+		  * @param addr The address.
+		  *
+		  * @return An ASCII representation of the address.
+		  */
+		static string Render(const threading::Value::addr_t& addr);
+
+		/** Helper method to render an subnet value as a string.
+		  *
+		  * @param addr The address.
+		  *
+		  * @return An ASCII representation of the address.
+		  */
+		static string Render(const threading::Value::subnet_t& subnet);
+
+		/** Helper method to render a double in Bro's standard precision.
+		  *
+		  * @param d The double.
+		  *
+		  * @return An ASCII representation of the double.
+		  */
+		static string Render(double d);
+
+		/**
+		 *  Convert a string into a TransportProto. This is just a utility
+		 *  function for Readers.
+		 *
+		 * @param proto the transport protocol
+		 */
+		TransportProto StringToProto(const string &proto) const;
+
+		/**
+		 * Convert a string into a Value::addr_t.  This is just a utility
+		 * function for Readers.
+		 *
+		 * @param addr containing an ipv4 or ipv6 address
+		 */
+		threading::Value::addr_t StringToAddr(const string &addr) const;
+
+	private:
+		bool CheckNumberError(const string& s, const char * end) const;
+
+		string separator;
+		string set_separator;
+		string empty_field;
+		string unset_field;
+		string meta_prefix;
+
+		threading::MsgThread* thread;
+};
+
+#endif /* AsciiInputOuput_h */

src/CMakeLists.txt

@@ -284,6 +284,7 @@ set(bro_SRCS
     Analyzer.cc
     Anon.cc
     ARP.cc
+    AsciiInputOutput.cc
     Attr.cc
     AYIYA.cc
     BackDoor.cc

src/Desc.h

@@ -57,9 +57,13 @@ public:
 	void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); }
 	void AddEscapeSequence(const char* s, size_t n)
 	    { escape_sequences.push_back(string(s, n)); }
+	void AddEscapeSequence(const string & s)
+	    { escape_sequences.push_back(s); }
 	void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); }
 	void RemoveEscapeSequence(const char* s, size_t n)
 	    { escape_sequences.remove(string(s, n)); }
+	void RemoveEscapeSequence(const string & s)
+	    { escape_sequences.remove(s); }	
 
 	void PushIndent();
 	void PopIndent();
@@ -114,6 +118,7 @@ public:
 
 	// Bypasses the escaping enabled via SetEscape().
 	void AddRaw(const char* s, int len)	{ AddBytesRaw(s, len); }
+	void AddRaw(const string &s)		{ AddBytesRaw(s.data(), s.size()); }
 
 	// Returns the description as a string.
 	const char* Description() const		{ return (const char*) base; }

src/input/ReaderBackend.cc

@@ -281,50 +281,4 @@ bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
 	return DoHeartbeat(network_time, current_time);
 	}
 
-TransportProto ReaderBackend::StringToProto(const string &proto)
-	{
-	if ( proto == "unknown" )
-		return TRANSPORT_UNKNOWN;
-	else if ( proto == "tcp" )
-		return TRANSPORT_TCP;
-	else if ( proto == "udp" )
-		return TRANSPORT_UDP;
-	else if ( proto == "icmp" )
-		return TRANSPORT_ICMP;
-
-	Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
-
-	return TRANSPORT_UNKNOWN;
-	}
-
-
-// More or less verbose copy from IPAddr.cc -- which uses reporter.
-Value::addr_t ReaderBackend::StringToAddr(const string &s)
-	{
-	Value::addr_t val;
-
-	if ( s.find(':') == std::string::npos ) // IPv4.
-		{
-		val.family = IPv4;
-
-		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
-			{
-			Error(Fmt("Bad address: %s", s.c_str()));
-			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
-			}
-		}
-
-	else
-		{
-		val.family = IPv6;
-		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
-			{
-			Error(Fmt("Bad address: %s", s.c_str()));
-			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
-			}
-		}
-
-	return val;
-	}
-
 }

src/input/ReaderBackend.h

@@ -315,21 +315,6 @@ protected:
 	 */
 	void EndCurrentSend();
 
-	/**
-	 *  Convert a string into a TransportProto. This is just a utility
-	 *  function for Readers.
-	 *
-	 * @param proto the transport protocol
-	 */
-	TransportProto StringToProto(const string &proto);
-
-	/**
-	 * Convert a string into a Value::addr_t.  This is just a utility
-	 * function for Readers.
-	 *
-	 * @param addr containing an ipv4 or ipv6 address
-	 */
-	threading::Value::addr_t StringToAddr(const string &addr);
 
 private:
 	// Frontend that instantiated us. This object must not be accessed

src/input/readers/Ascii.cc

@@ -67,11 +67,14 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
 
 	unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(),
 			    BifConst::InputAscii::unset_field->Len());
+
+	io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field);	
 }
 
 Ascii::~Ascii()
 	{
 	DoClose();
+	delete io;	
 	}
 
 void Ascii::DoClose()
@@ -210,228 +213,7 @@ bool Ascii::GetLine(string& str)
 	return false;
 	}
 
-bool Ascii::CheckNumberError(const string& s, const char * end)
-	{
-	// Do this check first, before executing s.c_str() or similar.
-	// otherwise the value to which *end is pointing at the moment might
-	// be gone ...
-	bool endnotnull =  (*end != '\0');
 
-	if ( s.length() == 0 )
-		{
-		Error("Got empty string for number field");
-		return true;
-		}
-
-	if ( end == s.c_str() ) {
-		Error(Fmt("String '%s' contained no parseable number", s.c_str()));
-		return true;
-	}
-
-	if ( endnotnull )
-		Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
-
-	if ( errno == EINVAL )
-		{
-		Error(Fmt("String '%s' could not be converted to a number", s.c_str()));
-		return true;
-		}
-
-	else if ( errno == ERANGE )
-		{
-		Error(Fmt("Number '%s' out of supported range.", s.c_str()));
-		return true;
-		}
-
-	return false;
-	}
-
-
-Value* Ascii::EntryToVal(string s, FieldMapping field)
-	{
-	if ( s.compare(unset_field) == 0 )  // field is not set...
-		return new Value(field.type, false);
-
-	Value* val = new Value(field.type, true);
-	char* end = 0;
-	errno = 0;
-
-	switch ( field.type ) {
-	case TYPE_ENUM:
-	case TYPE_STRING:
-		s = get_unescaped_string(s);
-		val->val.string_val.length = s.size();
-		val->val.string_val.data = copy_string(s.c_str());
-		break;
-
-	case TYPE_BOOL:
-		if ( s == "T" )
-			val->val.int_val = 1;
-		else if ( s == "F" )
-			val->val.int_val = 0;
-		else
-			{
-			Error(Fmt("Field: %s Invalid value for boolean: %s",
-				  field.name.c_str(), s.c_str()));
-			return 0;
-			}
-		break;
-
-	case TYPE_INT:
-		val->val.int_val = strtoll(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			return 0;
-		break;
-
-	case TYPE_DOUBLE:
-	case TYPE_TIME:
-	case TYPE_INTERVAL:
-		val->val.double_val = strtod(s.c_str(), &end);
-		if ( CheckNumberError(s, end) )
-			return 0;
-		break;
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		val->val.uint_val = strtoull(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			return 0;
-		break;
-
-	case TYPE_PORT:
-		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			return 0;
-
-		val->val.port_val.proto = TRANSPORT_UNKNOWN;
-		break;
-
-	case TYPE_SUBNET:
-		{
-		s = get_unescaped_string(s);
-		size_t pos = s.find("/");
-		if ( pos == s.npos )
-			{
-			Error(Fmt("Invalid value for subnet: %s", s.c_str()));
-			return 0;
-			}
-
-		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
-
-		if ( CheckNumberError(s, end) )
-			return 0;
-
-		string addr = s.substr(0, pos);
-
-		val->val.subnet_val.prefix = StringToAddr(addr);
-		val->val.subnet_val.length = width;
-		break;
-		}
-
-	case TYPE_ADDR:
-		s = get_unescaped_string(s);
-		val->val.addr_val = StringToAddr(s);
-		break;
-
-	case TYPE_TABLE:
-	case TYPE_VECTOR:
-		// First - common initialization
-		// Then - initialization for table.
-		// Then - initialization for vector.
-		// Then - common stuff
-		{
-		// how many entries do we have...
-		unsigned int length = 1;
-		for ( unsigned int i = 0; i < s.size(); i++ )
-			{
-			if ( s[i] == set_separator[0] )
-				length++;
-			}
-
-		unsigned int pos = 0;
-
-		if ( s.compare(empty_field) == 0 )
-			length = 0;
-
-		Value** lvals = new Value* [length];
-
-		if ( field.type == TYPE_TABLE )
-			{
-			val->val.set_val.vals = lvals;
-			val->val.set_val.size = length;
-			}
-
-		else if ( field.type == TYPE_VECTOR )
-			{
-			val->val.vector_val.vals = lvals;
-			val->val.vector_val.size = length;
-			}
-
-		else
-			assert(false);
-
-		if ( length == 0 )
-			break; //empty
-
-		istringstream splitstream(s);
-		while ( splitstream )
-			{
-			string element;
-
-			if ( ! getline(splitstream, element, set_separator[0]) )
-				break;
-
-			if ( pos >= length )
-				{
-				Error(Fmt("Internal error while parsing set. pos %d >= length %d."
-				          " Element: %s", pos, length, element.c_str()));
-				break;
-				}
-
-			Value* newval = EntryToVal(element, field.subType());
-			if ( newval == 0 )
-				{
-				Error("Error while reading set");
-				return 0;
-				}
-
-			lvals[pos] = newval;
-
-			pos++;
-			}
-
-		// Test if the string ends with a set_separator... or if the
-		// complete string is empty. In either of these cases we have
-		// to push an empty val on top of it.
-		if ( s.empty() || *s.rbegin() == set_separator[0] )
-			{
-			lvals[pos] = EntryToVal("", field.subType());
-			if ( lvals[pos] == 0 )
-				{
-				Error("Error while trying to add empty set element");
-				return 0;
-				}
-
-			pos++;
-			}
-
-		if ( pos != length )
-			{
-			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
-			return 0;
-			}
-
-		break;
-		}
-
-	default:
-		Error(Fmt("unsupported field format %d for %s", field.type,
-		field.name.c_str()));
-		return 0;
-	}
-
-	return val;
-	}
 
 // read the entire file and send appropriate thingies back to InputMgr
 bool Ascii::DoUpdate()
@@ -543,7 +325,7 @@ bool Ascii::DoUpdate()
 				return false;
 				}
 
-			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
+			Value* val = io->EntryToVal(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype);
 			if ( val == 0 )
 				{
 				Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str()));
@@ -557,7 +339,7 @@ bool Ascii::DoUpdate()
 				assert(val->type == TYPE_PORT );
 				//	Error(Fmt("Got type %d != PORT with secondary position!", val->type));
 
-				val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]);
+				val->val.port_val.proto = io->StringToProto(stringfields[(*fit).secondary_position]);
 				}
 
 			fields[fpos] = val;

src/input/readers/Ascii.h

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "../ReaderBackend.h"
+#include "../../AsciiInputOutput.h"
 
 namespace input { namespace reader {
 
@@ -47,8 +48,6 @@ private:
 
 	bool ReadHeader(bool useCached);
 	bool GetLine(string& str);
-	threading::Value* EntryToVal(string s, FieldMapping type);
-	bool CheckNumberError(const string& s, const char * end);
 
 	ifstream* file;
 	time_t mtime;
@@ -64,6 +63,8 @@ private:
 	string set_separator;
 	string empty_field;
 	string unset_field;
+
+	AsciiInputOutput* io;
 };
 
 

src/input/readers/Benchmark.cc

@@ -25,11 +25,15 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend)
 	stopspreadat = int(BifConst::InputBenchmark::stopspreadat);
 	timedspread = double(BifConst::InputBenchmark::timedspread);
 	heartbeat_interval = double(BifConst::Threading::heartbeat_interval);
+
+	io = new AsciiInputOutput(this);
 	}
 
 Benchmark::~Benchmark()
 	{
 	DoClose();
+
+	delete io;
 	}
 
 void Benchmark::DoClose()
@@ -162,13 +166,13 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 
 	case TYPE_SUBNET:
 		{
-		val->val.subnet_val.prefix = StringToAddr("192.168.17.1");
+		val->val.subnet_val.prefix = io->StringToAddr("192.168.17.1");
 		val->val.subnet_val.length = 16;
 		}
 		break;
 
 	case TYPE_ADDR:
-		val->val.addr_val = StringToAddr("192.168.17.1");
+		val->val.addr_val = io->StringToAddr("192.168.17.1");
 		break;
 
 	case TYPE_TABLE:

src/input/readers/Benchmark.h

@@ -4,6 +4,7 @@
 #define INPUT_READERS_BENCHMARK_H
 
 #include "../ReaderBackend.h"
+#include "../../AsciiInputOutput.h"
 
 namespace input { namespace reader {
 
@@ -38,6 +39,8 @@ private:
 	double heartbeatstarttime;
 	double timedspread;
 	double heartbeat_interval;
+
+	AsciiInputOutput* io;
 };
 
 

src/logging/WriterBackend.cc

@@ -1,7 +1,6 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "util.h"
-#include "bro_inet_ntop.h"
 #include "threading/SerialTypes.h"
 
 #include "Manager.h"
@@ -328,46 +327,3 @@ bool WriterBackend::OnHeartbeat(double network_time, double current_time)
 	SendOut(new FlushWriteBufferMessage(frontend));
 	return DoHeartbeat(network_time, current_time);
 	}
-
-string WriterBackend::Render(const threading::Value::addr_t& addr) const
-	{
-	if ( addr.family == IPv4 )
-		{
-		char s[INET_ADDRSTRLEN];
-
-		if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
-			return "<bad IPv4 address conversion>";
-		else
-			return s;
-		}
-	else
-		{
-		char s[INET6_ADDRSTRLEN];
-
-		if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
-			return "<bad IPv6 address conversion>";
-		else
-			return s;
-		}
-	}
-
-string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
-	{
-	char l[16];
-
-	if ( subnet.prefix.family == IPv4 )
-		modp_uitoa10(subnet.length - 96, l);
-	else
-		modp_uitoa10(subnet.length, l);
-
-	string s = Render(subnet.prefix) + "/" + l;
-
-	return s;
-	}
-
-string WriterBackend::Render(double d) const
-	{
-	char buf[256];
-	modp_dtoa(d, buf, 6);
-	return buf;
-	}

src/logging/WriterBackend.h

@@ -256,30 +256,6 @@ public:
 	 */
 	bool FinishedRotation();
 
-	/** Helper method to render an IP address as a string.
-	  *
-	  * @param addr The address.
-	  *
-	  * @return An ASCII representation of the address.
-	  */
-	string Render(const threading::Value::addr_t& addr) const;
-
-	/** Helper method to render an subnet value as a string.
-	  *
-	  * @param addr The address.
-	  *
-	  * @return An ASCII representation of the address.
-	  */
-	string Render(const threading::Value::subnet_t& subnet) const;
-
-	/** Helper method to render a double in Bro's standard precision.
-	  *
-	  * @param d The double.
-	  *
-	  * @return An ASCII representation of the double.
-	  */
-	string Render(double d) const;
-
 	// Overridden from MsgThread.
 	virtual bool OnHeartbeat(double network_time, double current_time);
 	virtual bool OnFinish(double network_time);

src/logging/writers/Ascii.cc

@@ -24,33 +24,35 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
 	include_meta = BifConst::LogAscii::include_meta;
 
-	separator_len = BifConst::LogAscii::separator->Len();
+	separator.assign(
-	separator = new char[separator_len];
+			(const char*) BifConst::LogAscii::separator->Bytes(),
-	memcpy(separator, BifConst::LogAscii::separator->Bytes(),
+			BifConst::LogAscii::separator->Len()
-	       separator_len);
+			);
 
-	set_separator_len = BifConst::LogAscii::set_separator->Len();
+	set_separator.assign(
-	set_separator = new char[set_separator_len];
+			(const char*) BifConst::LogAscii::set_separator->Bytes(),
-	memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(),
+			BifConst::LogAscii::set_separator->Len()
-	       set_separator_len);
+			);
 
-	empty_field_len = BifConst::LogAscii::empty_field->Len();
+	empty_field.assign(
-	empty_field = new char[empty_field_len];
+			(const char*) BifConst::LogAscii::empty_field->Bytes(),
-	memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(),
+			BifConst::LogAscii::empty_field->Len()
-	       empty_field_len);
+			);
 
-	unset_field_len = BifConst::LogAscii::unset_field->Len();
+	unset_field.assign(
-	unset_field = new char[unset_field_len];
+			(const char*) BifConst::LogAscii::unset_field->Bytes(),
-	memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(),
+			BifConst::LogAscii::unset_field->Len()
-	       unset_field_len);
+			);
 
-	meta_prefix_len = BifConst::LogAscii::meta_prefix->Len();
+	meta_prefix.assign(
-	meta_prefix = new char[meta_prefix_len];
+			(const char*) BifConst::LogAscii::meta_prefix->Bytes(),
-	memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(),
+			BifConst::LogAscii::meta_prefix->Len()
-	       meta_prefix_len);
+			);
 
 	desc.EnableEscaping();
-	desc.AddEscapeSequence(separator, separator_len);
+	desc.AddEscapeSequence(separator);
+
+	io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field);
 	}
 
 Ascii::~Ascii()
@@ -61,17 +63,12 @@ Ascii::~Ascii()
 		abort();
 		}
 
-	delete [] separator;
+	delete io;
-	delete [] set_separator;
-	delete [] empty_field;
-	delete [] unset_field;
-	delete [] meta_prefix;
 	}
 
 bool Ascii::WriteHeaderField(const string& key, const string& val)
 	{
-	string str = string(meta_prefix, meta_prefix_len) +
+	string str = meta_prefix + key + separator + val + "\n";
-		key + string(separator, separator_len) + val + "\n";
 
 	return safe_write(fd, str.c_str(), str.length());
 	}
@@ -136,8 +133,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 			{
 			if ( i > 0 )
 				{
-				names += string(separator, separator_len);
+				names += separator;
-				types += string(separator, separator_len);
+				types += separator;
 				}
 
 			names += string(fields[i]->name);
@@ -154,20 +151,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 			return true;
 			}
 
-		string str = string(meta_prefix, meta_prefix_len)
+		string str = meta_prefix
 			+ "separator " // Always use space as separator here.
-			+ get_escaped_string(string(separator, separator_len), false)
+			+ get_escaped_string(separator, false)
 			+ "\n";
 
 		if ( ! safe_write(fd, str.c_str(), str.length()) )
 			goto write_error;
 
-		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
+		if ( ! (WriteHeaderField("set_separator", get_escaped_string(set_separator, false)) &&
-			    string(set_separator, set_separator_len), false)) &&
+			WriteHeaderField("empty_field", get_escaped_string(empty_field, false)) &&
-			WriteHeaderField("empty_field", get_escaped_string(
+			WriteHeaderField("unset_field", get_escaped_string(unset_field, false)) &&
-			    string(empty_field, empty_field_len), false)) &&
-			WriteHeaderField("unset_field", get_escaped_string(
-			    string(unset_field, unset_field_len), false)) &&
 			WriteHeaderField("path", get_escaped_string(path, false)) &&
 			WriteHeaderField("open", Timestamp(0))) )
 			goto write_error;
@@ -205,151 +199,6 @@ bool Ascii::DoFinish(double network_time)
 	return true;
 	}
 
-
-bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
-	{
-	if ( ! val->present )
-		{
-		desc->AddN(unset_field, unset_field_len);
-		return true;
-		}
-
-	switch ( val->type ) {
-
-	case TYPE_BOOL:
-		desc->Add(val->val.int_val ? "T" : "F");
-		break;
-
-	case TYPE_INT:
-		desc->Add(val->val.int_val);
-		break;
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		desc->Add(val->val.uint_val);
-		break;
-
-	case TYPE_PORT:
-		desc->Add(val->val.port_val.port);
-		break;
-
-	case TYPE_SUBNET:
-		desc->Add(Render(val->val.subnet_val));
-		break;
-
-	case TYPE_ADDR:
-		desc->Add(Render(val->val.addr_val));
-		break;
-
-	case TYPE_DOUBLE:
-		// Rendering via Add() truncates trailing 0s after the
-		// decimal point. The difference with TIME/INTERVAL is mainly
-		// to keep the log format consistent.
-		desc->Add(val->val.double_val);
-		break;
-
-	case TYPE_INTERVAL:
-	case TYPE_TIME:
-		// Rendering via Render() keeps trailing 0s after the decimal
-		// point. The difference with DOUBLEis mainly to keep the log
-		// format consistent.
-		desc->Add(Render(val->val.double_val));
-		break;
-
-	case TYPE_ENUM:
-	case TYPE_STRING:
-	case TYPE_FILE:
-	case TYPE_FUNC:
-		{
-		int size = val->val.string_val.length;
-		const char* data = val->val.string_val.data;
-
-		if ( ! size )
-			{
-			desc->AddN(empty_field, empty_field_len);
-			break;
-			}
-
-		if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 )
-			{
-			// The value we'd write out would match exactly the
-			// place-holder we use for unset optional fields. We
-			// escape the first character so that the output
-			// won't be ambigious.
-			static const char hex_chars[] = "0123456789abcdef";
-			char hex[6] = "\\x00";
-			hex[2] = hex_chars[((*data) & 0xf0) >> 4];
-			hex[3] = hex_chars[(*data) & 0x0f];
-			desc->AddRaw(hex, 4);
-
-			++data;
-			--size;
-			}
-
-		if ( size )
-			desc->AddN(data, size);
-
-		break;
-		}
-
-	case TYPE_TABLE:
-		{
-		if ( ! val->val.set_val.size )
-			{
-			desc->AddN(empty_field, empty_field_len);
-			break;
-			}
-
-		desc->AddEscapeSequence(set_separator, set_separator_len);
-		for ( int j = 0; j < val->val.set_val.size; j++ )
-			{
-			if ( j > 0 )
-				desc->AddRaw(set_separator, set_separator_len);
-
-			if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) )
-				{
-				desc->RemoveEscapeSequence(set_separator, set_separator_len);
-				return false;
-				}
-			}
-		desc->RemoveEscapeSequence(set_separator, set_separator_len);
-
-		break;
-		}
-
-	case TYPE_VECTOR:
-		{
-		if ( ! val->val.vector_val.size )
-			{
-			desc->AddN(empty_field, empty_field_len);
-			break;
-			}
-
-		desc->AddEscapeSequence(set_separator, set_separator_len);
-		for ( int j = 0; j < val->val.vector_val.size; j++ )
-			{
-			if ( j > 0 )
-				desc->AddRaw(set_separator, set_separator_len);
-
-			if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) )
-				{
-				desc->RemoveEscapeSequence(set_separator, set_separator_len);
-				return false;
-				}
-			}
-		desc->RemoveEscapeSequence(set_separator, set_separator_len);
-
-		break;
-		}
-
-	default:
-		Error(Fmt("unsupported field format %d for %s", val->type, field->name));
-		return false;
-	}
-
-	return true;
-	}
-
 bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
@@ -361,9 +210,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	for ( int i = 0; i < num_fields; i++ )
 		{
 		if ( i > 0 )
-			desc.AddRaw(separator, separator_len);
+			desc.AddRaw(separator);
 
-		if ( ! DoWriteOne(&desc, vals[i], fields[i]) )
+		if ( ! io->ValToODesc(&desc, vals[i], fields[i]) )
 			return false;
 		}
 
@@ -372,7 +221,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	const char* bytes = (const char*)desc.Bytes();
 	int len = desc.Len();
 
-	if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 )
+	if ( strncmp(bytes, meta_prefix.data(), meta_prefix.size()) == 0 )
 		{
 		// It would so escape the first character.
 		char buf[16];

src/logging/writers/Ascii.h

@@ -6,6 +6,7 @@
 #define LOGGING_WRITER_ASCII_H
 
 #include "../WriterBackend.h"
+#include "../../AsciiInputOutput.h"
 
 namespace logging { namespace writer {
 
@@ -32,7 +33,6 @@ protected:
 
 private:
 	bool IsSpecial(string path) 	{ return path.find("/dev/") == 0; }
-	bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
 	bool WriteHeaderField(const string& key, const string& value);
 	void CloseFile(double t);
 	string Timestamp(double t); // Uses current time if t is zero.
@@ -47,20 +47,13 @@ private:
 	bool include_meta;
 	bool only_single_header_row;
 
-	char* separator;
+	string separator;
-	int separator_len;
+	string set_separator;
+	string empty_field;
+	string unset_field;
+	string meta_prefix;
 
-	char* set_separator;
+	AsciiInputOutput* io;
-	int set_separator_len;
-
-	char* empty_field;
-	int empty_field_len;
-
-	char* unset_field;
-	int unset_field_len;
-
-	char* meta_prefix;
-	int meta_prefix_len;
 };
 
 }

src/logging/writers/DataSeries.cc

@@ -46,10 +46,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
 		}
 
 	case TYPE_SUBNET:
-		return Render(val->val.subnet_val);
+		return AsciiInputOutput::Render(val->val.subnet_val);
 
 	case TYPE_ADDR:
-		return Render(val->val.addr_val);
+		return AsciiInputOutput::Render(val->val.addr_val);
 
 	// Note: These two cases are relatively special.  We need to convert
 	// these values into their integer equivalents to maximize precision.
@@ -69,10 +69,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
 			return ostr.str();
 			}
 		else
-			return Render(val->val.double_val);
+			return AsciiInputOutput::Render(val->val.double_val);
 
 	case TYPE_DOUBLE:
-		return Render(val->val.double_val);
+		return AsciiInputOutput::Render(val->val.double_val);
 
 	case TYPE_ENUM:
 	case TYPE_STRING:

src/logging/writers/ElasticSearch.cc

@@ -16,6 +16,7 @@
 #include "BroString.h"
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
+#include "../../AsciiInputOutput.h"
 
 #include <curl/curl.h>
 #include <curl/easy.h>
@@ -124,13 +125,13 @@ bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val)
 
 		case TYPE_SUBNET:
 			b->AddRaw("\"", 1);
-			b->Add(Render(val->val.subnet_val));
+			b->Add(AsciiInputOutput::Render(val->val.subnet_val));
 			b->AddRaw("\"", 1);
 			break;
 
 		case TYPE_ADDR:
 			b->AddRaw("\"", 1);
-			b->Add(Render(val->val.addr_val));
+			b->Add(AsciiInputOutput::Render(val->val.addr_val));
 			b->AddRaw("\"", 1);
 			break;
 

src/threading/MsgThread.h

@@ -400,7 +400,7 @@ private:
 };
 
 /**
- * A paremeterized OututMessage that stores a pointer to an argument object.
+ * A paremeterized OutputMessage that stores a pointer to an argument object.
  * Normally, the objects will be used from the Process() callback.
  */
 template<typename O>