and factor stuff out the input framework too.

scripts/base/frameworks/input/main.bro

@@ -11,6 +11,24 @@ export {
 	## The default reader mode used. Defaults to `MANUAL`.
 	const default_mode = MANUAL &redef;
 
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	## Can be overwritten by individual writers.	
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;
+
 	## Flag that controls if the input framework accepts records
 	## that contain types that are not supported (at the moment
 	## file and function). If true, the input framework will

scripts/base/frameworks/input/readers/ascii.bro

@@ -7,15 +7,15 @@ module InputAscii;
 export {
 	## Separator between fields.
 	## Please note that the separator has to be exactly one character long
-	const separator = "\t" &redef;
+	const separator = Input::separator &redef;
 
 	## Separator between set elements.
 	## Please note that the separator has to be exactly one character long
-	const set_separator = "," &redef;
+	const set_separator = Input::set_separator &redef;
 
 	## String to use for empty fields.
-	const empty_field = "(empty)" &redef;
+	const empty_field = Input::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Input::unset_field &redef;
 }

src/AsciiInputOutput.cc

@@ -2,9 +2,16 @@
 
 #include "config.h"
 
+#include <sstream>
+#include <errno.h>
 #include "AsciiInputOutput.h"
 #include "bro_inet_ntop.h"
 
+AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t) 
+	{
+	thread = t;
+	}
+
 AsciiInputOutput::AsciiInputOutput(threading::MsgThread* t, const string & separator, const string & set_separator, 
 				const string & empty_field, const string & unset_field) 
 	{
@@ -165,6 +172,228 @@ bool AsciiInputOutput::ValToODesc(ODesc* desc, threading::Value* val, const thre
 	}
 
 
+threading::Value* AsciiInputOutput::EntryToVal(string s, string name, TypeTag type, TypeTag subtype) const
+	{
+	if ( s.compare(unset_field) == 0 )  // field is not set...
+		return new threading::Value(type, false);
+
+	threading::Value* val = new threading::Value(type, true);
+	char* end = 0;
+	errno = 0;
+
+	switch ( type ) {
+	case TYPE_ENUM:
+	case TYPE_STRING:
+		s = get_unescaped_string(s);
+		val->val.string_val.length = s.size();
+		val->val.string_val.data = copy_string(s.c_str());
+		break;
+
+	case TYPE_BOOL:
+		if ( s == "T" )
+			val->val.int_val = 1;
+		else if ( s == "F" )
+			val->val.int_val = 0;
+		else
+			{
+			thread->Error(thread->Fmt("Field: %s Invalid value for boolean: %s",
+				  name.c_str(), s.c_str()));
+			return 0;
+			}
+		break;
+
+	case TYPE_INT:
+		val->val.int_val = strtoll(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		val->val.double_val = strtod(s.c_str(), &end);
+		if ( CheckNumberError(s, end) )
+			return 0;
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+		break;
+
+	case TYPE_PORT:
+		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			return 0;
+
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		break;
+
+	case TYPE_SUBNET:
+		{
+		s = get_unescaped_string(s);
+		size_t pos = s.find("/");
+		if ( pos == s.npos )
+			{
+			thread->Error(thread->Fmt("Invalid value for subnet: %s", s.c_str()));
+			return 0;
+			}
+
+		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
+
+		if ( CheckNumberError(s, end) )
+			return 0;
+
+		string addr = s.substr(0, pos);
+
+		val->val.subnet_val.prefix = StringToAddr(addr);
+		val->val.subnet_val.length = width;
+		break;
+		}
+
+	case TYPE_ADDR:
+		s = get_unescaped_string(s);
+		val->val.addr_val = StringToAddr(s);
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		// First - common initialization
+		// Then - initialization for table.
+		// Then - initialization for vector.
+		// Then - common stuff
+		{
+		// how many entries do we have...
+		unsigned int length = 1;
+		for ( unsigned int i = 0; i < s.size(); i++ )
+			{
+			if ( s[i] == set_separator[0] )
+				length++;
+			}
+
+		unsigned int pos = 0;
+
+		if ( s.compare(empty_field) == 0 )
+			length = 0;
+
+		threading::Value** lvals = new threading::Value* [length];
+
+		if ( type == TYPE_TABLE )
+			{
+			val->val.set_val.vals = lvals;
+			val->val.set_val.size = length;
+			}
+
+		else if ( type == TYPE_VECTOR )
+			{
+			val->val.vector_val.vals = lvals;
+			val->val.vector_val.size = length;
+			}
+
+		else
+			assert(false);
+
+		if ( length == 0 )
+			break; //empty
+
+		istringstream splitstream(s);
+		while ( splitstream )
+			{
+			string element;
+
+			if ( ! getline(splitstream, element, set_separator[0]) )
+				break;
+
+			if ( pos >= length )
+				{
+				thread->Error(thread->Fmt("Internal error while parsing set. pos %d >= length %d."
+				          " Element: %s", pos, length, element.c_str()));
+				break;
+				}
+
+			threading::Value* newval = EntryToVal(element, name, subtype);
+			if ( newval == 0 )
+				{
+				thread->Error("Error while reading set");
+				return 0;
+				}
+
+			lvals[pos] = newval;
+
+			pos++;
+			}
+
+		// Test if the string ends with a set_separator... or if the
+		// complete string is empty. In either of these cases we have
+		// to push an empty val on top of it.
+		if ( s.empty() || *s.rbegin() == set_separator[0] )
+			{
+			lvals[pos] = EntryToVal("", name, subtype);
+			if ( lvals[pos] == 0 )
+				{
+				thread->Error("Error while trying to add empty set element");
+				return 0;
+				}
+
+			pos++;
+			}
+
+		if ( pos != length )
+			{
+			thread->Error(thread->Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
+			return 0;
+			}
+
+		break;
+		}
+
+	default:
+		thread->Error(thread->Fmt("unsupported field format %d for %s", type,
+		name.c_str()));
+		return 0;
+	}
+
+	return val;
+	}
+
+bool AsciiInputOutput::CheckNumberError(const string& s, const char * end) const
+	{
+	// Do this check first, before executing s.c_str() or similar.
+	// otherwise the value to which *end is pointing at the moment might
+	// be gone ...
+	bool endnotnull =  (*end != '\0');
+
+	if ( s.length() == 0 )
+		{
+		thread->Error("Got empty string for number field");
+		return true;
+		}
+
+	if ( end == s.c_str() ) {
+		thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str()));
+		return true;
+	}
+
+	if ( endnotnull )
+		thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+
+	if ( errno == EINVAL )
+		{
+		thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str()));
+		return true;
+		}
+
+	else if ( errno == ERANGE )
+		{
+		thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str()));
+		return true;
+		}
+
+	return false;
+	}
+
 string AsciiInputOutput::Render(const threading::Value::addr_t& addr) 
 	{
 	if ( addr.family == IPv4 )
@@ -187,6 +416,52 @@ string AsciiInputOutput::Render(const threading::Value::addr_t& addr)
 		}
 	}
 
+TransportProto AsciiInputOutput::StringToProto(const string &proto) const
+	{
+	if ( proto == "unknown" )
+		return TRANSPORT_UNKNOWN;
+	else if ( proto == "tcp" )
+		return TRANSPORT_TCP;
+	else if ( proto == "udp" )
+		return TRANSPORT_UDP;
+	else if ( proto == "icmp" )
+		return TRANSPORT_ICMP;
+
+	thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
+
+	return TRANSPORT_UNKNOWN;
+	}
+
+
+// More or less verbose copy from IPAddr.cc -- which uses reporter.
+threading::Value::addr_t AsciiInputOutput::StringToAddr(const string &s) const
+	{
+		threading::Value::addr_t val;
+
+	if ( s.find(':') == std::string::npos ) // IPv4.
+		{
+		val.family = IPv4;
+
+		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
+			{
+			thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
+			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
+			}
+		}
+
+	else
+		{
+		val.family = IPv6;
+		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
+			{
+			thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
+			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
+			}
+		}
+
+	return val;
+	}
+
 string AsciiInputOutput::Render(const threading::Value::subnet_t& subnet) 
 	{
 	char l[16];

src/AsciiInputOutput.h

@@ -8,6 +8,12 @@
 
 class AsciiInputOutput {
 	public: 
+		// Constructor that leaves separators, etc empty.
+		// Use if you just need functionality like StringToAddr, etc.
+		AsciiInputOutput(threading::MsgThread*);
+
+		// Constructor that defines all separators, etc.
+		// Use if you need either ValToODesc or EntryToVal.
 		AsciiInputOutput(threading::MsgThread*, const string & separator, const string & set_separator, 
 				const string & empty_field, const string & unset_field);
 		~AsciiInputOutput();
@@ -17,6 +23,9 @@ class AsciiInputOutput {
 		// returns false & logs an error with reporter in case an error occurs
 		bool ValToODesc(ODesc* desc, threading::Value* val, const threading::Field* field) const;
 
+		// convert the ascii representation of a field into a Value
+		threading::Value* EntryToVal(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
+
 		/** Helper method to render an IP address as a string.
 		  *
 		  * @param addr The address.
@@ -41,8 +50,24 @@ class AsciiInputOutput {
 		  */
 		static string Render(double d);
 
+		/**
+		 *  Convert a string into a TransportProto. This is just a utility
+		 *  function for Readers.
+		 *
+		 * @param proto the transport protocol
+		 */
+		TransportProto StringToProto(const string &proto) const;
+
+		/**
+		 * Convert a string into a Value::addr_t.  This is just a utility
+		 * function for Readers.
+		 *
+		 * @param addr containing an ipv4 or ipv6 address
+		 */
+		threading::Value::addr_t StringToAddr(const string &addr) const;
 
 	private:
+		bool CheckNumberError(const string& s, const char * end) const;
 
 		string separator;
 		string set_separator;

src/input/ReaderBackend.cc

@@ -281,50 +281,4 @@ bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
 	return DoHeartbeat(network_time, current_time);
 	}
 
-TransportProto ReaderBackend::StringToProto(const string &proto)
-	{
-	if ( proto == "unknown" )
-		return TRANSPORT_UNKNOWN;
-	else if ( proto == "tcp" )
-		return TRANSPORT_TCP;
-	else if ( proto == "udp" )
-		return TRANSPORT_UDP;
-	else if ( proto == "icmp" )
-		return TRANSPORT_ICMP;
-
-	Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
-
-	return TRANSPORT_UNKNOWN;
-	}
-
-
-// More or less verbose copy from IPAddr.cc -- which uses reporter.
-Value::addr_t ReaderBackend::StringToAddr(const string &s)
-	{
-	Value::addr_t val;
-
-	if ( s.find(':') == std::string::npos ) // IPv4.
-		{
-		val.family = IPv4;
-
-		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
-			{
-			Error(Fmt("Bad address: %s", s.c_str()));
-			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
-			}
-		}
-
-	else
-		{
-		val.family = IPv6;
-		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
-			{
-			Error(Fmt("Bad address: %s", s.c_str()));
-			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
-			}
-		}
-
-	return val;
-	}
-
 }

src/input/ReaderBackend.h

@@ -315,21 +315,6 @@ protected:
 	 */
 	void EndCurrentSend();
 
-	/**
-	 *  Convert a string into a TransportProto. This is just a utility
-	 *  function for Readers.
-	 *
-	 * @param proto the transport protocol
-	 */
-	TransportProto StringToProto(const string &proto);
-
-	/**
-	 * Convert a string into a Value::addr_t.  This is just a utility
-	 * function for Readers.
-	 *
-	 * @param addr containing an ipv4 or ipv6 address
-	 */
-	threading::Value::addr_t StringToAddr(const string &addr);
 
 private:
 	// Frontend that instantiated us. This object must not be accessed

src/input/readers/Ascii.cc

@@ -67,11 +67,14 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
 
 	unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(),
 			    BifConst::InputAscii::unset_field->Len());
+
+	io = new AsciiInputOutput(this, separator, set_separator, empty_field, unset_field);	
 }
 
 Ascii::~Ascii()
 	{
 	DoClose();
+	delete io;	
 	}
 
 void Ascii::DoClose()
@@ -210,228 +213,7 @@ bool Ascii::GetLine(string& str)
 	return false;
 	}
 
-bool Ascii::CheckNumberError(const string& s, const char * end)
-	{
-	// Do this check first, before executing s.c_str() or similar.
-	// otherwise the value to which *end is pointing at the moment might
-	// be gone ...
-	bool endnotnull =  (*end != '\0');
 
-	if ( s.length() == 0 )
-		{
-		Error("Got empty string for number field");
-		return true;
-		}
-
-	if ( end == s.c_str() ) {
-		Error(Fmt("String '%s' contained no parseable number", s.c_str()));
-		return true;
-	}
-
-	if ( endnotnull )
-		Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
-
-	if ( errno == EINVAL )
-		{
-		Error(Fmt("String '%s' could not be converted to a number", s.c_str()));
-		return true;
-		}
-
-	else if ( errno == ERANGE )
-		{
-		Error(Fmt("Number '%s' out of supported range.", s.c_str()));
-		return true;
-		}
-
-	return false;
-	}
-
-
-Value* Ascii::EntryToVal(string s, FieldMapping field)
-	{
-	if ( s.compare(unset_field) == 0 )  // field is not set...
-		return new Value(field.type, false);
-
-	Value* val = new Value(field.type, true);
-	char* end = 0;
-	errno = 0;
-
-	switch ( field.type ) {
-	case TYPE_ENUM:
-	case TYPE_STRING:
-		s = get_unescaped_string(s);
-		val->val.string_val.length = s.size();
-		val->val.string_val.data = copy_string(s.c_str());
-		break;
-
-	case TYPE_BOOL:
-		if ( s == "T" )
-			val->val.int_val = 1;
-		else if ( s == "F" )
-			val->val.int_val = 0;
-		else
-			{
-			Error(Fmt("Field: %s Invalid value for boolean: %s",
-				  field.name.c_str(), s.c_str()));
-			return 0;
-			}
-		break;
-
-	case TYPE_INT:
-		val->val.int_val = strtoll(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			return 0;
-		break;
-
-	case TYPE_DOUBLE:
-	case TYPE_TIME:
-	case TYPE_INTERVAL:
-		val->val.double_val = strtod(s.c_str(), &end);
-		if ( CheckNumberError(s, end) )
-			return 0;
-		break;
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		val->val.uint_val = strtoull(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			return 0;
-		break;
-
-	case TYPE_PORT:
-		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			return 0;
-
-		val->val.port_val.proto = TRANSPORT_UNKNOWN;
-		break;
-
-	case TYPE_SUBNET:
-		{
-		s = get_unescaped_string(s);
-		size_t pos = s.find("/");
-		if ( pos == s.npos )
-			{
-			Error(Fmt("Invalid value for subnet: %s", s.c_str()));
-			return 0;
-			}
-
-		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
-
-		if ( CheckNumberError(s, end) )
-			return 0;
-
-		string addr = s.substr(0, pos);
-
-		val->val.subnet_val.prefix = StringToAddr(addr);
-		val->val.subnet_val.length = width;
-		break;
-		}
-
-	case TYPE_ADDR:
-		s = get_unescaped_string(s);
-		val->val.addr_val = StringToAddr(s);
-		break;
-
-	case TYPE_TABLE:
-	case TYPE_VECTOR:
-		// First - common initialization
-		// Then - initialization for table.
-		// Then - initialization for vector.
-		// Then - common stuff
-		{
-		// how many entries do we have...
-		unsigned int length = 1;
-		for ( unsigned int i = 0; i < s.size(); i++ )
-			{
-			if ( s[i] == set_separator[0] )
-				length++;
-			}
-
-		unsigned int pos = 0;
-
-		if ( s.compare(empty_field) == 0 )
-			length = 0;
-
-		Value** lvals = new Value* [length];
-
-		if ( field.type == TYPE_TABLE )
-			{
-			val->val.set_val.vals = lvals;
-			val->val.set_val.size = length;
-			}
-
-		else if ( field.type == TYPE_VECTOR )
-			{
-			val->val.vector_val.vals = lvals;
-			val->val.vector_val.size = length;
-			}
-
-		else
-			assert(false);
-
-		if ( length == 0 )
-			break; //empty
-
-		istringstream splitstream(s);
-		while ( splitstream )
-			{
-			string element;
-
-			if ( ! getline(splitstream, element, set_separator[0]) )
-				break;
-
-			if ( pos >= length )
-				{
-				Error(Fmt("Internal error while parsing set. pos %d >= length %d."
-				          " Element: %s", pos, length, element.c_str()));
-				break;
-				}
-
-			Value* newval = EntryToVal(element, field.subType());
-			if ( newval == 0 )
-				{
-				Error("Error while reading set");
-				return 0;
-				}
-
-			lvals[pos] = newval;
-
-			pos++;
-			}
-
-		// Test if the string ends with a set_separator... or if the
-		// complete string is empty. In either of these cases we have
-		// to push an empty val on top of it.
-		if ( s.empty() || *s.rbegin() == set_separator[0] )
-			{
-			lvals[pos] = EntryToVal("", field.subType());
-			if ( lvals[pos] == 0 )
-				{
-				Error("Error while trying to add empty set element");
-				return 0;
-				}
-
-			pos++;
-			}
-
-		if ( pos != length )
-			{
-			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
-			return 0;
-			}
-
-		break;
-		}
-
-	default:
-		Error(Fmt("unsupported field format %d for %s", field.type,
-		field.name.c_str()));
-		return 0;
-	}
-
-	return val;
-	}
 
 // read the entire file and send appropriate thingies back to InputMgr
 bool Ascii::DoUpdate()
@@ -543,7 +325,7 @@ bool Ascii::DoUpdate()
 				return false;
 				}
 
-			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
+			Value* val = io->EntryToVal(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype);
 			if ( val == 0 )
 				{
 				Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str()));
@@ -557,7 +339,7 @@ bool Ascii::DoUpdate()
 				assert(val->type == TYPE_PORT );
 				//	Error(Fmt("Got type %d != PORT with secondary position!", val->type));
 
-				val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]);
+				val->val.port_val.proto = io->StringToProto(stringfields[(*fit).secondary_position]);
 				}
 
 			fields[fpos] = val;

src/input/readers/Ascii.h

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "../ReaderBackend.h"
+#include "../../AsciiInputOutput.h"
 
 namespace input { namespace reader {
 
@@ -47,8 +48,6 @@ private:
 
 	bool ReadHeader(bool useCached);
 	bool GetLine(string& str);
-	threading::Value* EntryToVal(string s, FieldMapping type);
-	bool CheckNumberError(const string& s, const char * end);
 
 	ifstream* file;
 	time_t mtime;
@@ -64,6 +63,8 @@ private:
 	string set_separator;
 	string empty_field;
 	string unset_field;
+
+	AsciiInputOutput* io;
 };
 
 

src/input/readers/Benchmark.cc

@@ -25,11 +25,15 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend)
 	stopspreadat = int(BifConst::InputBenchmark::stopspreadat);
 	timedspread = double(BifConst::InputBenchmark::timedspread);
 	heartbeat_interval = double(BifConst::Threading::heartbeat_interval);
+
+	io = new AsciiInputOutput(this);
 	}
 
 Benchmark::~Benchmark()
 	{
 	DoClose();
+
+	delete io;
 	}
 
 void Benchmark::DoClose()
@@ -162,13 +166,13 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 
 	case TYPE_SUBNET:
 		{
-		val->val.subnet_val.prefix = StringToAddr("192.168.17.1");
+		val->val.subnet_val.prefix = io->StringToAddr("192.168.17.1");
 		val->val.subnet_val.length = 16;
 		}
 		break;
 
 	case TYPE_ADDR:
-		val->val.addr_val = StringToAddr("192.168.17.1");
+		val->val.addr_val = io->StringToAddr("192.168.17.1");
 		break;
 
 	case TYPE_TABLE:

src/input/readers/Benchmark.h

@@ -4,6 +4,7 @@
 #define INPUT_READERS_BENCHMARK_H
 
 #include "../ReaderBackend.h"
+#include "../../AsciiInputOutput.h"
 
 namespace input { namespace reader {
 
@@ -38,6 +39,8 @@ private:
 	double heartbeatstarttime;
 	double timedspread;
 	double heartbeat_interval;
+
+	AsciiInputOutput* io;
 };