add an option to the input framework that allows the user

to chose to not die upon encountering files/functions.

I am not entirely sure if I like the approach I took for
this, it is a bit... hacky.

scripts/base/frameworks/input/main.bro

@@ -10,6 +10,13 @@ export {
 
 	const default_mode = MANUAL &redef;
 
+	## Flag that controls if the input framework accepts records
+	## that contain types that are not supported (at the moment
+	## file and function). If true, the input framework will
+	## warn in these cases, but continue. If false, it will
+	## abort. Defaults to false (abort)
+	const accept_unsupported_types = F &redef;
+
 	## TableFilter description type used for the `table` method.
 	type TableDescription: record {
 		## Common definitions for tables and events

src/input.bif

@@ -34,6 +34,10 @@ function Input::__force_update%(id: string%) : bool
 	return new Val(res, TYPE_BOOL);
 	%}
 
+# Options for the input framework
+
+const accept_unsupported_types: bool;
+
 # Options for Ascii Reader
 
 module InputAscii;

src/input/Manager.cc

@@ -388,6 +388,8 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	FuncType* etype = event->FType()->AsFuncType();
 
+	bool allow_file_func = false;
+
 	if ( ! etype->IsEvent() )
 		{
 		reporter->Error("stream event is a function, not an event");
@@ -453,6 +455,8 @@ bool Manager::CreateEventStream(RecordVal* fval)
 			return false;
 			}
 
+		allow_file_func = BifConst::Input::accept_unsupported_types;
+
 		}
 
 	else
@@ -461,7 +465,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	vector<Field*> fieldsV; // vector, because UnrollRecordType needs it
 
-	bool status = !UnrollRecordType(&fieldsV, fields, "");
+	bool status = !UnrollRecordType(&fieldsV, fields, "", allow_file_func);
 
 	if ( status )
 		{
@@ -609,12 +613,12 @@ bool Manager::CreateTableStream(RecordVal* fval)
 
 	vector<Field*> fieldsV; // vector, because we don't know the length beforehands
 
-	bool status = !UnrollRecordType(&fieldsV, idx, "");
+	bool status = !UnrollRecordType(&fieldsV, idx, "", false);
 
 	int idxfields = fieldsV.size();
 
 	if ( val ) // if we are not a set
-		status = status || !UnrollRecordType(&fieldsV, val, "");
+		status = status || !UnrollRecordType(&fieldsV, val, "", BifConst::Input::accept_unsupported_types);
 
 	int valfields = fieldsV.size() - idxfields;
 
@@ -773,7 +777,7 @@ bool Manager::RemoveStreamContinuation(ReaderFrontend* reader)
 	}
 
 bool Manager::UnrollRecordType(vector<Field*> *fields,
-		const RecordType *rec, const string& nameprepend)
+		const RecordType *rec, const string& nameprepend, bool allow_file_func)
 	{
 
 	for ( int i = 0; i < rec->NumFields(); i++ )
@@ -781,6 +785,23 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 
 		if ( ! IsCompatibleType(rec->FieldType(i)) )
 	       		{
+
+			// if the field is a file or a function type
+			// and it is optional, we accept it nevertheless.
+			// This allows importing logfiles containing this
+			// stuff that we actually cannot read :)
+			if ( allow_file_func ) 
+				{
+				if ( ( rec->FieldType(i)->Tag() == TYPE_FILE ||
+				       rec->FieldType(i)->Tag() == TYPE_FUNC ) &&
+				     rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)
+				   ) 
+					{
+					reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag()));
+					continue;
+					}
+				}
+
 			reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag()));
 			return false;
 			}
@@ -789,7 +810,7 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 			{
 			string prep = nameprepend + rec->FieldName(i) + ".";
 
-			if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) )
+			if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) )
 				{
 				return false;
 				}
@@ -1675,6 +1696,15 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals,
 		Val* fieldVal = 0;
 		if ( request_type->FieldType(i)->Tag() == TYPE_RECORD )
 			fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position);
+		else if ( request_type->FieldType(i)->Tag() == TYPE_FILE || 
+			  request_type->FieldType(i)->Tag() == TYPE_FUNC )
+			{
+			// If those two unsupported types are encountered here, they have
+			// been let through by the type checking.
+			// That means that they are optional & the user agreed to ignore
+			// them and has been warned by reporter.
+			// Hence -> assign null to the field, done.
+			}
 		else
 			{
 			fieldVal = ValueToVal(vals[*position], request_type->FieldType(i));

src/input/Manager.h

@@ -158,7 +158,7 @@ private:
 	// Check if a record is made up of compatible types and return a list
 	// of all fields that are in the record in order. Recursively unrolls
 	// records
-	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend);
+	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend, bool allow_file_func);
 
 	// Send events
 	void SendEvent(EventHandlerPtr ev, const int numvals, ...);

testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out

@@ -0,0 +1,14 @@
+{
+[-42] = [fi=<uninitialized>, b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}

testing/btest/scripts/base/frameworks/input/unsupported_types.bro

@@ -0,0 +1,64 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	fi	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	file	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+whatever	T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+redef Input::accept_unsupported_types = T;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	fi: file &optional;
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}