make optional fields possible for input framework.

This do not have to be present in the input file and are marked as &optional in the record description.

Those can e.g. be used to create field values on the file in a predicate while reading a file - example:

	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers,
				$pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; }

src/input/Manager.cc

@@ -492,7 +492,11 @@ bool Manager::CreateTableStream(RecordVal* fval) {
 	Unref(pred);
 
 	if ( valfields > 1 ) {
-		assert(filter->want_record);
+		if ( ! filter->want_record ) {
+			reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str());
+			delete filter;
+			return false;
+		}
 	}
 
 
@@ -631,6 +635,10 @@ bool Manager::UnrollRecordType(vector<Field*> *fields, const RecordType *rec, co
 				field->secondary_name = c->AsStringVal()->AsString()->CheckString();
 			}
 
+			if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) {
+				field->optional = true;
+			}
+
 			fields->push_back(field);
 		}
 	}

src/input/readers/Ascii.cc

@@ -26,6 +26,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int
 {
 	position = arg_position;
 	secondary_position = -1;
+	present = true;
 }
 
 FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) 
@@ -33,10 +34,11 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, cons
 {
 	position = arg_position;
 	secondary_position = -1;
+	present = true;
 }
 
 FieldMapping::FieldMapping(const FieldMapping& arg) 
-	: name(arg.name), type(arg.type), subtype(arg.subtype)
+	: name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present)
 {
 	position = arg.position;
 	secondary_position = arg.secondary_position;
@@ -162,7 +164,15 @@ bool Ascii::ReadHeader(bool useCached) {
 		
 		map<string, uint32_t>::iterator fit = ifields.find(field->name);	
 		if ( fit == ifields.end() ) {
-			Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str()));
+			if ( field->optional ) {
+				// we do not really need this field. mark it as not present and always send an undef back.
+				FieldMapping f(field->name, field->type, field->subtype, -1);
+				f.present = false;
+				columnMap.push_back(f);
+				continue;
+			}
+
+			Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str()));
 			return false;
 		}
 
@@ -220,7 +230,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) {
 		} else if ( s == "F" ) {
 			val->val.int_val = 0;
 		} else {
-			Error(Fmt("Invalid value for boolean: %s", s.c_str()));
+			Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str()));
 			return false;
 		}
 		break;
@@ -423,6 +433,14 @@ bool Ascii::DoUpdate() {
 			fit != columnMap.end();
 			fit++ ){
 
+			if ( ! fit->present ) {
+				// add non-present field
+				fields[fpos] =  new Value((*fit).type, false);
+				fpos++;
+				continue;
+			}
+			
+			assert(fit->position >= 0 );
 
 			if ( (*fit).position > pos || (*fit).secondary_position > pos ) {
 				Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos,  (*fit).position, (*fit).secondary_position));

src/input/readers/Ascii.h

@@ -19,6 +19,7 @@ struct FieldMapping {
 	int position;
 	// for ports: pos of the second field
 	int secondary_position;
+	bool present;
 
 	FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); 
 	FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); 

src/threading/SerialTypes.cc

@@ -13,7 +13,7 @@ bool Field::Read(SerializationFormat* fmt)
 	int st;
 
 	bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && 
-			fmt->Read(&t, "type") && fmt->Read(&st, "subtype") );
+			fmt->Read(&t, "type") && fmt->Read(&st, "subtype") && fmt->Read(&optional, "optional"));
 	type = (TypeTag) t;
 	subtype = (TypeTag) st;
 
@@ -23,7 +23,7 @@ bool Field::Read(SerializationFormat* fmt)
 bool Field::Write(SerializationFormat* fmt) const
 	{
 	return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && 
-			fmt->Write((int)subtype, "subtype"));
+			fmt->Write((int)subtype, "subtype"), fmt->Write(optional, "optional"));
 	}
 
 Value::~Value()

src/threading/SerialTypes.h

@@ -24,17 +24,18 @@ struct Field {
 	string secondary_name;	
 	TypeTag type;	//! Type of the field.
 	TypeTag subtype;	//! Inner type for sets.
+	bool optional; //! needed by input framework. Is the field optional or does it have to be present in the input data
 
 	/**
 	 * Constructor.
 	 */
-	Field() 	{ subtype = TYPE_VOID; }
+	Field() 	{ subtype = TYPE_VOID; optional = false; }
 
 	/**
 	 * Copy constructor.
 	 */
 	Field(const Field& other)
-		: name(other.name), type(other.type), subtype(other.subtype) {  }
+		: name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) {  }
 
 	/**
 	 * Unserializes a field.

testing/btest/Baseline/scripts.base.frameworks.input.optional/out

@@ -0,0 +1,9 @@
+{
+[2] = [b=T, notb=F],
+[4] = [b=F, notb=T],
+[6] = [b=F, notb=T],
+[7] = [b=T, notb=F],
+[1] = [b=T, notb=F],
+[5] = [b=F, notb=T],
+[3] = [b=F, notb=T]
+}

testing/btest/scripts/base/frameworks/input/optional.bro

@@ -0,0 +1,45 @@
+#
+# @TEST-EXEC: bro -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	notb: bool &optional;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+{
+	# first read in the old stuff into the table...
+	Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
+				$pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; }
+				]);
+	Input::remove("input");
+}
+
+event Input::update_finished(name: string, source: string) {
+	print servers;
+}