diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d0386fbb3f..f6ba6f9f49 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -492,7 +492,11 @@ bool Manager::CreateTableStream(RecordVal* fval) { Unref(pred); if ( valfields > 1 ) { - assert(filter->want_record); + if ( ! filter->want_record ) { + reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); + delete filter; + return false; + } } @@ -631,6 +635,10 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) { + field->optional = true; + } + fields->push_back(field); } } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a04a40e780..20ae79ab19 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -26,6 +26,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) @@ -33,10 +34,11 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, cons { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), subtype(arg.subtype) + : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) { position = arg.position; secondary_position = arg.secondary_position; @@ -162,7 +164,15 @@ bool Ascii::ReadHeader(bool useCached) { map::iterator fit = ifields.find(field->name); if ( fit == ifields.end() ) { - Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + if ( field->optional ) { + // we do not really need this field. mark it as not present and always send an undef back. + FieldMapping f(field->name, field->type, field->subtype, -1); + f.present = false; + columnMap.push_back(f); + continue; + } + + Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); return false; } @@ -220,7 +230,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { } else if ( s == "F" ) { val->val.int_val = 0; } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); + Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); return false; } break; @@ -423,6 +433,14 @@ bool Ascii::DoUpdate() { fit != columnMap.end(); fit++ ){ + if ( ! fit->present ) { + // add non-present field + fields[fpos] = new Value((*fit).type, false); + fpos++; + continue; + } + + assert(fit->position >= 0 ); if ( (*fit).position > pos || (*fit).secondary_position > pos ) { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 40f92be717..a9b14768fb 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -19,6 +19,7 @@ struct FieldMapping { int position; // for ports: pos of the second field int secondary_position; + bool present; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 78556e5271..1d7255d695 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -13,7 +13,7 @@ bool Field::Read(SerializationFormat* fmt) int st; bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && - fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + fmt->Read(&t, "type") && fmt->Read(&st, "subtype") && fmt->Read(&optional, "optional")); type = (TypeTag) t; subtype = (TypeTag) st; @@ -23,7 +23,7 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && - fmt->Write((int)subtype, "subtype")); + fmt->Write((int)subtype, "subtype"), fmt->Write(optional, "optional")); } Value::~Value() diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index ac34f3e476..bee84f2b54 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -24,17 +24,18 @@ struct Field { string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. + bool optional; //! needed by input framework. Is the field optional or does it have to be present in the input data /** * Constructor. */ - Field() { subtype = TYPE_VOID; } + Field() { subtype = TYPE_VOID; optional = false; } /** * Copy constructor. */ Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { } /** * Unserializes a field. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.optional/out b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out new file mode 100644 index 0000000000..7a304fc918 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out @@ -0,0 +1,9 @@ +{ +[2] = [b=T, notb=F], +[4] = [b=F, notb=T], +[6] = [b=F, notb=T], +[7] = [b=T, notb=F], +[1] = [b=T, notb=F], +[5] = [b=F, notb=T], +[3] = [b=F, notb=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro new file mode 100644 index 0000000000..c354f7c3ab --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -0,0 +1,45 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + notb: bool &optional; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +}