diff --git a/CHANGES b/CHANGES index 4d81261a1f..e8cfb5dcbf 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.1-beta-45 | 2012-08-22 16:11:10 -0700 + + * Add an option to the input framework that allows the user to chose + to not die upon encountering files/functions. (Bernhard Amann) + 2.1-beta-41 | 2012-08-22 16:05:21 -0700 * Add test serialization to "leak" unit tests that use diff --git a/VERSION b/VERSION index ef73421c6e..fa60c5dd44 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-beta-41 +2.1-beta-45 diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 758bc94732..b5f44669c2 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,13 @@ export { ## The default reader mode used. Defaults to `MANUAL`. const default_mode = MANUAL &redef; + ## Flag that controls if the input framework accepts records + ## that contain types that are not supported (at the moment + ## file and function). If true, the input framework will + ## warn in these cases, but continue. If false, it will + ## abort. Defaults to false (abort) + const accept_unsupported_types = F &redef; + ## TableFilter description type used for the `table` method. type TableDescription: record { ## Common definitions for tables and events diff --git a/src/input.bif b/src/input.bif index f494ef3b2f..199b665fa6 100644 --- a/src/input.bif +++ b/src/input.bif @@ -34,6 +34,10 @@ function Input::__force_update%(id: string%) : bool return new Val(res, TYPE_BOOL); %} +# Options for the input framework + +const accept_unsupported_types: bool; + # Options for Ascii Reader module InputAscii; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 3c29f14928..67e4931046 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -388,6 +388,8 @@ bool Manager::CreateEventStream(RecordVal* fval) FuncType* etype = event->FType()->AsFuncType(); + bool allow_file_func = false; + if ( ! etype->IsEvent() ) { reporter->Error("stream event is a function, not an event"); @@ -453,6 +455,8 @@ bool Manager::CreateEventStream(RecordVal* fval) return false; } + allow_file_func = BifConst::Input::accept_unsupported_types; + } else @@ -461,7 +465,7 @@ bool Manager::CreateEventStream(RecordVal* fval) vector fieldsV; // vector, because UnrollRecordType needs it - bool status = !UnrollRecordType(&fieldsV, fields, ""); + bool status = (! UnrollRecordType(&fieldsV, fields, "", allow_file_func)); if ( status ) { @@ -609,12 +613,12 @@ bool Manager::CreateTableStream(RecordVal* fval) vector fieldsV; // vector, because we don't know the length beforehands - bool status = !UnrollRecordType(&fieldsV, idx, ""); + bool status = (! UnrollRecordType(&fieldsV, idx, "", false)); int idxfields = fieldsV.size(); if ( val ) // if we are not a set - status = status || !UnrollRecordType(&fieldsV, val, ""); + status = status || ! UnrollRecordType(&fieldsV, val, "", BifConst::Input::accept_unsupported_types); int valfields = fieldsV.size() - idxfields; @@ -772,15 +776,29 @@ bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) return true; } -bool Manager::UnrollRecordType(vector *fields, - const RecordType *rec, const string& nameprepend) +bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, + const string& nameprepend, bool allow_file_func) { - for ( int i = 0; i < rec->NumFields(); i++ ) { if ( ! IsCompatibleType(rec->FieldType(i)) ) - { + { + // If the field is a file or a function type + // and it is optional, we accept it nevertheless. + // This allows importing logfiles containing this + // stuff that we actually cannot read :) + if ( allow_file_func ) + { + if ( ( rec->FieldType(i)->Tag() == TYPE_FILE || + rec->FieldType(i)->Tag() == TYPE_FUNC ) && + rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) + { + reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag())); + continue; + } + } + reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; } @@ -789,7 +807,7 @@ bool Manager::UnrollRecordType(vector *fields, { string prep = nameprepend + rec->FieldName(i) + "."; - if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) ) { return false; } @@ -1675,6 +1693,18 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals, Val* fieldVal = 0; if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + else if ( request_type->FieldType(i)->Tag() == TYPE_FILE || + request_type->FieldType(i)->Tag() == TYPE_FUNC ) + { + // If those two unsupported types are encountered here, they have + // been let through by the type checking. + // That means that they are optional & the user agreed to ignore + // them and has been warned by reporter. + // Hence -> assign null to the field, done. + + // Better check that it really is optional. Uou never know. + assert(request_type->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)); + } else { fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); diff --git a/src/input/Manager.h b/src/input/Manager.h index 1590042183..cc81df38b7 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -158,7 +158,7 @@ private: // Check if a record is made up of compatible types and return a list // of all fields that are in the record in order. Recursively unrolls // records - bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend, bool allow_file_func); // Send events void SendEvent(EventHandlerPtr ev, const int numvals, ...); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out new file mode 100644 index 0000000000..7ef82cf368 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out @@ -0,0 +1,14 @@ +{ +[-42] = [fi=, b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro new file mode 100644 index 0000000000..7affa4065d --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro @@ -0,0 +1,64 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields fi b i e c p sn a d t iv s sc ss se vc ve f +#types file bool int enum count port subnet addr double time interval string table table table vector vector func +whatever T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load base/protocols/ssh +@load frameworks/communication/listen + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; +redef Input::accept_unsupported_types = T; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + fi: file &optional; + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); + } + +event Input::update_finished(name: string, source:string) + { + print outfile, servers; + close(outfile); + terminate(); + }