Merge remote-tracking branch 'origin/topic/bernhard/input-error-fixes'

* origin/topic/bernhard/input-error-fixes:
  Several fixes for input manager error handling.

BIT-1106 #merged
This commit is contained in:
Robin Sommer 2013-12-09 15:19:37 -08:00
commit e8739f785b
10 changed files with 325 additions and 79 deletions

View file

@ -1,4 +1,10 @@
2.2-68 | 2013-12-09 15:19:37 -0800
* Several improvements to input framework error handling for more
robustness and more helpful error messages. Includes tests for
many cases. (Bernhard Amann)
2.2-66 | 2013-12-09 13:54:16 -0800
* Fix table &default reference counting for record ctor expressions.

View file

@ -1 +1 @@
2.2-66
2.2-68

View file

@ -393,16 +393,9 @@ bool Manager::CreateEventStream(RecordVal* fval)
return false;
}
EventStream* stream = new EventStream();
{
bool res = CreateStream(stream, fval);
if ( res == false )
{
delete stream;
return false;
}
}
Val* name_val = fval->Lookup("name", true);
string stream_name = name_val->AsString()->CheckString();
Unref(name_val);
RecordType *fields = fval->Lookup("fields", true)->AsType()->AsTypeType()->Type()->AsRecordType();
@ -418,8 +411,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
if ( etype->Flavor() != FUNC_FLAVOR_EVENT )
{
reporter->Error("stream event is a function, not an event");
delete stream;
reporter->Error("Input stream %s: Stream event is a function, not an event", stream_name.c_str());
return false;
}
@ -427,22 +419,19 @@ bool Manager::CreateEventStream(RecordVal* fval)
if ( args->length() < 2 )
{
reporter->Error("event takes not enough arguments");
delete stream;
reporter->Error("Input stream %s: Event does not take enough arguments", stream_name.c_str());
return false;
}
if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) )
{
reporter->Error("events second attribute must be of type Input::Event");
delete stream;
reporter->Error("Input stream %s: Event's second attribute must be of type Input::Event", stream_name.c_str());
return false;
}
if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) )
{
reporter->Error("events first attribute must be of type Input::EventDescription");
delete stream;
reporter->Error("Input stream %s: Event's first attribute must be of type Input::EventDescription", stream_name.c_str());
return false;
}
@ -450,17 +439,24 @@ bool Manager::CreateEventStream(RecordVal* fval)
{
if ( args->length() != fields->NumFields() + 2 )
{
reporter->Error("event has wrong number of arguments");
delete stream;
reporter->Error("Input stream %s: Event has wrong number of arguments", stream_name.c_str());
return false;
}
for ( int i = 0; i < fields->NumFields(); i++ )
{
if ( !same_type((*args)[i+2], fields->FieldType(i) ) )
if ( ! same_type((*args)[i + 2], fields->FieldType(i) ) )
{
reporter->Error("Incompatible type for event");
delete stream;
ODesc desc1;
ODesc desc2;
(*args)[i + 2]->Describe(&desc1);
fields->FieldType(i)->Describe(&desc2);
reporter->Error("Input stream %s: Incompatible type for event in field %d. Need type '%s':%s, got '%s':%s",
stream_name.c_str(), i + 3,
type_name(fields->FieldType(i)->Tag()), desc2.Description(),
type_name((*args)[i + 2]->Tag()), desc1.Description());
return false;
}
}
@ -471,8 +467,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
{
if ( args->length() != 3 )
{
reporter->Error("event has wrong number of arguments");
delete stream;
reporter->Error("Input stream %s: Event has wrong number of arguments", stream_name.c_str());
return false;
}
@ -482,10 +477,10 @@ bool Manager::CreateEventStream(RecordVal* fval)
ODesc desc2;
(*args)[2]->Describe(&desc1);
fields->Describe(&desc2);
reporter->Error("Incompatible type '%s':%s for event, which needs type '%s':%s\n",
reporter->Error("Input stream %s: Incompatible type '%s':%s for event, which needs type '%s':%s\n",
stream_name.c_str(),
type_name((*args)[2]->Tag()), desc1.Description(),
type_name(fields->Tag()), desc2.Description());
delete stream;
return false;
}
@ -496,14 +491,21 @@ bool Manager::CreateEventStream(RecordVal* fval)
else
assert(false);
vector<Field*> fieldsV; // vector, because UnrollRecordType needs it
bool status = (! UnrollRecordType(&fieldsV, fields, "", allow_file_func));
if ( status )
{
reporter->Error("Problem unrolling");
reporter->Error("Input stream %s: Problem unrolling", stream_name.c_str());
return false;
}
EventStream* stream = new EventStream();
bool res = CreateStream(stream, fval);
if ( ! res )
{
delete stream;
return false;
}
@ -540,15 +542,9 @@ bool Manager::CreateTableStream(RecordVal* fval)
return false;
}
TableStream* stream = new TableStream();
{
bool res = CreateStream(stream, fval);
if ( res == false )
{
delete stream;
return false;
}
}
Val* name_val = fval->Lookup("name", true);
string stream_name = name_val->AsString()->CheckString();
Unref(name_val);
Val* pred = fval->Lookup("pred", true);
@ -571,28 +567,54 @@ bool Manager::CreateTableStream(RecordVal* fval)
{
if ( j >= num )
{
reporter->Error("Table type has more indexes than index definition");
delete stream;
reporter->Error("Input stream %s: Table type has more indexes than index definition", stream_name.c_str());
return false;
}
if ( ! same_type(idx->FieldType(j), (*tl)[j]) )
{
reporter->Error("Table type does not match index type");
delete stream;
ODesc desc1;
ODesc desc2;
idx->FieldType(j)->Describe(&desc1);
(*tl)[j]->Describe(&desc2);
reporter->Error("Input stream %s: Table type does not match index type. Need type '%s':%s, got '%s':%s", stream_name.c_str(),
type_name(idx->FieldType(j)->Tag()), desc1.Description(),
type_name((*tl)[j]->Tag()), desc2.Description());
return false;
}
}
if ( num != j )
{
reporter->Error("Table has less elements than index definition");
delete stream;
reporter->Error("Input stream %s: Table has less elements than index definition", stream_name.c_str());
return false;
}
Val *want_record = fval->Lookup("want_record", true);
{
const BroType* table_yield = dst->Type()->AsTableType()->YieldType();
const BroType* compare_type = val;
if ( want_record->InternalInt() == 0 )
compare_type = val->FieldType(0);
if ( ! same_type(table_yield, compare_type) )
{
ODesc desc1;
ODesc desc2;
compare_type->Describe(&desc1);
table_yield->Describe(&desc2);
reporter->Error("Input stream %s: Table type does not match value type. Need type '%s', got '%s'", stream_name.c_str(),
desc1.Description(), desc2.Description());
return false;
}
}
Val* event_val = fval->Lookup("ev", true);
Func* event = event_val ? event_val->AsFunc() : 0;
Unref(event_val);
@ -603,8 +625,7 @@ bool Manager::CreateTableStream(RecordVal* fval)
if ( etype->Flavor() != FUNC_FLAVOR_EVENT )
{
reporter->Error("stream event is a function, not an event");
delete stream;
reporter->Error("Input stream %s: Stream event is a function, not an event", stream_name.c_str());
return false;
}
@ -612,43 +633,52 @@ bool Manager::CreateTableStream(RecordVal* fval)
if ( args->length() != 4 )
{
reporter->Error("Table event must take 4 arguments");
delete stream;
reporter->Error("Input stream %s: Table event must take 4 arguments", stream_name.c_str());
return false;
}
if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) )
{
reporter->Error("table events first attribute must be of type Input::TableDescription");
delete stream;
reporter->Error("Input stream %s: Table event's first attribute must be of type Input::TableDescription", stream_name.c_str());
return false;
}
if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) )
{
reporter->Error("table events second attribute must be of type Input::Event");
delete stream;
reporter->Error("Input stream %s: Table event's second attribute must be of type Input::Event", stream_name.c_str());
return false;
}
if ( ! same_type((*args)[2], idx) )
{
reporter->Error("table events index attributes do not match");
delete stream;
ODesc desc1;
ODesc desc2;
idx->Describe(&desc1);
(*args)[2]->Describe(&desc2);
reporter->Error("Input stream %s: Table event's index attributes do not match. Need '%s', got '%s'", stream_name.c_str(),
desc1.Description(), desc2.Description());
return false;
}
if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) )
{
reporter->Error("table events value attributes do not match");
delete stream;
ODesc desc1;
ODesc desc2;
val->Describe(&desc1);
(*args)[3]->Describe(&desc2);
reporter->Error("Input stream %s: Table event's value attributes do not match. Need '%s', got '%s'", stream_name.c_str(),
desc1.Description(), desc2.Description());
return false;
}
else if ( want_record->InternalInt() == 0
&& !same_type((*args)[3], val->FieldType(0) ) )
{
reporter->Error("table events value attribute does not match");
delete stream;
ODesc desc1;
ODesc desc2;
val->FieldType(0)->Describe(&desc1);
(*args)[3]->Describe(&desc2);
reporter->Error("Input stream %s: Table event's value attribute does not match. Need '%s', got '%s'", stream_name.c_str(),
desc1.Description(), desc2.Description());
return false;
}
@ -667,15 +697,30 @@ bool Manager::CreateTableStream(RecordVal* fval)
int valfields = fieldsV.size() - idxfields;
if ( (valfields > 1) && (want_record->InternalInt() != 1) )
{
reporter->Error("Input stream %s: Stream does not want a record (want_record=F), but has more then one value field.", stream_name.c_str());
return false;
}
if ( ! val )
assert(valfields == 0);
if ( status )
{
reporter->Error("Problem unrolling");
reporter->Error("Input stream %s: Problem unrolling", stream_name.c_str());
return false;
}
TableStream* stream = new TableStream();
{
bool res = CreateStream(stream, fval);
if ( ! res )
{
delete stream;
return false;
}
}
Field** fields = new Field*[fieldsV.size()];
for ( unsigned int i = 0; i < fieldsV.size(); i++ )
@ -697,17 +742,6 @@ bool Manager::CreateTableStream(RecordVal* fval)
Unref(want_record); // ref'd by lookupwithdefault
Unref(pred);
if ( valfields > 1 )
{
if ( ! stream->want_record )
{
reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", stream->name.c_str());
delete stream;
return false;
}
}
assert(stream->reader);
stream->reader->Init(fieldsV.size(), fields );
@ -866,6 +900,7 @@ bool Manager::UnrollRecordType(vector<Field*> *fields, const RecordType *rec,
if ( ! IsCompatibleType(rec->FieldType(i)) )
{
string name = nameprepend + rec->FieldName(i);
// If the field is a file, function, or opaque
// and it is optional, we accept it nevertheless.
// This allows importing logfiles containing this
@ -877,12 +912,12 @@ bool Manager::UnrollRecordType(vector<Field*> *fields, const RecordType *rec,
rec->FieldType(i)->Tag() == TYPE_OPAQUE ) &&
rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) )
{
reporter->Info("Encountered incompatible type \"%s\" in type definition for ReaderFrontend. Ignoring optional field.", type_name(rec->FieldType(i)->Tag()));
reporter->Info("Encountered incompatible type \"%s\" in type definition for field \"%s\" in ReaderFrontend. Ignoring optional field.", type_name(rec->FieldType(i)->Tag()), name.c_str());
continue;
}
}
reporter->Error("Incompatible type \"%s\" in type definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag()));
reporter->Error("Incompatible type \"%s\" in type definition for for field \"%s\" in ReaderFrontend", type_name(rec->FieldType(i)->Tag()), name.c_str());
return false;
}
@ -890,6 +925,12 @@ bool Manager::UnrollRecordType(vector<Field*> *fields, const RecordType *rec,
{
string prep = nameprepend + rec->FieldName(i) + ".";
if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) )
{
reporter->Info("The input framework does not support optional record fields: \"%s\"", rec->FieldName(i));
return false;
}
if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) )
{
return false;

View file

@ -0,0 +1,36 @@
error: Incompatible type "file" in type definition for for field "s" in ReaderFrontend
error: Input stream file: Problem unrolling
The input framework does not support optional record fields: "r"
error: Input stream optionalrecord: Problem unrolling
Encountered incompatible type "file" in type definition for field "s" in ReaderFrontend. Ignoring optional field.
error: Incompatible type "file" in type definition for for field "s" in ReaderFrontend
error: Input stream filetable: Problem unrolling
The input framework does not support optional record fields: "r"
error: Input stream optionalrecordtable: Problem unrolling
Encountered incompatible type "file" in type definition for field "s" in ReaderFrontend. Ignoring optional field.
error: Input stream optionalfiletable: Table type does not match value type. Need type 'record { i:int; s:file of string; }', got 'record { i:int; r:record { i:int; s:file of string; }; }'
error: Input stream optionalfiletable2: Table type does not match index type. Need type 'count':count, got 'string':string
error: Input stream optionalfiletable3: Stream event is a function, not an event
error: Input stream optionalfiletable3: Table event must take 4 arguments
error: Input stream optionalfiletable4: Table event's first attribute must be of type Input::TableDescription
error: Input stream optionalfiletable5: Table event's second attribute must be of type Input::Event
error: Input stream optionalfiletable6: Table event's index attributes do not match. Need 'record { c:count; }', got 'record { i:int; r:record { i:int; s:file of string; }; }'
error: Input stream optionalfiletable7: Table event's value attributes do not match. Need 'record { i:int; s:file of string; }', got 'record { i:int; r:record { i:int; s:file of string; }; }'
error: Input stream optionalfiletable8: Stream does not want a record (want_record=F), but has more then one value field.
error: Input stream optionalfiletable9: Table has less elements than index definition
error: Input stream optionalfiletable10: Table type has more indexes than index definition
error: Input stream optionalfiletable11: Table type does not match value type. Need type 'count', got 'int'
error: Input stream optionalfiletable12: Table type does not match value type. Need type 'count', got 'record { i:int; s:string; a:addr; }'
error: Input stream optionalfiletable14: Table type does not match value type. Need type 'int', got 'record { i:int; s:file of string; }'
error: Input stream optionalfiletable15: Table type does not match value type. Need type 'record { c:count; }', got 'record { i:int; s:string; a:addr; }'
error: Input stream event1: Stream event is a function, not an event
error: Input stream event2: Event does not take enough arguments
error: Input stream event3: Event's first attribute must be of type Input::EventDescription
error: Input stream event4: Event's second attribute must be of type Input::Event
error: Input stream event5: Incompatible type 'record':record { i:int; r:record { i:int; s:file of string; }; } for event, which needs type 'record':record { i:int; s:file of string; }
error: Input stream event6: Event has wrong number of arguments
error: Input stream event7: Incompatible type for event in field 3. Need type 'int':int, got 'record':record { i:int; r:record { i:int; s:file of string; }; }
error: Input stream event8: Incompatible type for event in field 5. Need type 'addr':addr, got 'string':string
error: Input stream event9: Event has wrong number of arguments
received termination signal

View file

@ -0,0 +1,2 @@
optionalfile
[i=-42, s=<uninitialized>]

View file

@ -0,0 +1,161 @@
# Test different kinds of errors of the input framework
#
# @TEST-EXEC: bro -b %INPUT
# @TEST-EXEC: btest-diff .stderr
# @TEST-EXEC: btest-diff out
@TEST-START-FILE input.log
#separator \x09
#path ssh
#fields b i e c p sn a d t iv s sc ss se vc ve ns
#types bool int enum count port subnet addr double time interval string table table table vector vector string
T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242
@TEST-END-FILE
redef Input::accept_unsupported_types = T;
redef exit_only_after_terminate = T;
module Test;
global outfile: file;
type Idx: record {
c: count;
};
type Idx2: record {
c: count;
i: int;
};
type FileVal: record {
i: int;
s: file;
};
type Val: record {
i: int;
s: string;
a: addr;
};
type OptionalRecordVal: record {
i: int;
r: FileVal &optional;
};
type OptionalFileVal: record {
i: int;
s: file &optional;
};
global file_table: table[count] of FileVal = table();
global optional_file_table: table[count] of OptionalFileVal = table();
global record_table: table[count] of OptionalRecordVal = table();
global string_table: table[string] of OptionalRecordVal = table();
global val_table: table[count] of Val = table();
global val_table2: table[count, int] of Val = table();
global val_table3: table[count, int] of int = table();
global val_table4: table[count] of int;
event line_file(description: Input::EventDescription, tpe: Input::Event, r:FileVal)
{
print outfile, description$name;
print outfile, r;
}
event optional_line_file(description: Input::EventDescription, tpe: Input::Event, r:OptionalFileVal)
{
print outfile, description$name;
print outfile, r;
}
event line_record(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal)
{
print outfile, description$name;
print outfile, r;
}
event event1(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal, r2: OptionalRecordVal)
{
}
event event2(description: Input::TableDescription, tpe: string, r: OptionalRecordVal, r2: OptionalRecordVal)
{
}
event event3(description: Input::TableDescription, tpe: Input::Event, r: OptionalRecordVal, r2: OptionalRecordVal)
{
}
event event4(description: Input::TableDescription, tpe: Input::Event, r: Idx, r2: OptionalRecordVal)
{
}
event event5(description: Input::EventDescription, tpe: string, r: OptionalRecordVal, r2: OptionalRecordVal)
{
}
event event6(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal)
{
}
event event7(description: Input::EventDescription, tpe: Input::Event, r: OptionalRecordVal, r2:OptionalRecordVal)
{
}
event event8(description: Input::EventDescription, tpe: Input::Event, i: int, s:string, a:string)
{
}
event event9(description: Input::EventDescription, tpe: Input::Event, i: int, s:string, a:addr, ii: int)
{
}
event event10(description: Input::TableDescription, tpe: Input::Event, i: Idx, c: count)
{
}
event kill_me()
{
terminate();
}
event bro_init()
{
outfile = open("out");
Input::add_event([$source="input.log", $name="file", $fields=FileVal, $ev=line_file, $want_record=T]);
Input::add_event([$source="input.log", $name="optionalrecord", $fields=OptionalRecordVal, $ev=line_record, $want_record=T]);
Input::add_event([$source="input.log", $name="optionalfile", $fields=OptionalFileVal, $ev=optional_line_file, $want_record=T]);
Input::add_table([$source="input.log", $name="filetable", $idx=Idx, $val=FileVal, $destination=file_table]);
Input::add_table([$source="input.log", $name="optionalrecordtable", $idx=Idx, $val=OptionalRecordVal, $destination=record_table]);
Input::add_table([$source="input.log", $name="optionalfiletable", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table]);
Input::add_table([$source="input.log", $name="optionalfiletable", $idx=Idx, $val=OptionalFileVal, $destination=record_table]);
Input::add_table([$source="input.log", $name="optionalfiletable2", $idx=Idx, $val=OptionalFileVal, $destination=string_table]);
Input::add_table([$source="input.log", $name="optionalfiletable3", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=terminate]);
Input::add_table([$source="input.log", $name="optionalfiletable3", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=kill_me]);
Input::add_table([$source="input.log", $name="optionalfiletable4", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event1]);
Input::add_table([$source="input.log", $name="optionalfiletable5", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event2]);
Input::add_table([$source="input.log", $name="optionalfiletable6", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event3]);
Input::add_table([$source="input.log", $name="optionalfiletable7", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event4]);
Input::add_table([$source="input.log", $name="optionalfiletable8", $idx=Idx, $val=Val, $destination=val_table4, $want_record=F]);
Input::add_table([$source="input.log", $name="optionalfiletable9", $idx=Idx2, $val=Val, $destination=val_table, $want_record=F]);
Input::add_table([$source="input.log", $name="optionalfiletable10", $idx=Idx, $val=Val, $destination=val_table2, $want_record=F]);
Input::add_table([$source="input.log", $name="optionalfiletable11", $idx=Idx2, $val=Idx, $destination=val_table3, $want_record=F]);
Input::add_table([$source="input.log", $name="optionalfiletable12", $idx=Idx2, $val=Idx, $destination=val_table2, $want_record=F]);
Input::add_table([$source="input.log", $name="optionalfiletable14", $idx=Idx, $val=OptionalFileVal, $destination=optional_file_table, $ev=event10, $want_record=F]);
Input::add_table([$source="input.log", $name="optionalfiletable15", $idx=Idx2, $val=Idx, $destination=val_table2, $want_record=T]);
Input::add_event([$source="input.log", $name="event1", $fields=OptionalFileVal, $ev=terminate, $want_record=T]);
Input::add_event([$source="input.log", $name="event2", $fields=OptionalFileVal, $ev=kill_me, $want_record=T]);
Input::add_event([$source="input.log", $name="event3", $fields=OptionalFileVal, $ev=event3, $want_record=T]);
Input::add_event([$source="input.log", $name="event4", $fields=OptionalFileVal, $ev=event5, $want_record=T]);
Input::add_event([$source="input.log", $name="event5", $fields=OptionalFileVal, $ev=event6, $want_record=T]);
Input::add_event([$source="input.log", $name="event6", $fields=OptionalFileVal, $ev=event7, $want_record=T]);
Input::add_event([$source="input.log", $name="event7", $fields=OptionalFileVal, $ev=event7, $want_record=F]);
Input::add_event([$source="input.log", $name="event8", $fields=Val, $ev=event8, $want_record=F]);
Input::add_event([$source="input.log", $name="event9", $fields=Val, $ev=event9, $want_record=F]);
schedule 3secs { kill_me() };
}

View file

@ -26,7 +26,7 @@ type Val: record {
b: bool;
};
global servers: table[int] of Val = table();
global servers: table[int] of bool = table();
event bro_init()
{

View file

@ -36,7 +36,7 @@ type Val: record {
b: bool;
};
global servers: table[int] of Val = table();
global servers: table[int] of bool = table();
global ct: int;
event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool)

View file

@ -27,7 +27,7 @@ type Val: record {
b: bool;
};
global destination: table[int] of Val = table();
global destination: table[int] of bool = table();
const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32};

View file

@ -31,7 +31,7 @@ type Val: record {
b: bool;
};
global destination: table[int] of Val = table();
global destination: table[int] of bool = table();
event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool)
{
@ -51,5 +51,5 @@ event bro_init()
{
try = 0;
outfile = open("../out");
Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]);
Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F, $ev=line]);
}