From 4a3c9923253c8c26973a24c537a3ca231961db5e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 10:57:45 -0800 Subject: [PATCH] InputReader can read Sets. --- src/InputMgr.cc | 244 +++++++++++++++++++++++++--------------- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 211 ++++++++++++++++++++++------------ src/InputReaderAscii.h | 5 + src/LogMgr.cc | 6 +- src/LogMgr.h | 4 +- 6 files changed, 310 insertions(+), 162 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5f99583c10..ba17d8448f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -234,11 +234,14 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_TABLE: - return false; + { + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } case TYPE_VECTOR: { - return IsCompatibleType(t->AsVectorType()->YieldType()); + return false; // do me... + //return IsCompatibleType(t->AsVectorType()->YieldType()); } default: @@ -329,6 +332,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec LogField* field = new LogField(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); + if ( field->type == TYPE_TABLE ) { + field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } fields->push_back(field); } @@ -810,49 +816,133 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } + +int InputMgr::GetLogValLength(const LogVal* val) { + int length = 0; + + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + + case TYPE_TABLE: { + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += GetLogValLength(val->val.set_val.vals[i]); + } + break; + } + + default: + reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + } + + return length; + +} + +int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); + memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+startpos) = val->val.int_val; + return sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + //*(data+startpos) = val->val.uint_val; + memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); + return sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + //*(data+startpos) = val->val.double_val; + memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + return sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); + return val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+startpos, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + return NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: { + int length = 0; + memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + length += sizeof(val->val.subnet_val.width); + memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + length += sizeof(val->val.subnet_val.net); + return length; + break; + } + + case TYPE_TABLE: { + int length = 0; + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + } + return length; + break; + } + + default: + reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + return 0; + } + + reporter->InternalError("internal error"); + return 0; + +} + HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch (val->type) { - case TYPE_BOOL: - case TYPE_INT: - length += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - length += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - length += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - length += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - length += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - length += sizeof(val->val.subnet_val.width); - length += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals"); - } - + length += GetLogValLength(val); } //reporter->Error("Length: %d", length); @@ -864,56 +954,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch ( val->type ) { - case TYPE_BOOL: - case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); - memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+position) = val->val.int_val; - position += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - //*(data+position) = val->val.uint_val; - memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); - position += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - //*(data+position) = val->val.double_val; - memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); - position += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); - position += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); - position += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); - position += sizeof(val->val.subnet_val.width); - memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); - position += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals2"); - } - - + position += CopyLogVal(data, position, val); } assert(position == length); @@ -966,6 +1007,29 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_TABLE: { + if ( val->val.set_val.size == 0 ) { + // empty table + TypeList* set_index = new TypeList(base_type(TYPE_ANY)); + // iim quite sure this does not work... we probably need the internal set type for this... + reporter->InternalError("Implement me."); + return new TableVal(new SetType(set_index, 0)); + } else { + // all entries have to have the same type... + TypeTag type = val->val.set_val.vals[0]->type; + TypeList* set_index = new TypeList(base_type(type)); + set_index->Append(base_type(type)); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + assert( val->val.set_val.vals[i]->type == type); + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; + } + break; + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 74914e65ad..d4bfa5c355 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -52,6 +52,8 @@ private: void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + int GetLogValLength(const LogVal* val); + int CopyLogVal(char *data, const int startpos, const LogVal* val); Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index f91c55b666..60a8c5685a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,12 +11,22 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) + : name(arg_name), type(arg_type), set_type(arg_set_type) +{ + position = arg_position; +} + FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type) + : name(arg.name), type(arg.type), set_type(arg.set_type) { position = arg.position; } +FieldMapping FieldMapping::setType() { + return FieldMapping(name, set_type, position); +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -81,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, i); + FieldMapping f(field->name, field->type, field->set_type, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -126,6 +136,132 @@ bool InputReaderAscii::GetLine(string& str) { return false; } + +LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { + + LogVal* val = new LogVal(field.type, true); + //bzero(val, sizeof(LogVal)); + + switch ( field.type ) { + case TYPE_ENUM: + case TYPE_STRING: + val->val.string_val = new string(s); + break; + + case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else if ( s == "F" ) { + val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; + } + break; + + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + val->val.uint_val = atoi(s.c_str()); + break; + + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos+1); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + + } + case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + case TYPE_TABLE: { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } + + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + if ( s[i] == ',') length++; + + unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + + istringstream splitstream(s); + while ( splitstream ) { + string element; + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); + break; + } + + if ( !getline(splitstream, element, ',') ) + break; + + + LogVal* newval = EntryToVal(element, field.setType()); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + + pos++; + + } + + if ( pos != length ) { + Error("Internal error while parsing set: did not find all elements"); + return 0; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d for %s", field.type, + field.name.c_str())); + return 0; + } + + return val; + +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -161,7 +297,6 @@ bool InputReaderAscii::DoUpdate() { // split on tabs istringstream splitstream(line); - string s; LogVal** fields = new LogVal*[num_fields]; //string string_fields[num_fields]; @@ -170,6 +305,7 @@ bool InputReaderAscii::DoUpdate() { unsigned int currField = 0; while ( splitstream ) { + string s; if ( !getline(splitstream, s, '\t') ) break; @@ -193,73 +329,10 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = new LogVal(currMapping.type, true); - //bzero(val, sizeof(LogVal)); - - switch ( currMapping.type ) { - case TYPE_ENUM: - case TYPE_STRING: - val->val.string_val = new string(s); - break; - - case TYPE_BOOL: - if ( s == "T" ) { - val->val.int_val = 1; - } else if ( s == "F" ) { - val->val.int_val = 0; - } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); - return false; - } - break; - - case TYPE_INT: - val->val.int_val = atoi(s.c_str()); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - val->val.uint_val = atoi(s.c_str()); - break; - - case TYPE_SUBNET: { - int pos = s.find("/"); - string width = s.substr(pos+1); - val->val.subnet_val.width = atoi(width.c_str()); - string addr = s.substr(0, pos); - s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - val->val.subnet_val.net = dotted_to_addr(s.c_str()); - break; - - } - case TYPE_ADDR: { - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); -#ifdef BROv6 - copy_addr(t, val->val.addr_val); -#else - copy_addr(&t, val->val.addr_val); -#endif - break; - } - - - default: - Error(Fmt("unsupported field format %d for %s", currMapping.type, - currMapping.name.c_str())); + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { return false; - } - + } fields[currMapping.position] = val; //string_fields[currMapping.position] = s; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index f86cfd0062..ab2b89339c 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,11 +12,15 @@ struct FieldMapping { string name; TypeTag type; + TypeTag set_type; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } + + FieldMapping setType(); bool IsEmpty() { return position == -1; } }; @@ -38,6 +42,7 @@ protected: private: bool ReadHeader(); + LogVal* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f78a2a19e0..9818d9cdfb 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); type = (TypeTag) t; + set_type = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..40dab8677b 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // needed by input framework. otherwise it cannot determine the inner type of a set. + TypeTag set_type; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), set_type(other.set_type) { } // (Un-)serialize. bool Read(SerializationFormat* fmt);