diff --git a/doc/input.rst b/doc/input.rst index 78e96fe06e..e201af9fed 100644 --- a/doc/input.rst +++ b/doc/input.rst @@ -34,9 +34,11 @@ very similar to the abstracts used in the logging framework: Readers A reader defines the input format for the specific input stream. - At the moment, Bro comes with only one type of reader, which can - read the tab seperated ASCII logfiles that were generated by the + At the moment, Bro comes with two types of reader. The default reader is READER_ASCII, + which can read the tab seperated ASCII logfiles that were generated by the logging framework. + READER_RAW can files containing records separated by a character(like e.g. newline) and send + one event per line. Basics @@ -68,7 +70,21 @@ The fields that can be set when creating a stream are: ``reader`` The reader used for this stream. Default is ``READER_ASCII``. - + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. + + ``autostart`` + If set to yes, the first update operation is triggered automatically after the first filter has been added to the stream. + This has to be set to false if several filters are added to the input source. + In this case Input::force_update has to be called manually once after all filters have been added. Filters ======= @@ -101,9 +117,6 @@ could be defined as follows: ... Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); - - # read the file after all filters have been set - Input::force_update(Foo::INPUT); } The fields that can be set for an event filter are: @@ -156,7 +169,7 @@ an approach similar to this: Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); - # read the file after all filters have been set + # read the file after all filters have been set (only needed if autostart is set to false) Input::force_update(Foo::INPUT); } diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 445f947106..e06dfae005 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,15 +5,15 @@ module Input; export { - redef enum Input::ID += { TABLE_READ }; - ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; const default_mode = MANUAL &redef; - ## Stream decription type used for the `create_stream` method - type StreamDescription: record { + ## TableFilter description type used for the `table` method. + type TableDescription: record { + ## Common definitions for tables and events + ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; @@ -26,13 +26,12 @@ export { ## Automatically start the input stream after the first filter has been added autostart: bool &default=T; - }; - ## TableFilter description type used for the `add_tablefilter` method. - type TableFilter: record { ## Descriptive name. Used to remove a filter at a later time name: string; + ## Special definitions for tables + ## Table which will contain the data read by the input framework destination: any; ## Record that defines the values used as the index of the table @@ -55,11 +54,28 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; - ## EventFilter description type used for the `add_eventfilter` method. - type EventFilter: record { - ## Descriptive name. Used to remove a filter at a later time - name: string; + ## EventFilter description type used for the `event` method. + type EventDescription: record { + ## Common definitions for tables and events + + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. + source: string; + + ## Reader to use for this steam + reader: Reader &default=default_reader; + ## Read mode to use for this stream + mode: Mode &default=default_mode; + + ## Automatically start the input stream after the first filter has been added + autostart: bool &default=T; + + ## Descriptive name. Used to remove a filter at a later time + name: string; + + ## Special definitions for events + ## Record describing the fields to be retrieved from the source input. fields: any; ## If want_record if false (default), the event receives each value in fields as a seperate argument. @@ -72,59 +88,29 @@ export { }; - #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - - ## Create a new input stream from a given source. Returns true on success. + ## Create a new table input from a given source. Returns true on success. ## - ## id: `Input::ID` enum value identifying this stream - ## description: `StreamDescription` record describing the source. - global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; - - ## Remove a current input stream. Returns true on success. + ## description: `TableDescription` record describing the source. + global add_table: function(description: Input::TableDescription) : bool; + + ## Create a new event input from a given source. Returns true on success. ## - ## id: `Input::ID` enum value identifying the stream to be removed - global remove_stream: function(id: Input::ID) : bool; + ## description: `TableDescription` record describing the source. + global add_event: function(description: Input::EventDescription) : bool; + + ## Remove a input stream. Returns true on success and false if the named stream was not found. + ## + ## id: string value identifying the stream to be removed + global remove: function(id: string) : bool; ## Forces the current input to be checked for changes. + ## Returns true on success and false if the named stream was not found ## - ## id: `Input::ID` enum value identifying the stream - global force_update: function(id: Input::ID) : bool; - - ## Adds a table filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## filter: the `TableFilter` record describing the filter. - global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; - - ## Removes a named table filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## name: the name of the filter to be removed. - global remove_tablefilter: function(id: Input::ID, name: string) : bool; - - ## Adds an event filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## filter: the `EventFilter` record describing the filter. - global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; - - ## Removes a named event filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## name: the name of the filter to be removed. - global remove_eventfilter: function(id: Input::ID, name: string) : bool; - #global get_filter: function(id: ID, name: string) : Filter; - - ## Convenience function for reading a specific input source exactly once using - ## exactly one tablefilter - ## - ## id: `Input::ID` enum value identifying the stream - ## description: `StreamDescription` record describing the source. - ## filter: the `TableFilter` record describing the filter. - global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; - - global update_finished: event(id: Input::ID); + ## id: string value identifying the stream + global force_update: function(id: string) : bool; + ## Event that is called, when the update of a specific source is finished + global update_finished: event(name: string, source:string); } @load base/input.bif @@ -132,72 +118,26 @@ export { module Input; -#global filters: table[ID, string] of Filter; +#global streams: table[string] of Filter; +# ^ change to set containing the names -function create_stream(id: Input::ID, description: Input::StreamDescription) : bool +function add_table(description: Input::TableDescription) : bool { - return __create_stream(id, description); + return __create_table_stream(description); } -function remove_stream(id: Input::ID) : bool +function add_event(description: Input::EventDescription) : bool + { + return __create_event_stream(description); + } + +function remove(id: string) : bool { return __remove_stream(id); } -function force_update(id: Input::ID) : bool +function force_update(id: string) : bool { return __force_update(id); } -function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool - { -# filters[id, filter$name] = filter; - return __add_tablefilter(id, filter); - } - -function remove_tablefilter(id: Input::ID, name: string) : bool - { -# delete filters[id, name]; - return __remove_tablefilter(id, name); - } - -function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool - { -# filters[id, filter$name] = filter; - return __add_eventfilter(id, filter); - } - -function remove_eventfilter(id: Input::ID, name: string) : bool - { -# delete filters[id, name]; - return __remove_eventfilter(id, name); - } - -function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { - local ok: bool = T; - # since we create and delete it ourselves this should be ok... at least for singlethreaded operation - local id: Input::ID = Input::TABLE_READ; - - ok = create_stream(id, description); - if ( ok ) { - ok = add_tablefilter(id, filter); - } - if ( ok ) { - ok = force_update(id); - } - if ( ok ) { - ok = remove_stream(id); - } else { - remove_stream(id); - } - - return ok; -} - -#function get_filter(id: ID, name: string) : Filter -# { -# if ( [id, name] in filters ) -# return filters[id, name]; -# -# return no_filter; -# } diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index c41a0552c6..3394486ff2 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false }, { "threading", 0, false } + { "logging", 0, false }, {"input", 0, false }, + { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 71e21bfa26..ca422072c5 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_INPUT, // Input streams DBG_THREADING, // Threading system NUM_DBGS // Has to be last diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 4b8f527f2b..f29e907790 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -234,7 +234,7 @@ static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; // Buffer size for remote-log data. -static const int LOG_BUFFER_SIZE = 50 * 1024; +static const int LOG_BUFFER_SIZE = 512; struct ping_args { uint32 seq; @@ -532,6 +532,7 @@ RemoteSerializer::RemoteSerializer() terminating = false; in_sync = 0; last_flush = 0; + received_logs = 0; } RemoteSerializer::~RemoteSerializer() @@ -1353,6 +1354,14 @@ double RemoteSerializer::NextTimestamp(double* local_network_time) { Poll(false); + if ( received_logs > 0 ) + { + // If we processed logs last time, assume there's more. + idle = false; + received_logs = 0; + return timer_mgr->Time(); + } + double et = events.length() ? events[0]->time : -1; double pt = packets.length() ? packets[0]->time : -1; @@ -2552,7 +2561,9 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( ! peer->logs_requested ) return false; - assert(peer->log_buffer); + if ( ! peer->log_buffer ) + // Peer shutting down. + return false; // Serialize the log record entry. @@ -2587,7 +2598,10 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) || (network_time - last_flush > 1.0) ) { if ( ! FlushLogBuffer(peer) ) + { + delete [] data; return false; + } } // If the data is actually larger than our complete buffer, just send it out. @@ -2631,6 +2645,12 @@ bool RemoteSerializer::ProcessLogCreateWriter() if ( current_peer->state == Peer::CLOSING ) return false; +#ifdef USE_PERFTOOLS + // Don't track allocations here, they'll be released only after the + // main loop exists. And it's just a tiny amount anyway. + HeapLeakChecker::Disabler disabler; +#endif + assert(current_args); EnumVal* id_val = 0; @@ -2666,7 +2686,7 @@ bool RemoteSerializer::ProcessLogCreateWriter() id_val = new EnumVal(id, BifType::Enum::Log::ID); writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); - if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) ) + if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields, true, false) ) goto error; Unref(id_val); @@ -2735,6 +2755,8 @@ bool RemoteSerializer::ProcessLogWrite() fmt.EndRead(); + ++received_logs; + return true; error: @@ -3376,6 +3398,9 @@ void SocketComm::Run() small_timeout.tv_usec = io->CanWrite() || io->CanRead() ? 1 : 10; + if ( ! io->CanWrite() ) + usleep(10); + int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, &small_timeout); diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index eabcb18a38..05d25ca525 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -338,6 +338,7 @@ private: int propagate_accesses; bool ignore_accesses; bool terminating; + int received_logs; Peer* source_peer; PeerID id_counter; // Keeps track of assigned IDs. uint32 current_sync_point; diff --git a/src/input.bif b/src/input.bif index 5418b7bbd4..1157b7b62b 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,52 +7,33 @@ module Input; #include "NetVar.h" %%} -type StreamDescription: record; -type TableFilter: record; -type EventFilter: record; +type TableDescription: record; +type EventDescription: record; -function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool +function Input::__create_table_stream%(description: Input::TableDescription%) : bool %{ - input::ReaderFrontend *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); - return new Val( the_reader != 0, TYPE_BOOL ); - %} - -function Input::__remove_stream%(id: Input::ID%) : bool - %{ - bool res = input_mgr->RemoveStream(id->AsEnumVal()); + bool res = input_mgr->CreateTableStream(description->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Input::ID%) : bool +function Input::__create_event_stream%(description: Input::EventDescription%) : bool %{ - bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + bool res = input_mgr->CreateEventStream(description->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool +function Input::__remove_stream%(id: string%) : bool %{ - bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->RemoveStream(id->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool +function Input::__force_update%(id: string%) : bool %{ - bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL); - %} - -function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool - %{ - bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->ForceUpdate(id->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL); - %} - # Options for Ascii Reader module InputAscii; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a647b3c945..32b60f05f2 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -31,14 +31,38 @@ declare(PDict, InputHash); class Manager::Filter { public: - EnumVal* id; string name; + string source; + + int mode; FilterType filter_type; // to distinguish between event and table filters + EnumVal* type; + ReaderFrontend* reader; + + RecordVal* description; + + Filter(); virtual ~Filter(); }; +Manager::Filter::Filter() { + type = 0; + reader = 0; + description = 0; +} + +Manager::Filter::~Filter() { + if ( type ) + Unref(type); + if ( description ) + Unref(description); + + if ( reader ) + delete(reader); +} + class Manager::TableFilter: public Manager::Filter { public: @@ -71,70 +95,48 @@ public: bool want_record; EventFilter(); + ~EventFilter(); }; -Manager::TableFilter::TableFilter() { +Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { filter_type = TABLE_FILTER; tab = 0; itype = 0; rtype = 0; + + currDict = 0; + lastDict = 0; + + pred = 0; } -Manager::EventFilter::EventFilter() { +Manager::EventFilter::EventFilter() : Manager::Filter::Filter() { + fields = 0; filter_type = EVENT_FILTER; } -Manager::Filter::~Filter() { - Unref(id); +Manager::EventFilter::~EventFilter() { + if ( fields ) { + Unref(fields); + } } Manager::TableFilter::~TableFilter() { - Unref(tab); - Unref(itype); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); if ( rtype ) // can be 0 for sets Unref(rtype); - delete currDict; - delete lastDict; + if ( currDict != 0 ) + delete currDict; + + if ( lastDict != 0 ) + delete lastDict; } -struct Manager::ReaderInfo { - EnumVal* id; - EnumVal* type; - ReaderFrontend* reader; - - //list events; // events we fire when "something" happens - map filters; // filters that can prevent our actions - - bool HasFilter(int id); - - ~ReaderInfo(); - }; - -Manager::ReaderInfo::~ReaderInfo() { - map::iterator it = filters.begin(); - - while ( it != filters.end() ) { - delete (*it).second; - ++it; - } - - Unref(type); - Unref(id); - - delete(reader); -} - -bool Manager::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} - - struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -160,7 +162,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - reporter->Error("unknown reader when creating reader"); + reporter->Error("The reader that was requested was not found and could not be initialized."); return 0; } @@ -181,7 +183,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } else { // ohok. init failed, kill factory for all eternity ir->factory = 0; - DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); return 0; } @@ -204,64 +206,86 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } // create a new input reader object to be used at whomevers leisure lateron. -ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) +bool Manager::CreateStream(Filter* info, RecordVal* description) { ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) { - reporter->Error("Streamdescription argument not of right type"); - return 0; + reporter->Error("Streamdescription argument not of right type for new input stream"); + return false; + } + + Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); + string name = name_val->AsString()->CheckString(); + Unref(name_val); + + { + Filter *i = FindFilter(name); + if ( i != 0 ) { + reporter->Error("Trying create already existing input stream %s", name.c_str()); + return false; + } } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); - bool do_autostart = ( autostart->InternalInt() == 1 ); - Unref(autostart); // Ref'd by LookupWithDefault - - ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); - assert(reader_obj); + + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); + assert(reader_obj); // get the source... - const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); + assert ( sourceval != 0 ); + const BroString* bsource = sourceval->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); + Unref(sourceval); + + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); + info->mode = mode->InternalInt(); + Unref(mode); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->id = id->Ref()->AsEnumVal(); + info->name = name; + info->source = source; + Ref(description); + info->description = description; - readers.push_back(info); - - reader_obj->Init(source, mode->InternalInt(), do_autostart); + DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", + name.c_str()); - return reader_obj; + return true; } -bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Stream not found"); - return false; - } +bool Manager::CreateEventStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) { reporter->Error("filter argument not of right type"); return false; } + + EventFilter* filter = new EventFilter(); + { + bool res = CreateStream(filter, fval); + if ( res == false ) { + delete filter; + return false; + } + } - Val* name = fval->Lookup(rtype->FieldOffset("name")); - RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); - Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val->AsFunc(); + Unref(event_val); { FuncType* etype = event->FType()->AsFuncType(); @@ -278,32 +302,38 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { return false; } - if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) { - reporter->Error("events first attribute must be of type Input::Event"); + reporter->Error("events second attribute must be of type Input::Event"); return false; } + + if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) + { + reporter->Error("events first attribute must be of type Input::EventDescription"); + return false; + } if ( want_record->InternalInt() == 0 ) { - if ( args->length() != fields->NumFields() + 1 ) { - reporter->Error("events has wrong number of arguments"); + if ( args->length() != fields->NumFields() + 2 ) { + reporter->Error("event has wrong number of arguments"); return false; } for ( int i = 0; i < fields->NumFields(); i++ ) { - if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { reporter->Error("Incompatible type for event"); return false; } } } else if ( want_record->InternalInt() == 1 ) { - if ( args->length() != 2 ) { - reporter->Error("events has wrong number of arguments"); + if ( args->length() != 3 ) { + reporter->Error("event has wrong number of arguments"); return false; } - if ( !same_type((*args)[1], fields ) ) { + if ( !same_type((*args)[2], fields ) ) { reporter->Error("Incompatible type for event"); return false; } @@ -324,56 +354,80 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { return false; } - Field** logf = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { logf[i] = fieldsV[i]; } - EventFilter* filter = new EventFilter(); - filter->name = name->AsString()->CheckString(); - filter->id = id->Ref()->AsEnumVal(); + Unref(fields); // ref'd by lookupwithdefault filter->num_fields = fieldsV.size(); filter->fields = fields->Ref()->AsRecordType(); filter->event = event_registry->Lookup(event->GetID()->Name()); filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - int filterid = 0; - if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> - } - i->filters[filterid] = filter; - i->reader->AddFilter( filterid, fieldsV.size(), logf ); + assert(filter->reader); + filter->reader->Init(filter->source, filter->mode, filter->num_fields, logf ); + + readers[filter->reader] = filter; + + DBG_LOG(DBG_INPUT, "Successfully created event stream %s", + filter->name.c_str()); return true; } -bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Stream not found"); - return false; - } - +bool Manager::CreateTableStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) { reporter->Error("filter argument not of right type"); return false; } + TableFilter* filter = new TableFilter(); + { + bool res = CreateStream(filter, fval); + if ( res == false ) { + delete filter; + return false; + } + } - Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; - if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) { val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + Unref(val); // The lookupwithdefault in the if-clause ref'ed val. } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); + // check if index fields match table description + { + int num = idx->NumFields(); + const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); + + loop_over_list(*tl, j) + { + if ( j >= num ) { + reporter->Error("Table type has more indexes than index definition"); + return false; + } + + if ( !same_type(idx->FieldType(j), (*tl)[j]) ) { + reporter->Error("Table type does not match index type"); + return false; + } + } + + if ( num != j ) { + reporter->Error("Table has less elements than index definition"); + return false; + } + } + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); @@ -390,29 +444,35 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { const type_list* args = etype->ArgTypes()->Types(); - if ( args->length() != 3 ) + if ( args->length() != 4 ) { - reporter->Error("Table event must take 3 arguments"); + reporter->Error("Table event must take 4 arguments"); return false; } - if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) { - reporter->Error("table events first attribute must be of type Input::Event"); + reporter->Error("table events first attribute must be of type Input::TableDescription"); return false; } - if ( ! same_type((*args)[1], idx) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events second attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[2], idx) ) { reporter->Error("table events index attributes do not match"); return false; } - if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) { reporter->Error("table events value attributes do not match"); return false; - } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[3], val->FieldType(0) ) ) { reporter->Error("table events value attribute does not match"); return false; } @@ -445,9 +505,6 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - TableFilter* filter = new TableFilter(); - filter->name = name->AsString()->CheckString(); - filter->id = id->Ref()->AsEnumVal(); filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; @@ -460,19 +517,24 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - Unref(name); Unref(pred); if ( valfields > 1 ) { - assert(filter->want_record); + if ( ! filter->want_record ) { + reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); + delete filter; + return false; + } } - - int filterid = 0; - if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> - } - i->filters[filterid] = filter; - i->reader->AddFilter( filterid, fieldsV.size(), fields ); + + + assert(filter->reader); + filter->reader->Init(filter->source, filter->mode, fieldsV.size(), fields ); + + readers[filter->reader] = filter; + + DBG_LOG(DBG_INPUT, "Successfully created table stream %s", + filter->name.c_str()); return true; } @@ -528,16 +590,8 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) } -bool Manager::RemoveStream(const EnumVal* id) { - ReaderInfo *i = 0; - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->id == id ) - { - i = (*s); - break; - } - } +bool Manager::RemoveStream(const string &name) { + Filter *i = FindFilter(name); if ( i == 0 ) { return false; // not found @@ -545,27 +599,30 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s", + name.c_str()); +#endif + return true; } -bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { - ReaderInfo *i = 0; +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); - - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->reader && (*s)->reader == reader ) - { - i = *s; - delete(i); - readers.erase(s); - return true; - } + if ( i == 0 ) { + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; } - - reporter->Error("Stream not found in RemoveStreamContinuation"); - return false; + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", + i->name.c_str()); +#endif + readers.erase(reader); + delete(i); + return true; } bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { @@ -606,6 +663,10 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) { + field->optional = true; + } + fields->push_back(field); } } @@ -613,104 +674,24 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co return true; } -bool Manager::ForceUpdate(const EnumVal* id) +bool Manager::ForceUpdate(const string &name) { - ReaderInfo *i = FindReader(id); + Filter *i = FindFilter(name); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->Error("Stream %s not found", name.c_str()); return false; } i->reader->Update(); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Forcing update of stream %s", + name.c_str()); +#endif + return true; // update is async :( } -bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - bool found = false; - int filterId; - - for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { - if ( (*it).second->name == name ) { - found = true; - filterId = (*it).first; - - if ( (*it).second->filter_type != TABLE_FILTER ) { - reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); - return false; - } - - break; - } - } - - if ( !found ) { - reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); - return false; - } - - i->reader->RemoveFilter(filterId); - - return true; -} - -bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId) { - ReaderInfo *i = FindReader(reader); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - map::iterator it = i->filters.find(filterId); - if ( it == i->filters.end() ) { - reporter->Error("Got RemoveFilterContinuation where filter nonexistant for %d", filterId); - return false; - } - - delete (*it).second; - i->filters.erase(it); - - return true; -} - -bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - bool found = false; - int filterId; - for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { - if ( (*it).second->name == name ) { - found = true; - filterId = (*it).first; - - if ( (*it).second->filter_type != EVENT_FILTER ) { - reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); - return false; - } - - break; - } - } - - if ( !found ) { - reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); - return false; - } - - i->reader->RemoveFilter(filterId); - return true; -} - Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; @@ -732,31 +713,25 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu idxval = l; } - //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; } -void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); - return; - } - - if ( !i->HasFilter(id) ) { - reporter->InternalError("Unknown filter"); + reporter->InternalError("Unknown reader in SendEntry"); return; } int readFields; - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - readFields = SendEntryTable(reader, id, vals); - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) { + readFields = SendEntryTable(i, vals); + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(i, type, vals); } else { assert(false); } @@ -765,37 +740,35 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals delete vals[i]; } delete [] vals; - - } -int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool updated = false; assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; - //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - //reporter->Error("Result: %d\n", (uint64_t) idxhash->Hash()); - //reporter->Error("Hashing %d val fields", i->num_val_fields); + + if ( idxhash == 0 ) { + reporter->Error("Could not hash line. Ignoring"); + return filter->num_val_fields + filter->num_idx_fields; + } hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); - valhash = valhashkey->Hash(); - delete(valhashkey); + if ( valhashkey == 0 ) { + // empty line. index, but no values. + // hence we also have no hash value... + } else { + valhash = valhashkey->Hash(); + delete(valhashkey); + } } - //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); - - //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before @@ -842,17 +815,15 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } + + bool result; + if ( filter->num_val_fields > 0 ) { // we have values + result = CallPred(filter->pred, 3, ev, predidx, valval); + } else { + // no values + result = CallPred(filter->pred, 2, ev, predidx); + } - val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. - vl.append(ev); - vl.append(predidx); - if ( filter->num_val_fields > 0 ) - vl.append(valval); - - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... @@ -901,14 +872,15 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 3, ev, predidx, oldval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, predidx); + Ref(filter->description); + SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 3, ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); } } } @@ -918,22 +890,25 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } -void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { - ReaderInfo *i = FindReader(reader); +void Manager::EndCurrentSend(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in EndCurrentSend"); return; } - assert(i->HasFilter(id)); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", + i->name.c_str()); +#endif - if ( i->filters[id]->filter_type == EVENT_FILTER ) { + if ( i->filter_type == EVENT_FILTER ) { // nothing to do.. return; } - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = filter->lastDict->InitForIteration(); @@ -945,18 +920,20 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { ListVal * idx = 0; Val *val = 0; + + Val* predidx = 0; + EnumVal* ev = 0; + int startpos = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); val = filter->tab->Lookup(idx); assert(val != 0); + predidx = ListValToRecordVal(idx, filter->itype, &startpos); + ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); } - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - if ( filter->pred ) { // ask predicate, if we want to expire this element... @@ -964,14 +941,8 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { Ref(predidx); Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(predidx); - vl.append(val); - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - + bool result = CallPred(filter->pred, 3, ev, predidx, val); + if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... @@ -989,8 +960,10 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { SendEvent(filter->event, 3, ev, predidx, val); } - Unref(predidx); - Unref(ev); + if ( predidx ) // if we have a filter or an event... + Unref(predidx); + if ( ev ) + Unref(ev); filter->tab->Delete(ih->idxkey); filter->lastDict->Remove(lastDictIdxKey); // deletex in next line @@ -1003,6 +976,11 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", + i->name.c_str()); +#endif + // Send event that the current update is indeed finished. EventHandler* handler = event_registry->Lookup("Input::update_finished"); if ( handler == 0 ) { @@ -1010,46 +988,45 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { } - Ref(i->id); - SendEvent(handler, 1, i->id); + SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); } -void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +void Manager::Put(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Put"); return; } - if ( !i->HasFilter(id) ) { - reporter->InternalError("Unknown filter"); - return; - } - - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - PutTable(reader, id, vals); - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + int readFields; + if ( i->filter_type == TABLE_FILTER ) { + readFields = PutTable(i, vals); + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(i, type, vals); } else { assert(false); } + + for ( int i = 0; i < readFields; i++ ) { + delete vals[i]; + } + delete [] vals; } -int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { bool updated = false; assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == EVENT_FILTER); - EventFilter* filter = (EventFilter*) i->filters[id]; + assert(i->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i; Val *val; list out_vals; + Ref(filter->description); + out_vals.push_back(filter->description); // no tracking, send everything with a new event... //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); @@ -1077,14 +1054,11 @@ int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, i } -int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::PutTable(Filter* i, const Value* const *vals) { assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; @@ -1128,18 +1102,15 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } + + bool result; + if ( filter->num_val_fields > 0 ) { // we have values + result = CallPred(filter->pred, 3, ev, predidx, valval); + } else { + // no values + result = CallPred(filter->pred, 2, ev, predidx); + } - val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. - vl.append(ev); - vl.append(predidx); - if ( filter->num_val_fields > 0 ) - vl.append(valval); - - - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - if ( result == false ) { // do nothing Unref(idxval); @@ -1150,7 +1121,6 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } - filter->tab->Assign(idxval, valval); if ( filter->event ) { @@ -1162,23 +1132,18 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 3, ev, predidx, oldval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, predidx); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 3, ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); } } - } - - - } else { // no predicates or other stuff @@ -1188,35 +1153,38 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * return filter->num_idx_fields + filter->num_val_fields; } -void Manager::Clear(const ReaderFrontend* reader, int id) { - ReaderInfo *i = FindReader(reader); +// Todo:: perhaps throw some kind of clear-event? +void Manager::Clear(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Clear"); return; } - assert(i->HasFilter(id)); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Got Clear for stream %s", + i->name.c_str()); +#endif - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; filter->tab->RemoveAll(); } -bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +// put interface: delete old entry from table. +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Delete"); return false; } - assert(i->HasFilter(id)); - bool success = false; int readVals = 0; - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - TableFilter* filter = (TableFilter*) i->filters[id]; + if ( i->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; @@ -1231,13 +1199,7 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - val_list vl(3); - vl.append(ev); - vl.append(predidx); - vl.append(val); - Val* v = filter->pred->Call(&vl); - filterresult = v->AsBool(); - Unref(v); + filterresult = CallPred(filter->pred, 3, ev, predidx, val); if ( filterresult == false ) { // keep it. @@ -1253,7 +1215,7 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { assert(val != 0); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(filter->event, 3, ev, idxval, val); + SendEvent(filter->event, 4, filter->description->Ref(), ev, idxval, val); } } @@ -1264,9 +1226,9 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { reporter->Error("Internal error while deleting values from input table"); } } - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - readVals = SendEventFilterEvent(reader, type, id, vals); + readVals = SendEventFilterEvent(i, type, vals); success = true; } else { assert(false); @@ -1281,6 +1243,26 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { return success; } +bool Manager::CallPred(Func* pred_func, const int numvals, ...) +{ + bool result; + val_list vl(numvals); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl.append( va_arg(lP, Val*) ); + } + va_end(lP); + + Val* v = pred_func->Call(&vl); + result = v->AsBool(); + Unref(v); + + return(result); +} + bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -1337,10 +1319,18 @@ void Manager::SendEvent(EventHandlerPtr ev, list events) mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } - +// Convert a bro list value to a bro record value. I / we could think about moving this functionality to val.cc RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + assert(position != 0 ); // we need the pointer to point to data; + + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("ListValToRecordVal called on non-record-value."); + return 0; + } + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + assert(list != 0); int maxpos = list->Length(); for ( int i = 0; i < request_type->NumFields(); i++ ) { @@ -1360,20 +1350,14 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, return rec; } - - +// Convert a threading value to a record value RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { - if ( position == 0 ) { - reporter->InternalError("Need position"); - return 0; - } + assert(position != 0); // we need the pointer to point to data. - /* if ( request_type->Tag() != TYPE_RECORD ) { - reporter->InternalError("I only work with records"); + reporter->InternalError("ValueToRecordVal called on non-record-value."); return 0; - } */ - + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); for ( int i = 0; i < request_type->NumFields(); i++ ) { @@ -1390,11 +1374,12 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *reque } return rec; - } - +// Count the length of the values +// used to create a correct length buffer for hashing later int Manager::GetValueLength(const Value* val) { + assert( val->present ); // presence has to be checked elsewhere int length = 0; switch (val->type) { @@ -1481,19 +1466,20 @@ int Manager::GetValueLength(const Value* val) { } +// Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. +// Used for hashing the values for lookup in the bro table int Manager::CopyValue(char *data, const int startpos, const Value* val) { + assert( val->present ); // presence has to be checked elsewhere + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+startpos) = val->val.int_val; return sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: - //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; @@ -1512,7 +1498,6 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - //*(data+startpos) = val->val.double_val; memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); return sizeof(val->val.double_val); break; @@ -1594,12 +1579,11 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { return 0; } - reporter->InternalError("internal error"); assert(false); return 0; - } +// Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { int length = 0; @@ -1609,14 +1593,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { length += GetValueLength(val); } - //reporter->Error("Length: %d", length); + if ( length == 0 ) { + reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); + return NULL; + } int position = 0; char *data = (char*) malloc(length); if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } - //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; if ( val->present ) @@ -1627,10 +1613,9 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { assert(position == length); return new HashKey(data, length, key, true); - - } +// convert threading value to Bro value Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1641,7 +1626,6 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( !val->present ) { return 0; // unset field } - switch ( val->type ) { case TYPE_BOOL: @@ -1754,35 +1738,28 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { reporter->InternalError("unsupported type for input_read"); } - - reporter->InternalError("Impossible error"); + assert(false); return NULL; } -Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) +Manager::Filter* Manager::FindFilter(const string &name) { - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader && (*s)->reader == reader ) + if ( (*s).second->name == name ) { - return *s; + return (*s).second; } } return 0; } - -Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) - { - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) - { - return *s; - } - } - - return 0; +Manager::Filter* Manager::FindFilter(ReaderFrontend* reader) +{ + map::iterator s = readers.find(reader); + if ( s != readers.end() ) { + return s->second; } - + return 0; +} diff --git a/src/input/Manager.h b/src/input/Manager.h index b4fc6cff7f..71169c4bc2 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -11,7 +11,7 @@ #include "../EventHandler.h" #include "../RemoteSerializer.h" -#include +#include namespace input { @@ -35,6 +35,9 @@ public: /** * Creates a new input stream. + * Add a filter to an input source, which will write the data from the data source into + * a Bro table. + * Add a filter to an input source, which sends events for read input data. * * @param id The enum value corresponding the input stream. * @@ -43,7 +46,9 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - ReaderFrontend* CreateStream(EnumVal* id, RecordVal* description); + bool CreateTableStream(RecordVal* description); + bool CreateEventStream(RecordVal* description); + /** * Force update on a input stream. @@ -57,7 +62,7 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool ForceUpdate(const EnumVal* id); + bool ForceUpdate(const string &id); /** * Deletes an existing input stream @@ -67,53 +72,8 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool RemoveStream(const EnumVal* id); + bool RemoveStream(const string &id); - /** - * Add a filter to an input source, which will write the data from the data source into - * a Bro table. - * - * @param id The enum value corresponding the input stream. - * - * @param description A record of script type \c Input:TableFilter. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool AddTableFilter(EnumVal *id, RecordVal* filter); - - /** - * Removes a tablefilter from the log stream - * - * @param id The enum value corresponding the input stream. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool RemoveTableFilter(EnumVal* id, const string &name); - - /** - * Add a filter to an input source, which sends events for read input data. - * - * @param id The enum value corresponding the input stream. - * - * @param description A record of script type \c Input:EventFilter. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool AddEventFilter(EnumVal *id, RecordVal* filter); - - /** - * Removes a eventfilter from the log stream - * - * @param id The enum value corresponding the input stream. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool RemoveEventFilter(EnumVal* id, const string &name); - protected: friend class ReaderFrontend; friend class PutMessage; @@ -122,19 +82,18 @@ protected: friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; - friend class FilterRemovedMessage; friend class ReaderFinishedMessage; // For readers to write to input stream in direct mode (reporting new/deleted values directly) // Functions take ownership of threading::Value fields - void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); - void Clear(const ReaderFrontend* reader, int id); - bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); + void Put(ReaderFrontend* reader, threading::Value* *vals); + void Clear(ReaderFrontend* reader); + bool Delete(ReaderFrontend* reader, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) // Functions take ownership of threading::Value fields - void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); - void EndCurrentSend(const ReaderFrontend* reader, const int id); + void SendEntry(ReaderFrontend* reader, threading::Value* *vals); + void EndCurrentSend(ReaderFrontend* reader); // Allows readers to directly send Bro events. // The num_vals and vals must be the same the named event expects. @@ -150,20 +109,23 @@ protected: // Used to prevent race conditions where data for a specific filter is still in the queue when the // RemoveFilter directive is executed by the main thread. // This makes sure all data that has ben queued for a filter is still received. - bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); - bool RemoveStreamContinuation(const ReaderFrontend* reader); + bool RemoveStreamContinuation(ReaderFrontend* reader); private: - struct ReaderInfo; + class Filter; + class TableFilter; + class EventFilter; + + bool CreateStream(Filter*, RecordVal* description); // SendEntry implementation for Tablefilter - int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int SendEntryTable(Filter* i, const threading::Value* const *vals); // Put implementation for Tablefilter - int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int PutTable(Filter* i, const threading::Value* const *vals); // SendEntry and Put implementation for Eventfilter - int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + int SendEventFilterEvent(Filter* i, EnumVal* type, const threading::Value* const *vals); // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types // from the log framework @@ -177,6 +139,9 @@ private: void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); + // Call predicate function and return result + bool CallPred(Func* pred_func, const int numvals, ...); + // get a hashkey for a set of threading::Values HashKey* HashValues(const int num_elements, const threading::Value* const *vals); @@ -197,16 +162,12 @@ private: // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); - ReaderInfo* FindReader(const ReaderFrontend* reader); - ReaderInfo* FindReader(const EnumVal* id); - - vector readers; - - class Filter; - class TableFilter; - class EventFilter; + Filter* FindFilter(const string &name); + Filter* FindFilter(ReaderFrontend* reader); enum FilterType { TABLE_FILTER, EVENT_FILTER }; + + map readers; }; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 5af02b1acc..0a6ff37dc2 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -11,48 +11,44 @@ namespace input { class PutMessage : public threading::OutputMessage { public: - PutMessage(ReaderFrontend* reader, int id, Value* *val) + PutMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("Put", reader), - id(id), val(val) {} + val(val) {} virtual bool Process() { - input_mgr->Put(Object(), id, val); + input_mgr->Put(Object(), val); return true; } private: - int id; Value* *val; }; class DeleteMessage : public threading::OutputMessage { public: - DeleteMessage(ReaderFrontend* reader, int id, Value* *val) + DeleteMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("Delete", reader), - id(id), val(val) {} + val(val) {} virtual bool Process() { - return input_mgr->Delete(Object(), id, val); + return input_mgr->Delete(Object(), val); } private: - int id; Value* *val; }; class ClearMessage : public threading::OutputMessage { public: - ClearMessage(ReaderFrontend* reader, int id) - : threading::OutputMessage("Clear", reader), - id(id) {} + ClearMessage(ReaderFrontend* reader) + : threading::OutputMessage("Clear", reader) {} virtual bool Process() { - input_mgr->Clear(Object(), id); + input_mgr->Clear(Object()); return true; } private: - int id; }; class SendEventMessage : public threading::OutputMessage { @@ -73,47 +69,30 @@ private: class SendEntryMessage : public threading::OutputMessage { public: - SendEntryMessage(ReaderFrontend* reader, const int id, Value* *val) + SendEntryMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("SendEntry", reader), - id(id), val(val) { } + val(val) { } virtual bool Process() { - input_mgr->SendEntry(Object(), id, val); + input_mgr->SendEntry(Object(), val); return true; } private: - const int id; Value* *val; }; class EndCurrentSendMessage : public threading::OutputMessage { public: - EndCurrentSendMessage(ReaderFrontend* reader, const int id) - : threading::OutputMessage("EndCurrentSend", reader), - id(id) {} + EndCurrentSendMessage(ReaderFrontend* reader) + : threading::OutputMessage("EndCurrentSend", reader) {} virtual bool Process() { - input_mgr->EndCurrentSend(Object(), id); + input_mgr->EndCurrentSend(Object()); return true; } private: - const int id; -}; - -class FilterRemovedMessage : public threading::OutputMessage { -public: - FilterRemovedMessage(ReaderFrontend* reader, const int id) - : threading::OutputMessage("FilterRemoved", reader), - id(id) {} - - virtual bool Process() { - return input_mgr->RemoveFilterContinuation(Object(), id); - } - -private: - const int id; }; class ReaderFinishedMessage : public threading::OutputMessage { @@ -155,19 +134,19 @@ ReaderBackend::~ReaderBackend() } -void ReaderBackend::Put(int id, Value* *val) +void ReaderBackend::Put(Value* *val) { - SendOut(new PutMessage(frontend, id, val)); + SendOut(new PutMessage(frontend, val)); } -void ReaderBackend::Delete(int id, Value* *val) +void ReaderBackend::Delete(Value* *val) { - SendOut(new DeleteMessage(frontend, id, val)); + SendOut(new DeleteMessage(frontend, val)); } -void ReaderBackend::Clear(int id) +void ReaderBackend::Clear() { - SendOut(new ClearMessage(frontend, id)); + SendOut(new ClearMessage(frontend)); } void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) @@ -175,70 +154,34 @@ void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *va SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } -void ReaderBackend::EndCurrentSend(int id) +void ReaderBackend::EndCurrentSend() { - SendOut(new EndCurrentSendMessage(frontend, id)); + SendOut(new EndCurrentSendMessage(frontend)); } -void ReaderBackend::SendEntry(int id, Value* *vals) +void ReaderBackend::SendEntry(Value* *vals) { - SendOut(new SendEntryMessage(frontend, id, vals)); + SendOut(new SendEntryMessage(frontend, vals)); } -bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) +bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* arg_fields) { source = arg_source; - autostart = arg_autostart; SetName("InputReader/"+source); // disable if DoInit returns error. - disabled = !DoInit(arg_source, mode); + int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); - if ( disabled ) { + if ( !success ) { Error("Init failed"); DisableFrontend(); } - return !disabled; -} - -bool ReaderBackend::StartReading() { - if ( disabled ) - return false; - - int success = DoStartReading(); - - if ( success == false ) { - DisableFrontend(); - } + disabled = !success; return success; } -bool ReaderBackend::AddFilter(int id, int arg_num_fields, - const Field* const * arg_fields) -{ - if ( disabled ) - return false; - - bool success = DoAddFilter(id, arg_num_fields, arg_fields); - if ( success && autostart) { - autostart = false; - return StartReading(); - } - return success; -} - -bool ReaderBackend::RemoveFilter(int id) -{ - if ( disabled ) - return false; - - bool success = DoRemoveFilter(id); - SendOut(new FilterRemovedMessage(frontend, id)); - return success; // yes, I know, noone reads this. -} - void ReaderBackend::Finish() { DoFinish(); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index e167f8ff47..28fd99f2b9 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -53,46 +53,14 @@ public: * * @param mode the opening mode for the input source * - * @param autostart automatically start the input source after the first filter has been added - * - * @return False if an error occured. - */ - bool Init(string arg_source, int mode, bool autostart); - - /** - * One-time start method of the reader. - * - * This method is called from the scripting layer, after all filters have been added. - * No data should be read before this method is called. - * - * If autostart in Init is set to true, this method is called automatically by the backend after - * the first filter has been added. - */ - bool StartReading(); - - /** - * Add an input filter to the input stream - * - * @param id identifier of the input stream - * * @param arg_num_fields number of fields contained in \a fields * * @param fields the types and names of the fields to be retrieved from the input source * * @return False if an error occured. */ - bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields); - - /** - * Remove an input filter to the input stream - * - * @param id identifier of the input stream - * - * @return False if an error occured. - */ - bool RemoveFilter ( int id ); - /** * Finishes reading from this input stream in a regular fashion. Must not be * called if an error has been indicated earlier. After calling this, @@ -131,33 +99,7 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoInit(string arg_sources, int mode) = 0; - - /** - * Reader-specific start method. After this function has been called, data may be read from - * the input source and be sent to the specified filters - * - * A reader implementation must override this method. - * If it returns false, it will be assumed that a fatal error has occured - * that prevents the reader from further operation; it will then be - * disabled and eventually deleted. When returning false, an implementation - * should also call Error to indicate what happened. - */ - virtual bool DoStartReading() = 0; - - /** - * Reader-specific method to add a filter. - * - * A reader implementation must override this method. - */ - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; - - /** - * Reader-specific method to remove a filter. - * - * A reader implementation must override this method. - */ - virtual bool DoRemoveFilter( int id ) = 0; + virtual bool DoInit(string arg_sources, int mode, int arg_num_fields, const threading::Field* const* fields) = 0; /** * Reader-specific method implementing input finalization at @@ -209,31 +151,26 @@ protected: * * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void Put(int id, threading::Value* *val); + void Put(threading::Value* *val); /** * Method allowing a reader to delete a specific value from a bro table. * * If the receiving filter is an event, only a removed event is raised * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void Delete(int id, threading::Value* *val); + void Delete(threading::Value* *val); /** * Method allowing a reader to clear a value from a bro table. * * If the receiving filter is an event, this is ignored. * - * @param id the input filter id for which the values are sent */ - void Clear(int id); + void Clear(); // Content-sending-functions (tracking mode): Only changed lines are propagated. @@ -243,11 +180,9 @@ protected: * * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised. * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void SendEntry(int id, threading::Value* *vals); + void SendEntry(threading::Value* *vals); /** * Method telling the manager, that the current list of entries sent by SendEntry is finished. @@ -255,9 +190,8 @@ protected: * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer * present in the input source * - * @param id the input filter id for which the values are sent */ - void EndCurrentSend(int id); + void EndCurrentSend(); /** * Triggered by regular heartbeat messages from the main thread. diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index f7fc23bf72..9711997821 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,16 +12,17 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source, const int mode, const bool autostart) + InitMessage(ReaderBackend* backend, const string source, const int mode, const int num_fields, const threading::Field* const* fields) : threading::InputMessage("Init", backend), - source(source), mode(mode), autostart(autostart) { } + source(source), mode(mode), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(source, mode, autostart); } + virtual bool Process() { return Object()->Init(source, mode, num_fields, fields); } private: const string source; const int mode; - const bool autostart; + const int num_fields; + const threading::Field* const* fields; }; class UpdateMessage : public threading::InputMessage @@ -44,44 +45,6 @@ public: virtual bool Process() { Object()->Finish(); return true; } }; -class StartReadingMessage : public threading::InputMessage -{ -public: - StartReadingMessage(ReaderBackend* backend) - : threading::InputMessage("StartReading", backend) - { } - - virtual bool Process() { Object()->StartReading(); return true; } -}; - -class AddFilterMessage : public threading::InputMessage -{ -public: - AddFilterMessage(ReaderBackend* backend, const int id, const int num_fields, const threading::Field* const* fields) - : threading::InputMessage("AddFilter", backend), - id(id), num_fields(num_fields), fields(fields) { } - - virtual bool Process() { return Object()->AddFilter(id, num_fields, fields); } - -private: - const int id; - const int num_fields; - const threading::Field* const* fields; -}; - -class RemoveFilterMessage : public threading::InputMessage -{ -public: - RemoveFilterMessage(ReaderBackend* backend, const int id) - : threading::InputMessage("RemoveFilter", backend), - id(id) { } - - virtual bool Process() { return Object()->RemoveFilter(id); } - -private: - const int id; -}; - ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; @@ -95,7 +58,7 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) { ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { +void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, const threading::Field* const* fields) { if ( disabled ) return; @@ -105,37 +68,33 @@ void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { source = arg_source; initialized = true; - backend->SendIn(new InitMessage(backend, arg_source, mode, autostart)); + backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); } void ReaderFrontend::Update() { if ( disabled ) return; + if ( !initialized ) { + reporter->Error("Tried to call update on uninitialized reader"); + return; + } + backend->SendIn(new UpdateMessage(backend)); } void ReaderFrontend::Finish() { if ( disabled ) return; + + if ( !initialized ) { + reporter->Error("Tried to call finish on uninitialized reader"); + return; + } backend->SendIn(new FinishMessage(backend)); } -void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const threading::Field* const* fields) { - if ( disabled ) - return; - - backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); -} - -void ReaderFrontend::RemoveFilter(const int id) { - if ( disabled ) - return; - - backend->SendIn(new RemoveFilterMessage(backend, id)); -} - string ReaderFrontend::Name() const { if ( source.size() ) @@ -144,13 +103,5 @@ string ReaderFrontend::Name() const return ty_name + "/" + source; } -void ReaderFrontend::StartReading() { - if ( disabled ) - return; - - backend->SendIn(new StartReadingMessage(backend)); } -} - - diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 0df4c00e8c..1c3306e0c1 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -49,18 +49,7 @@ public: * See ReaderBackend::Init() for arguments. * This method must only be called from the main thread. */ - void Init(string arg_source, int mode, bool autostart); - - /** - * Start the reader. - * - * This methods starts the reader, after all necessary filters have been added. - * It is not necessary to call this function, if autostart has been set. - * If autostart has been set, the reader will be initialized automatically after the first filter has been added - * - * This method must only be called from the main thread. - */ - void StartReading(); + void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields); /** * Force an update of the current input source. Actual action depends on @@ -72,20 +61,6 @@ public: */ void Update(); - /** - * Add a filter to the current input source. - * - * See ReaderBackend::AddFilter for arguments. - * - * The method takes ownership of \a fields - */ - void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); - - /** - * Removes a filter to the current input source. - */ - void RemoveFilter ( const int id ); - /** * Finalizes writing to this tream. * diff --git a/src/input/fdstream.h b/src/input/fdstream.h new file mode 100644 index 0000000000..cda767dd52 --- /dev/null +++ b/src/input/fdstream.h @@ -0,0 +1,189 @@ +/* The following code declares classes to read from and write to + * file descriptore or file handles. + * + * See + * http://www.josuttis.com/cppcode + * for details and the latest version. + * + * - open: + * - integrating BUFSIZ on some systems? + * - optimized reading of multiple characters + * - stream for reading AND writing + * - i18n + * + * (C) Copyright Nicolai M. Josuttis 2001. + * Permission to copy, use, modify, sell and distribute this software + * is granted provided this copyright notice appears in all copies. + * This software is provided "as is" without express or implied + * warranty, and with no claim as to its suitability for any purpose. + * + * Version: Jul 28, 2002 + * History: + * Jul 28, 2002: bugfix memcpy() => memmove() + * fdinbuf::underflow(): cast for return statements + * Aug 05, 2001: first public version + */ +#ifndef BOOST_FDSTREAM_HPP +#define BOOST_FDSTREAM_HPP + +#include +#include +#include +// for EOF: +#include +// for memmove(): +#include + + + +// low-level read and write functions +#ifdef _MSC_VER +# include +#else +# include +# include +//extern "C" { +// int write (int fd, const char* buf, int num); +// int read (int fd, char* buf, int num); +//} +#endif + + +// BEGIN namespace BOOST +namespace boost { + + +/************************************************************ + * fdostream + * - a stream that writes on a file descriptor + ************************************************************/ + + +class fdoutbuf : public std::streambuf { + protected: + int fd; // file descriptor + public: + // constructor + fdoutbuf (int _fd) : fd(_fd) { + } + protected: + // write one character + virtual int_type overflow (int_type c) { + if (c != EOF) { + char z = c; + if (write (fd, &z, 1) != 1) { + return EOF; + } + } + return c; + } + // write multiple characters + virtual + std::streamsize xsputn (const char* s, + std::streamsize num) { + return write(fd,s,num); + } +}; + +class fdostream : public std::ostream { + protected: + fdoutbuf buf; + public: + fdostream (int fd) : std::ostream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +/************************************************************ + * fdistream + * - a stream that reads on a file descriptor + ************************************************************/ + +class fdinbuf : public std::streambuf { + protected: + int fd; // file descriptor + protected: + /* data buffer: + * - at most, pbSize characters in putback area plus + * - at most, bufSize characters in ordinary read buffer + */ + static const int pbSize = 4; // size of putback area + static const int bufSize = 1024; // size of the data buffer + char buffer[bufSize+pbSize]; // data buffer + + public: + /* constructor + * - initialize file descriptor + * - initialize empty data buffer + * - no putback area + * => force underflow() + */ + fdinbuf (int _fd) : fd(_fd) { + setg (buffer+pbSize, // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize); // end position + } + + protected: + // insert new characters into the buffer + virtual int_type underflow () { +#ifndef _MSC_VER + using std::memmove; +#endif + + // is read position before end of buffer? + if (gptr() < egptr()) { + return traits_type::to_int_type(*gptr()); + } + + /* process size of putback area + * - use number of characters read + * - but at most size of putback area + */ + int numPutback; + numPutback = gptr() - eback(); + if (numPutback > pbSize) { + numPutback = pbSize; + } + + /* copy up to pbSize characters previously read into + * the putback area + */ + memmove (buffer+(pbSize-numPutback), gptr()-numPutback, + numPutback); + + // read at most bufSize new characters + int num; + num = read (fd, buffer+pbSize, bufSize); + if ( num == EAGAIN ) { + return 0; + } + if (num <= 0) { + // ERROR or EOF + return EOF; + } + + // reset buffer pointers + setg (buffer+(pbSize-numPutback), // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize+num); // end of buffer + + // return next character + return traits_type::to_int_type(*gptr()); + } +}; + +class fdistream : public std::istream { + protected: + fdinbuf buf; + public: + fdistream (int fd) : std::istream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +} // END namespace boost + +#endif /*BOOST_FDSTREAM_HPP*/ diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index ab26442524..20ae79ab19 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -26,6 +26,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) @@ -33,10 +34,11 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, cons { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), subtype(arg.subtype) + : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) { position = arg.position; secondary_position = arg.secondary_position; @@ -76,7 +78,6 @@ Ascii::~Ascii() void Ascii::DoFinish() { - filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -84,13 +85,15 @@ void Ascii::DoFinish() } } -bool Ascii::DoInit(string path, int arg_mode) +bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { - started = false; fname = path; mode = arg_mode; mtime = 0; + num_fields = arg_num_fields; + fields = arg_fields; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; @@ -107,17 +110,7 @@ bool Ascii::DoInit(string path, int arg_mode) file->close(); return false; } - - return true; -} - -bool Ascii::DoStartReading() { - if ( started == true ) { - Error("Started twice"); - return false; - } - - started = true; + switch ( mode ) { case MANUAL: case REREAD: @@ -131,46 +124,11 @@ bool Ascii::DoStartReading() { return true; } -bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { - if ( HasFilter(id) ) { - Error("Filter was added twice, ignoring."); - return false; // no, we don't want to add this a second time - } - - Filter f; - f.num_fields = arg_num_fields; - f.fields = fields; - - filters[id] = f; - - return true; -} - -bool Ascii::DoRemoveFilter ( int id ) { - if (!HasFilter(id) ) { - Error("Filter removal of nonexisting filter requested."); - return false; - } - - assert ( filters.erase(id) == 1 ); - - return true; -} - - -bool Ascii::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} - bool Ascii::ReadHeader(bool useCached) { // try to read the header line... string line; - map fields; + map ifields; if ( !useCached ) { if ( !GetLine(line) ) { @@ -194,37 +152,43 @@ bool Ascii::ReadHeader(bool useCached) { if ( !getline(splitstream, s, separator[0])) break; - fields[s] = pos; + ifields[s] = pos; pos++; } //printf("Updating fields from description %s\n", line.c_str()); - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - (*it).second.columnMap.clear(); - - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const Field* field = (*it).second.fields[i]; - - map::iterator fit = fields.find(field->name); - if ( fit == fields.end() ) { - Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); - return false; + columnMap.clear(); + + for ( unsigned int i = 0; i < num_fields; i++ ) { + const Field* field = fields[i]; + + map::iterator fit = ifields.find(field->name); + if ( fit == ifields.end() ) { + if ( field->optional ) { + // we do not really need this field. mark it as not present and always send an undef back. + FieldMapping f(field->name, field->type, field->subtype, -1); + f.present = false; + columnMap.push_back(f); + continue; } - - FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); - if ( field->secondary_name != "" ) { - map::iterator fit2 = fields.find(field->secondary_name); - if ( fit2 == fields.end() ) { - Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); - return false; - } - f.secondary_position = fields[field->secondary_name]; - } - (*it).second.columnMap.push_back(f); + Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); + return false; } + + FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = ifields.find(field->secondary_name); + if ( fit2 == ifields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; + } + f.secondary_position = ifields[field->secondary_name]; + } + columnMap.push_back(f); } + // well, that seems to have worked... return true; @@ -266,7 +230,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { } else if ( s == "F" ) { val->val.int_val = 0; } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); + Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); return false; } break; @@ -461,57 +425,63 @@ bool Ascii::DoUpdate() { pos--; // for easy comparisons of max element. - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - Value** fields = new Value*[(*it).second.num_fields]; + Value** fields = new Value*[num_fields]; - int fpos = 0; - for ( vector::iterator fit = (*it).second.columnMap.begin(); - fit != (*it).second.columnMap.end(); - fit++ ){ - - - if ( (*fit).position > pos || (*fit).secondary_position > pos ) { - Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); - return false; - } - - Value* val = EntryToVal(stringfields[(*fit).position], *fit); - if ( val == 0 ) { - Error("Could not convert String value to Val"); - return false; - } - - if ( (*fit).secondary_position != -1 ) { - // we have a port definition :) - assert(val->type == TYPE_PORT ); - // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - - val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); - } - - fields[fpos] = val; + int fpos = 0; + for ( vector::iterator fit = columnMap.begin(); + fit != columnMap.end(); + fit++ ){ + if ( ! fit->present ) { + // add non-present field + fields[fpos] = new Value((*fit).type, false); fpos++; + continue; + } + + assert(fit->position >= 0 ); + + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); + return false; } - //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); - assert ( (unsigned int) fpos == (*it).second.num_fields ); + Value* val = EntryToVal(stringfields[(*fit).position], *fit); + if ( val == 0 ) { + Error("Could not convert String value to Val"); + return false; + } + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - if ( mode == STREAM ) { - Put((*it).first, fields); - } else { - SendEntry((*it).first, fields); + val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); } - /* Do not do this, ownership changes to other thread - * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - delete fields[i]; - } - delete [] fields; - */ + fields[fpos] = val; + + fpos++; } + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); + assert ( (unsigned int) fpos == num_fields ); + + if ( mode == STREAM ) { + Put(fields); + } else { + SendEntry(fields); + } + + /* Do not do this, ownership changes to other thread + * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; + */ + } @@ -519,9 +489,7 @@ bool Ascii::DoUpdate() { //file->seekg(0, ios::beg); // and seek to start. if ( mode != STREAM ) { - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - EndCurrentSend((*it).first); - } + EndCurrentSend(); } return true; @@ -530,7 +498,7 @@ bool Ascii::DoUpdate() { bool Ascii::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - + switch ( mode ) { case MANUAL: // yay, we do nothing :) diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 3bb0e91853..a9b14768fb 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -19,6 +19,7 @@ struct FieldMapping { int position; // for ports: pos of the second field int secondary_position; + bool present; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); @@ -39,33 +40,22 @@ public: protected: - virtual bool DoInit(string path, int mode); - - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); - - virtual bool DoRemoveFilter ( int id ); + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoFinish(); virtual bool DoUpdate(); - virtual bool DoStartReading(); - private: virtual bool DoHeartbeat(double network_time, double current_time); - struct Filter { - unsigned int num_fields; + unsigned int num_fields; - const threading::Field* const * fields; // raw mapping + const threading::Field* const * fields; // raw mapping - // map columns in the file to columns to send back to the manager - vector columnMap; - - }; - - bool HasFilter(int id); + // map columns in the file to columns to send back to the manager + vector columnMap; bool ReadHeader(bool useCached); threading::Value* EntryToVal(string s, FieldMapping type); @@ -75,8 +65,6 @@ private: ifstream* file; string fname; - map filters; - // Options set from the script-level. string separator; @@ -91,7 +79,6 @@ private: int mode; - bool started; time_t mtime; }; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f2892e7af5..fb9243e713 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -7,6 +7,7 @@ #include #include "../../threading/SerialTypes.h" +#include "../fdstream.h" #define MANUAL 0 #define REREAD 1 @@ -15,6 +16,7 @@ #include #include #include +#include using namespace input::reader; using threading::Value; @@ -23,6 +25,7 @@ using threading::Field; Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; + in = 0; //keyMap = new map(); @@ -40,57 +43,74 @@ Raw::~Raw() void Raw::DoFinish() { - filters.empty(); if ( file != 0 ) { - file->close(); - delete(file); - file = 0; + Close(); } } -bool Raw::DoInit(string path, int arg_mode) +bool Raw::Open() +{ + if ( execute ) { + file = popen(fname.c_str(), "r"); + if ( file == NULL ) { + Error(Fmt("Could not execute command %s", fname.c_str())); + return false; + } + } else { + file = fopen(fname.c_str(), "r"); + if ( file == NULL ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + } + + in = new boost::fdistream(fileno(file)); + + if ( execute && mode == STREAM ) { + fcntl(fileno(file), F_SETFL, O_NONBLOCK); + } + + return true; +} + +bool Raw::Close() +{ + if ( file == NULL ) { + InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); + return false; + } + + if ( execute ) { + delete(in); + pclose(file); + } else { + delete(in); + fclose(file); + } + + in = NULL; + file = NULL; + + return true; +} + +bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { - started = false; fname = path; mode = arg_mode; mtime = 0; + execute = false; + firstrun = true; + bool result; + + num_fields = arg_num_fields; + fields = arg_fields; + + if ( path.length() == 0 ) { + Error("No source path provided"); + return false; + } - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); - return false; - } - - file = new ifstream(path.c_str()); - if ( !file->is_open() ) { - Error(Fmt("Init: cannot open %s", fname.c_str())); - return false; - } - - return true; -} - -bool Raw::DoStartReading() { - if ( started == true ) { - Error("Started twice"); - return false; - } - - started = true; - switch ( mode ) { - case MANUAL: - case REREAD: - case STREAM: - DoUpdate(); - break; - default: - assert(false); - } - - return true; -} - -bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { - if ( arg_num_fields != 1 ) { Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); return false; @@ -101,42 +121,51 @@ bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) return false; } - if ( HasFilter(id) ) { - Error("Filter was added twice, ignoring"); - return false; // no, we don't want to add this a second time + // do Initialization + char last = path[path.length()-1]; + if ( last == '|' ) { + execute = true; + fname = path.substr(0, fname.length() - 1); + + if ( ( mode != MANUAL ) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str())); + return false; + } + + result = Open(); + + } else { + execute = false; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str())); + return false; + } + + result = Open(); + } - Filter f; - f.num_fields = arg_num_fields; - f.fields = fields; + if ( result == false ) { + return result; + } - filters[id] = f; +#ifdef DEBUG + Debug(DBG_INPUT, "Raw reader created, will perform first update"); +#endif + + // after initialization - do update + DoUpdate(); + +#ifdef DEBUG + Debug(DBG_INPUT, "First update went through"); +#endif return true; } -bool Raw::DoRemoveFilter ( int id ) { - if (!HasFilter(id) ) { - Error("Filter removal of nonexisting filter requested."); - return false; - } - - assert ( filters.erase(id) == 1 ); - - return true; -} - - -bool Raw::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} bool Raw::GetLine(string& str) { - while ( getline(*file, str, separator[0]) ) { + while ( getline(*in, str, separator[0]) ) { return true; } @@ -146,63 +175,59 @@ bool Raw::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool Raw::DoUpdate() { - switch ( mode ) { - case REREAD: - // check if the file has changed - struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { - Error(Fmt("Could not get stat for %s", fname.c_str())); - return false; - } + if ( firstrun ) { + firstrun = false; + } else { + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } - if ( sb.st_mtime <= mtime ) { - // no change - return true; - } + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } - mtime = sb.st_mtime; - // file changed. reread. + mtime = sb.st_mtime; + // file changed. reread. - // fallthrough - case MANUAL: - case STREAM: - - if ( file && file->is_open() ) { - if ( mode == STREAM ) { - file->clear(); // remove end of file evil bits + // fallthrough + case MANUAL: + case STREAM: + Debug(DBG_INPUT, "Updating"); + if ( mode == STREAM && file != NULL && in != NULL ) { + fpurge(file); + in->clear(); // remove end of file evil bits break; } - file->close(); - } - file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); - return false; - } - break; - default: - assert(false); + Close(); + if ( !Open() ) { + return false; + } + break; + default: + assert(false); + } } string line; while ( GetLine(line) ) { - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + assert (num_fields == 1); + + Value** fields = new Value*[1]; - assert ((*it).second.num_fields == 1); + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val = new string(line); + fields[0] = val; - Value** fields = new Value*[1]; - - // filter has exactly one text field. convert to it. - Value* val = new Value(TYPE_STRING, true); - val->val.string_val = new string(line); - fields[0] = val; - - Put((*it).first, fields); - - } - + Put(fields); } return true; @@ -213,6 +238,8 @@ bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); + Debug(DBG_INPUT, "Heartbeat"); + switch ( mode ) { case MANUAL: // yay, we do nothing :) diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index e046cb2ff7..1cbeff4f83 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -19,36 +19,25 @@ public: protected: - virtual bool DoInit(string path, int mode); - - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); - - virtual bool DoRemoveFilter ( int id ); + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoFinish(); virtual bool DoUpdate(); - virtual bool DoStartReading(); - private: virtual bool DoHeartbeat(double network_time, double current_time); - - struct Filter { - unsigned int num_fields; - - const threading::Field* const * fields; // raw mapping - }; - - bool HasFilter(int id); + bool Open(); + bool Close(); bool GetLine(string& str); - ifstream* file; - string fname; + istream* in; - map filters; + FILE* file; + + string fname; // Options set from the script-level. string separator; @@ -57,9 +46,14 @@ private: string headerline; int mode; + bool execute; + bool firstrun; - bool started; time_t mtime; + + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping }; diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 0753296cb4..74220ecde4 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -105,9 +105,6 @@ Manager::Stream::~Stream() { WriterInfo* winfo = i->second; - if ( ! winfo ) - continue; - if ( winfo->rotation_timer ) timer_mgr->Cancel(winfo->rotation_timer); @@ -207,7 +204,7 @@ Manager::WriterInfo* Manager::FindWriter(WriterFrontend* writer) { WriterInfo* winfo = i->second; - if ( winfo && winfo->writer == writer ) + if ( winfo->writer == writer ) return winfo; } } @@ -221,7 +218,7 @@ void Manager::RemoveDisabledWriters(Stream* stream) for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - if ( j->second && j->second->writer->Disabled() ) + if ( j->second->writer->Disabled() ) { j->second->writer->Stop(); delete j->second; @@ -680,11 +677,11 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) Val* path_arg; if ( filter->path_val ) - path_arg = filter->path_val; + path_arg = filter->path_val->Ref(); else path_arg = new StringVal(""); - vl.append(path_arg->Ref()); + vl.append(path_arg); Val* rec_arg; BroType* rt = filter->path_func->FType()->Args()->FieldType("rec"); @@ -718,7 +715,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( ! filter->path_val ) { - Unref(path_arg); filter->path = v->AsString()->CheckString(); filter->path_val = v->Ref(); } @@ -740,7 +736,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) if ( w != stream->writers.end() ) // We know this writer already. - writer = w->second ? w->second->writer : 0; + writer = w->second->writer; else { @@ -753,64 +749,25 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) for ( int j = 0; j < filter->num_fields; ++j ) arg_fields[j] = new Field(*filter->fields[j]); - if ( filter->remote ) - remote_serializer->SendLogCreateWriter(stream->id, - filter->writer, - path, - filter->num_fields, - arg_fields); + writer = CreateWriter(stream->id, filter->writer, + path, filter->num_fields, + arg_fields, filter->local, filter->remote); - if ( filter->local ) + if ( ! writer ) { - writer = CreateWriter(stream->id, filter->writer, - path, filter->num_fields, - arg_fields); - - if ( ! writer ) - { - Unref(columns); - return false; - } + Unref(columns); + return false; } - else - { - // Insert a null pointer into the map to make - // sure we don't try creating it again. - stream->writers.insert(Stream::WriterMap::value_type( - Stream::WriterPathPair(filter->writer->AsEnum(), path), 0)); - for( int i = 0; i < filter->num_fields; ++i) - delete arg_fields[i]; - - delete [] arg_fields; - } } // Alright, can do the write now. - if ( filter->local || filter->remote ) - { - threading::Value** vals = RecordToFilterVals(stream, filter, columns); - - if ( filter->remote ) - remote_serializer->SendLogWrite(stream->id, - filter->writer, - path, - filter->num_fields, - vals); - - if ( filter->local ) - { - // Write takes ownership of vals. - assert(writer); - writer->Write(filter->num_fields, vals); - } - - else - DeleteVals(filter->num_fields, vals); - - } + threading::Value** vals = RecordToFilterVals(stream, filter, columns); + // Write takes ownership of vals. + assert(writer); + writer->Write(filter->num_fields, vals); #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", @@ -976,7 +933,7 @@ Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, } WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const Field* const* fields) + int num_fields, const Field* const* fields, bool local, bool remote) { Stream* stream = FindStream(id); @@ -987,12 +944,12 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, string path, Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); - if ( w != stream->writers.end() && w->second ) + if ( w != stream->writers.end() ) // If we already have a writer for this. That's fine, we just // return it. return w->second->writer; - WriterFrontend* writer_obj = new WriterFrontend(writer->AsEnum()); + WriterFrontend* writer_obj = new WriterFrontend(id, writer, local, remote); assert(writer_obj); writer_obj->Init(path, num_fields, fields); @@ -1089,8 +1046,7 @@ bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, return false; } - if ( w->second ) - w->second->writer->Write(num_fields, vals); + w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to path '%s' on stream '%s'", @@ -1111,9 +1067,6 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) { - if ( ! i->second ) - continue; - WriterFrontend* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); @@ -1134,10 +1087,7 @@ bool Manager::SetBuf(EnumVal* id, bool enabled) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->SetBuf(enabled); - } + i->second->writer->SetBuf(enabled); RemoveDisabledWriters(stream); @@ -1155,10 +1105,7 @@ bool Manager::Flush(EnumVal* id) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - { - if ( i->second ) - i->second->writer->Flush(); - } + i->second->writer->Flush(); RemoveDisabledWriters(stream); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index d931bfaef8..bf097c5e1a 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -159,7 +159,8 @@ protected: // Takes ownership of fields. WriterFrontend* CreateWriter(EnumVal* id, EnumVal* writer, string path, - int num_fields, const threading::Field* const* fields); + int num_fields, const threading::Field* const* fields, + bool local, bool remote); // Takes ownership of values.. bool Write(EnumVal* id, EnumVal* writer, string path, diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 02f1a188d8..26e8eaf22e 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -99,21 +99,36 @@ public: using namespace logging; -WriterFrontend::WriterFrontend(bro_int_t type) +WriterFrontend::WriterFrontend(EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote) { + stream = arg_stream; + writer = arg_writer; + Ref(stream); + Ref(writer); + disabled = initialized = false; buf = true; + local = arg_local; + remote = arg_remote; write_buffer = 0; write_buffer_pos = 0; ty_name = ""; - backend = log_mgr->CreateBackend(this, type); - assert(backend); - backend->Start(); + if ( local ) + { + backend = log_mgr->CreateBackend(this, writer->AsEnum()); + assert(backend); + backend->Start(); + } + + else + backend = 0; } WriterFrontend::~WriterFrontend() { + Unref(stream); + Unref(writer); } string WriterFrontend::Name() const @@ -128,7 +143,9 @@ void WriterFrontend::Stop() { FlushWriteBuffer(); SetDisable(); - backend->Stop(); + + if ( backend ) + backend->Stop(); } void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* const * arg_fields) @@ -144,7 +161,17 @@ void WriterFrontend::Init(string arg_path, int arg_num_fields, const Field* cons fields = arg_fields; initialized = true; - backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( backend ) + backend->SendIn(new InitMessage(backend, arg_path, arg_num_fields, arg_fields)); + + if ( remote ) + remote_serializer->SendLogCreateWriter(stream, + writer, + arg_path, + arg_num_fields, + arg_fields); + } void WriterFrontend::Write(int num_fields, Value** vals) @@ -152,6 +179,19 @@ void WriterFrontend::Write(int num_fields, Value** vals) if ( disabled ) return; + if ( remote ) + remote_serializer->SendLogWrite(stream, + writer, + path, + num_fields, + vals); + + if ( ! backend ) + { + DeleteVals(vals); + return; + } + if ( ! write_buffer ) { // Need new buffer. @@ -173,7 +213,8 @@ void WriterFrontend::FlushWriteBuffer() // Nothing to do. return; - backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); + if ( backend ) + backend->SendIn(new WriteMessage(backend, num_fields, write_buffer_pos, write_buffer)); // Clear buffer (no delete, we pass ownership to child thread.) write_buffer = 0; @@ -187,7 +228,8 @@ void WriterFrontend::SetBuf(bool enabled) buf = enabled; - backend->SendIn(new SetBufMessage(backend, enabled)); + if ( backend ) + backend->SendIn(new SetBufMessage(backend, enabled)); if ( ! buf ) // Make sure no longer buffer any still queued data. @@ -200,7 +242,9 @@ void WriterFrontend::Flush() return; FlushWriteBuffer(); - backend->SendIn(new FlushMessage(backend)); + + if ( backend ) + backend->SendIn(new FlushMessage(backend)); } void WriterFrontend::Rotate(string rotated_path, double open, double close, bool terminating) @@ -209,7 +253,9 @@ void WriterFrontend::Rotate(string rotated_path, double open, double close, bool return; FlushWriteBuffer(); - backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); + + if ( backend ) + backend->SendIn(new RotateMessage(backend, this, rotated_path, open, close, terminating)); } void WriterFrontend::Finish() @@ -218,7 +264,18 @@ void WriterFrontend::Finish() return; FlushWriteBuffer(); - backend->SendIn(new FinishMessage(backend)); + + if ( backend ) + backend->SendIn(new FinishMessage(backend)); + } + +void WriterFrontend::DeleteVals(Value** vals) + { + // Note this code is duplicated in Manager::DeleteVals(). + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + + delete [] vals; } diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 4386a15f64..3e05d17c9e 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -25,14 +25,21 @@ public: /** * Constructor. * - * type: The backend writer type, with the value corresponding to the + * stream: The logging stream. + * + * writer: The backend writer type, with the value corresponding to the * script-level \c Log::Writer enum (e.g., \a WRITER_ASCII). The * frontend will internally instantiate a WriterBackend of the * corresponding type. + * + * local: If true, the writer will instantiate a local backend. + * + * remote: If true, the writer will forward all data to remote + * clients. * * Frontends must only be instantiated by the main thread. */ - WriterFrontend(bro_int_t type); + WriterFrontend(EnumVal* stream, EnumVal* writer, bool local, bool remote); /** * Destructor. @@ -187,10 +194,17 @@ public: protected: friend class Manager; + void DeleteVals(threading::Value** vals); + + EnumVal* stream; + EnumVal* writer; + WriterBackend* backend; // The backend we have instanatiated. bool disabled; // True if disabled. bool initialized; // True if initialized. bool buf; // True if buffering is enabled (default). + bool local; // True if logging locally. + bool remote; // True if loggin remotely. string ty_name; // Name of the backend type. Set by the manager. string path; // The log path. diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 51c4f7a3bc..e590b13434 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -20,8 +20,8 @@ BasicThread::BasicThread() terminating = false; pthread = 0; - buf = 0; - buf_len = 1024; + buf_len = 2048; + buf = (char*) malloc(buf_len); name = Fmt("thread-%d", ++thread_counter); @@ -57,9 +57,6 @@ void BasicThread::SetOSName(const string& name) const char* BasicThread::Fmt(const char* format, ...) { - if ( ! buf ) - buf = (char*) malloc(buf_len); - va_list al; va_start(al, format); int n = safe_vsnprintf(buf, buf_len, format, al); @@ -67,13 +64,15 @@ const char* BasicThread::Fmt(const char* format, ...) if ( (unsigned int) n >= buf_len ) { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); + int tmp_len = n + 32; + char* tmp = (char*) malloc(tmp_len); // Is it portable to restart? va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); + n = safe_vsnprintf(tmp, tmp_len, format, al); va_end(al); + + free(tmp); } return buf; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 3f5145d7a9..db86caa26f 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -102,25 +102,25 @@ void Manager::Process() next_beat = 0; } - if ( ! t->HasOut() ) - continue; - - Message* msg = t->RetrieveOut(); - - if ( msg->Process() ) + while ( t->HasOut() ) { - //if ( network_time ) //&& network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init. + Message* msg = t->RetrieveOut(); - did_process = true; - } - else - { - string s = msg->Name() + " failed, terminating thread " + t->Name() + " (in ThreadManager)"; - reporter->Error("%s", s.c_str()); - t->Stop(); + if ( msg->Process() ) + { + //if ( network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init. + did_process = true; + } + + else + { + string s = msg->Name() + " failed, terminating thread"; + reporter->Error("%s", s.c_str()); + t->Stop(); } - delete msg; + delete msg; + } } // fprintf(stderr, "P %.6f %.6f do_beat=%d did_process=%d next_next=%.6f\n", network_time, timer_mgr->Time(), do_beat, (int)did_process, next_beat); diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index 5f77a1c9f8..490c89c057 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -281,7 +281,7 @@ void MsgThread::GetStats(Stats* stats) { stats->sent_in = cnt_sent_in; stats->sent_out = cnt_sent_out; - stats->pending_in = cnt_sent_in - queue_in.Size(); - stats->pending_out = cnt_sent_out - queue_out.Size(); + stats->pending_in = queue_in.Size(); + stats->pending_out = queue_out.Size(); } diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 78556e5271..1d7255d695 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -13,7 +13,7 @@ bool Field::Read(SerializationFormat* fmt) int st; bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && - fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + fmt->Read(&t, "type") && fmt->Read(&st, "subtype") && fmt->Read(&optional, "optional")); type = (TypeTag) t; subtype = (TypeTag) st; @@ -23,7 +23,7 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && - fmt->Write((int)subtype, "subtype")); + fmt->Write((int)subtype, "subtype"), fmt->Write(optional, "optional")); } Value::~Value() diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index ac34f3e476..bee84f2b54 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -24,17 +24,18 @@ struct Field { string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. + bool optional; //! needed by input framework. Is the field optional or does it have to be present in the input data /** * Constructor. */ - Field() { subtype = TYPE_VOID; } + Field() { subtype = TYPE_VOID; optional = false; } /** * Copy constructor. */ Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { } /** * Unserializes a field. diff --git a/src/types.bif b/src/types.bif index a9c6ecb3a8..26850bfa93 100644 --- a/src/types.bif +++ b/src/types.bif @@ -182,10 +182,6 @@ enum Event %{ EVENT_REMOVED, %} -enum ID %{ - Unknown, -%} - enum Mode %{ MANUAL = 0, REREAD = 1, diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log new file mode 100644 index 0000000000..42fcd6a526 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path metrics +#fields ts metric_id filter_name index.host index.str index.network value +#types time enum string addr string subnet count +1331256494.591966 TEST_METRIC foo-bar 6.5.4.3 - - 4 +1331256494.591966 TEST_METRIC foo-bar 7.2.1.5 - - 2 +1331256494.591966 TEST_METRIC foo-bar 1.2.3.4 - - 6 diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log new file mode 100644 index 0000000000..5a26f322f4 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.failure +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log new file mode 100644 index 0000000000..9d2ba26f48 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure US +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure UK +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log new file mode 100644 index 0000000000..1b2ed452a0 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log @@ -0,0 +1,9 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test.success +#fields t id.orig_h id.orig_p id.resp_h id.resp_p status country +#types time addr port addr port string string +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success unknown +1331256472.375609 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out new file mode 100644 index 0000000000..f75248cf97 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out @@ -0,0 +1,4 @@ +{ +[2] = [b=], +[1] = [b=T] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out index e32a2aea00..59070cd88e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -1,21 +1,70 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 1 T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 2 T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 3 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 4 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 5 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 6 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 7 T diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out new file mode 100644 index 0000000000..06e28de441 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out @@ -0,0 +1,145 @@ +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.optional/out b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out new file mode 100644 index 0000000000..7a304fc918 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out @@ -0,0 +1,9 @@ +{ +[2] = [b=T, notb=F], +[4] = [b=F, notb=T], +[6] = [b=F, notb=T], +[7] = [b=T, notb=F], +[1] = [b=T, notb=F], +[5] = [b=F, notb=T], +[3] = [b=F, notb=T] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out index 2059013c5d..34a5599dc9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -1,8 +1,64 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out new file mode 100644 index 0000000000..71de0d2570 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out @@ -0,0 +1,160 @@ +input0 +input.log +{ +[1] = T +} +input1 +input.log +{ +[1] = T +} +input2 +input.log +{ +[1] = T +} +input3 +input.log +{ +[1] = T +} +input4 +input.log +{ +[1] = T +} +input5 +input.log +{ +[1] = T +} +input6 +input.log +{ +[1] = T +} +input7 +input.log +{ +[1] = T +} +input8 +input.log +{ +[1] = T +} +input9 +input.log +{ +[1] = T +} +input10 +input.log +{ +[1] = T +} +input11 +input.log +{ +[1] = T +} +input12 +input.log +{ +[1] = T +} +input13 +input.log +{ +[1] = T +} +input14 +input.log +{ +[1] = T +} +input15 +input.log +{ +[1] = T +} +input16 +input.log +{ +[1] = T +} +input17 +input.log +{ +[1] = T +} +input18 +input.log +{ +[1] = T +} +input19 +input.log +{ +[1] = T +} +input20 +input.log +{ +[1] = T +} +input21 +input.log +{ +[1] = T +} +input22 +input.log +{ +[1] = T +} +input23 +input.log +{ +[1] = T +} +input24 +input.log +{ +[1] = T +} +input25 +input.log +{ +[1] = T +} +input26 +input.log +{ +[1] = T +} +input27 +input.log +{ +[1] = T +} +input28 +input.log +{ +[1] = T +} +input29 +input.log +{ +[1] = T +} +input30 +input.log +{ +[1] = T +} +input31 +input.log +{ +[1] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index f244f11a73..46a30f387f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -14,8 +14,44 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -59,8 +95,56 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-43] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -116,8 +200,56 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_CHANGED +Left [i=-43] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -172,9 +304,177 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============EVENT============ +============PREDICATE============ Input::EVENT_NEW +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left [i=-44] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -187,9 +487,117 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============PREDICATE============ +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-45] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -203,9 +611,81 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Input::EVENT_NEW -[i=-45] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -217,9 +697,30 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============PREDICATE============ +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-46] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -233,9 +734,81 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Input::EVENT_NEW -[i=-46] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -247,9 +820,30 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============PREDICATE============ +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-47] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -263,9 +857,81 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Input::EVENT_NEW -[i=-47] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -277,24 +943,30 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_NEW -[i=-48] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-48] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -485,8 +1157,11 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description Input::EVENT_REMOVED +Type [i=-43] +Left [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -499,9 +1174,13 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +Right ============EVENT============ +Description Input::EVENT_REMOVED +Type [i=-46] +Left [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -514,9 +1193,13 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +Right ============EVENT============ +Description Input::EVENT_REMOVED +Type [i=-44] +Left [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -529,9 +1212,13 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +Right ============EVENT============ +Description Input::EVENT_REMOVED +Type [i=-47] +Left [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -544,9 +1231,13 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +Right ============EVENT============ +Description Input::EVENT_REMOVED +Type [i=-45] +Left [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -559,9 +1250,13 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +Right ============EVENT============ +Description Input::EVENT_REMOVED +Type [i=-42] +Left [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -574,6 +1269,7 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] +Right ==========SERVERS============ { [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out new file mode 100644 index 0000000000..d85c8f2e83 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out @@ -0,0 +1,128 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out new file mode 100644 index 0000000000..937acf428e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out @@ -0,0 +1,120 @@ +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index 54048a86b8..56b36a1a0e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,21 +1,126 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=1] T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=2] T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=3] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=4] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=5] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=6] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=7] T diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out deleted file mode 100644 index 5b1ee5e983..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out +++ /dev/null @@ -1,15 +0,0 @@ -VALID -VALID -VALID -VALID -VALID -VALID -VALID -MARK -VALID -VALID -VALID -VALID -VALID -VALID -VALID diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out new file mode 100644 index 0000000000..a61a4a2993 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -0,0 +1,349 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE 2============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-42] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Description +[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh2, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE 2============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-43] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-42] +Left +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro new file mode 100644 index 0000000000..a82f52c8b2 --- /dev/null +++ b/testing/btest/core/leaks/basic-cluster.bro @@ -0,0 +1,39 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT +# @TEST-EXEC: btest-bg-run proxy-1 BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-run worker-2 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT +# @TEST-EXEC: btest-bg-wait -k 30 +# @TEST-EXEC: btest-diff manager-1/metrics.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +redef enum Metrics::ID += { + TEST_METRIC, +}; + +event bro_init() &priority=5 + { + Metrics::add_filter(TEST_METRIC, + [$name="foo-bar", + $break_interval=3secs]); + + if ( Cluster::local_node_type() == Cluster::WORKER ) + { + Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); + Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); + Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); + } + } diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro new file mode 100644 index 0000000000..fa72ce6024 --- /dev/null +++ b/testing/btest/core/leaks/remote.bro @@ -0,0 +1,79 @@ +# Needs perftools support. +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 10 +# @TEST-EXEC: btest-diff sender/test.log +# @TEST-EXEC: btest-diff sender/test.failure.log +# @TEST-EXEC: btest-diff sender/test.success.log +# @TEST-EXEC: cmp receiver/test.log sender/test.log +# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log +# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log + +# This is the common part loaded by both sender and receiver. +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { LOG }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(Test::LOG, [$columns=Log]); + Log::add_filter(Test::LOG, [$name="f1", $path="test.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); +} + +##### + +@TEST-START-FILE sender.bro + +module Test; + +@load frameworks/communication/listen + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event remote_connection_handshake_done(p: event_peer) + { + Log::add_filter(Test::LOG, [$name="f2", $path="test.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + + # Log something. + Log::write(Test::LOG, r); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + disconnect(p); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +redef Communication::nodes += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 156898edca..8d4028a12e 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -45,12 +41,10 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source:string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro new file mode 100644 index 0000000000..77659d13ec --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +##types bool int +T 1 +- 2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); +} + +event Input::update_finished(name: string, source:string) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 41eba1613c..dca75334d0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -19,16 +19,13 @@ module A; -export { - redef enum Input::ID += { INPUT }; -} - type Val: record { i: int; b: bool; }; -event line(tpe: Input::Event, i: int, b: bool) { +event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + print description; print tpe; print i; print b; @@ -36,6 +33,6 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro new file mode 100644 index 0000000000..6fceebf885 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="wc input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/executestreamraw.bro b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro new file mode 100644 index 0000000000..d97a7b26a0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro @@ -0,0 +1,58 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. + +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + try = try + 1; + + if ( try == 9 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index bcbba05a3e..d6c81cb2db 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -30,12 +26,11 @@ global servers: table[int] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 1c532ba6a9..ca1e956f35 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -30,12 +26,11 @@ global servers: table[int] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$name="input", $source="input.log", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro new file mode 100644 index 0000000000..c354f7c3ab --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -0,0 +1,45 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + notb: bool &optional; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 801d6bac3f..88e86eb5dc 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -13,10 +13,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: addr; }; @@ -29,17 +25,14 @@ global servers: table[addr] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]); print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; - Input::remove_tablefilter(A::INPUT, "input"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index f08aaef998..20c69131cb 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -23,10 +23,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -38,7 +34,7 @@ type Val: record { global servers: table[int] of Val = table(); global ct: int; -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { ct = ct + 1; if ( ct < 3 ) { return; @@ -75,9 +71,10 @@ event bro_init() { ct = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log", $mode=Input::STREAM]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, + Input::add_table([$source="input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); + Input::remove("input"); + } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 009911e6a8..278ac7418e 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -20,10 +20,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -37,13 +33,13 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 5f196648b6..8ec6c12a78 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -16,20 +16,18 @@ sdf module A; -export { - redef enum Input::ID += { INPUT }; -} - type Val: record { s: string; }; -event line(tpe: Input::Event, s: string) { +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; print s; } event bro_init() { - Input::create_stream(A::INPUT, [$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM]); - Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro new file mode 100644 index 0000000000..58ce9a1675 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -0,0 +1,41 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32}; + +event bro_init() +{ + for ( i in one_to_32 ) { + Input::add_table([$source="input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]); + Input::remove(fmt("input%d", i)); + } +} + +event Input::update_finished(name: string, source: string) { + print name; + print source; + print destination; +} diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 742d68605b..f33b060fe0 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -62,10 +62,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -94,10 +90,15 @@ global outfile: file; global try: count; -event line(tpe: Input::Event, left: Idx, right: Val) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; print outfile, tpe; + print outfile, "Left"; print outfile, left; + print outfile, "Right"; print outfile, right; } @@ -106,8 +107,7 @@ event bro_init() outfile = open ("../out"); try = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, $pred(typ: Input::Event, left: Idx, right: Val) = { print outfile, "============PREDICATE============"; print outfile, typ; @@ -119,7 +119,7 @@ event bro_init() } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print outfile, "==========SERVERS============"; print outfile, servers; @@ -127,7 +127,6 @@ event Input::update_finished(id: Input::ID) { if ( try == 5 ) { print outfile, "done"; close(outfile); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } } diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro new file mode 100644 index 0000000000..33361ad27e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]); + Input::force_update("input"); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index db368074aa..571a2273c1 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -28,10 +28,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -60,7 +56,7 @@ global outfile: file; global try: count; -event line(tpe: Input::Event, left: Idx, right: Val) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; print outfile, tpe; print outfile, left; @@ -73,8 +69,7 @@ event line(tpe: Input::Event, left: Idx, right: Val) { if ( try == 3 ) { print outfile, "done"; close(outfile); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } } @@ -83,7 +78,6 @@ event bro_init() outfile = open ("../out"); try = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::STREAM]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); + Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); } diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro new file mode 100644 index 0000000000..cc0afd5ae8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -0,0 +1,56 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 0c86ac94b8..e40485dd12 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -18,12 +18,6 @@ redef InputAscii::empty_field = "EMPTY"; -module A; - -export { - redef enum Log::ID += { LOG }; -} - type Idx: record { i: int; }; @@ -34,7 +28,8 @@ type Val: record { global destination: table[int] of Val = table(); -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { + print description; print tpe; print left; print right; @@ -42,6 +37,6 @@ event line(tpe: Input::Event, left: Idx, right: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro deleted file mode 100644 index 260f73e58f..0000000000 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ /dev/null @@ -1,104 +0,0 @@ -# -# @TEST-EXEC: bro %INPUT >out -# @TEST-EXEC: btest-diff out - -@TEST-START-FILE input.log -#separator \x09 -#path ssh -#fields i b -#types int bool -1 T -2 T -3 F -4 F -5 F -6 F -7 T -@TEST-END-FILE - -redef InputAscii::empty_field = "EMPTY"; - -module A; - -export { - redef enum Input::ID += { INPUT }; -} - -type Idx: record { - i: int; -}; - -type Val: record { - b: bool; -}; - -global destination1: table[int] of Val = table(); -global destination2: table[int] of Val = table(); - -global done: bool = F; - -event bro_init() -{ - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log", $autostart=F]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, - $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } - ]); - Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - - Input::force_update(A::INPUT); -} - -event Input::update_finished(id: Input::ID) { - if ( done == T ) { - return; - } - - done = T; - - if ( 1 in destination1 ) { - print "VALID"; - } - if ( 2 in destination1 ) { - print "VALID"; - } - if ( !(3 in destination1) ) { - print "VALID"; - } - if ( !(4 in destination1) ) { - print "VALID"; - } - if ( !(5 in destination1) ) { - print "VALID"; - } - if ( !(6 in destination1) ) { - print "VALID"; - } - if ( 7 in destination1 ) { - print "VALID"; - } - - print "MARK"; - - if ( 2 in destination2 ) { - print "VALID"; - } - if ( 2 in destination2 ) { - print "VALID"; - } - if ( 3 in destination2 ) { - print "VALID"; - } - if ( 4 in destination2 ) { - print "VALID"; - } - if ( 5 in destination2 ) { - print "VALID"; - } - if ( 6 in destination2 ) { - print "VALID"; - } - if ( 7 in destination2 ) { - print "VALID"; - } -} diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro new file mode 100644 index 0000000000..6f18e0e939 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -0,0 +1,113 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE 2============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); +} + + +event Input::update_finished(name: string, source: string) { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 5 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} diff --git a/testing/external/scripts/perftools-adapt-paths b/testing/external/scripts/perftools-adapt-paths index 2eda2477c7..cfecd39993 100755 --- a/testing/external/scripts/perftools-adapt-paths +++ b/testing/external/scripts/perftools-adapt-paths @@ -7,4 +7,4 @@ cat $1 | sed "s#bro *\"\./#../../../build/src/bro \".tmp/$TEST_NAME/#g" | sed 's/ *--gv//g' >$1.tmp && mv $1.tmp $1 -grep -q "No leaks found" $1 +grep -qv "detected leaks of" $1