From 6e6073ff4c4e01ab3171d3724dd6fe5f44df463e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 17 Oct 2011 14:19:17 -0700 Subject: [PATCH 001/123] it compiles (but doesn't do anything useful) --- src/CMakeLists.txt | 4 ++ src/Func.cc | 3 ++ src/InputMgr.cc | 90 +++++++++++++++++++++++++++++++++++++++++ src/InputMgr.h | 24 +++++++++++ src/InputReader.cc | 12 ++++++ src/InputReader.h | 29 +++++++++++++ src/InputReaderAscii.cc | 12 ++++++ src/InputReaderAscii.h | 20 +++++++++ src/NetVar.cc | 2 + src/input.bif | 16 ++++++++ src/main.cc | 4 ++ src/types.bif | 7 ++++ 12 files changed, 223 insertions(+) create mode 100644 src/InputMgr.cc create mode 100644 src/InputMgr.h create mode 100644 src/InputReader.cc create mode 100644 src/InputReader.h create mode 100644 src/InputReaderAscii.cc create mode 100644 src/InputReaderAscii.h create mode 100644 src/input.bif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4779e1557..1693bad4eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -141,6 +141,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif logging.bif + input.bif event.bif const.bif types.bif @@ -332,6 +333,9 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc + InputMgr.cc + InputReader.cc + InputReaderAscii.cc LogMgr.cc LogWriter.cc LogWriterAscii.cc diff --git a/src/Func.cc b/src/Func.cc index 65cb22b09d..829bc89238 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -523,11 +523,13 @@ void builtin_error(const char* msg, BroObj* arg) #include "bro.bif.func_h" #include "logging.bif.func_h" +#include "input.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" #include "logging.bif.func_def" +#include "input.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -542,6 +544,7 @@ void init_builtin_funcs() #include "bro.bif.func_init" #include "logging.bif.func_init" +#include "input.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/InputMgr.cc b/src/InputMgr.cc new file mode 100644 index 0000000000..cacc512bcf --- /dev/null +++ b/src/InputMgr.cc @@ -0,0 +1,90 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "InputMgr.h" +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" + + +#include "InputReader.h" + +#include "InputReaderAscii.h" + + +struct InputReaderDefinition { + bro_int_t type; // the type + const char *name; // descriptive name for error messages + bool (*init)(); // optional one-time inifializing function + InputReader* (*factory)(); // factory function for creating instances +}; + +InputReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, InputReaderAscii::Instantiate }, + + // End marker + { BifEnum::Input::READER_DEFAULT, "None", 0, (InputReader* (*)())0 } +}; + +InputMgr::InputMgr() +{ + DBG_LOG(DBG_LOGGING, "this has to happen"); +} + + +// create a new input reader object to be used at whomevers leisure lateron. +InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +{ + InputReaderDefinition* ir = input_readers; + exit(12); + + while ( true ) { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); + reporter->Error("unknown reader when creating reader"); + return 0; + } + + if ( ir->type != reader->AsEnum() ) { + // no, didn't find the right one... + ++ir; + continue; + } + + + // call init function of writer if presnt + if ( ir->init ) + { + if ( (*ir->init)() ) + { + //clear it to be not called again + ir->init = 0; + } else { + // ohok. init failed, kill factory for all eternity + ir->factory = 0; + DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + return 0; + } + + } + + if ( !ir->factory ) + // no factory? + return 0; + + // all done. break. + break; + } + + assert(ir->factory); + InputReader* reader_obj = (*ir->factory)(); + assert(reader_obj); + + return reader_obj; + +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h new file mode 100644 index 0000000000..978481afba --- /dev/null +++ b/src/InputMgr.h @@ -0,0 +1,24 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUTMGR_H +#define INPUTMGR_H + +#include "InputReader.h" +#include "BroString.h" + +#include "Val.h" +#include "EventHandler.h" +#include "RemoteSerializer.h" + + +class InputMgr { +public: + InputMgr(); + + InputReader* CreateReader(EnumVal* reader, string source); +}; + +extern InputMgr* input_mgr; + + +#endif /* INPUTMGR_H */ diff --git a/src/InputReader.cc b/src/InputReader.cc new file mode 100644 index 0000000000..f2b3b05801 --- /dev/null +++ b/src/InputReader.cc @@ -0,0 +1,12 @@ + +#include "InputReader.h" + +InputReader::InputReader() +{ + +} + +InputReader::~InputReader() +{ + +} \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h new file mode 100644 index 0000000000..ce8303383a --- /dev/null +++ b/src/InputReader.h @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Same notes about thread safety as in LogWriter.h apply. + + +#ifndef INPUTREADER_H +#define INPUTREADER_H + +class InputReader { +public: + InputReader(); + virtual ~InputReader(); + +protected: + // Methods that have to be overwritten by the individual readers + +private: + friend class InputMgr; + + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. + + bool disabled; + + bool Disabled() { return disabled; } +}; + + +#endif /* INPUTREADER_H */ diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc new file mode 100644 index 0000000000..97933c2a6e --- /dev/null +++ b/src/InputReaderAscii.cc @@ -0,0 +1,12 @@ + +#include "InputReaderAscii.h" +#include "DebugLogger.h" + +InputReaderAscii::InputReaderAscii() +{ + DBG_LOG(DBG_LOGGING, "input reader initialized"); +} + +InputReaderAscii::~InputReaderAscii() +{ +} \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h new file mode 100644 index 0000000000..c471639fd4 --- /dev/null +++ b/src/InputReaderAscii.h @@ -0,0 +1,20 @@ + +#ifndef INPUTREADERASCII_H +#define INPUTREADERASCII_H + +#include "InputReader.h" + +class InputReaderAscii : public InputReader { +public: + InputReaderAscii(); + ~InputReaderAscii(); + + static InputReader* Instantiate() { return new InputReaderAscii; } + +protected: + +private: +}; + + +#endif /* INPUTREADERASCII_H */ diff --git a/src/NetVar.cc b/src/NetVar.cc index 25e4f7a0bc..5a6ac96fba 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -255,6 +255,7 @@ StringVal* cmd_line_bpf_filter; #include "types.bif.netvar_def" #include "event.bif.netvar_def" #include "logging.bif.netvar_def" +#include "input.bif.netvar_def" #include "reporter.bif.netvar_def" void init_event_handlers() @@ -315,6 +316,7 @@ void init_net_var() #include "const.bif.netvar_init" #include "types.bif.netvar_init" #include "logging.bif.netvar_init" +#include "input.bif.netvar_init" #include "reporter.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); diff --git a/src/input.bif b/src/input.bif new file mode 100644 index 0000000000..edb6b4e9bb --- /dev/null +++ b/src/input.bif @@ -0,0 +1,16 @@ +# functions and types for the input framework + +module Input; + +%%{ +#include "InputMgr.h" +#include "NetVar.h" +%%} + +function Input::__create_reader%(reader: Input::Reader, source: string%) : bool + %{ + exit(5); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + return new Val( the_reader != 0, TYPE_BOOL ); + %} + diff --git a/src/main.cc b/src/main.cc index dfa46c3050..b3f2512b40 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "File.h" #include "Reporter.h" #include "LogMgr.h" +#include "InputMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -72,6 +73,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; LogMgr* log_mgr; +InputMgr* input_mgr; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -724,6 +726,8 @@ int main(int argc, char** argv) remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); log_mgr = new LogMgr(); + + input_mgr = new InputMgr(); if ( events_file ) event_player = new EventPlayer(events_file); diff --git a/src/types.bif b/src/types.bif index da6bd6e031..ee43207ddd 100644 --- a/src/types.bif +++ b/src/types.bif @@ -167,4 +167,11 @@ enum ID %{ Unknown, %} +module Input; + +enum Reader %{ + READER_DEFAULT, + READER_ASCII, +%} + module GLOBAL; From 0eafeb03693e3134bb47219df43066a390a42d55 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 13:16:09 -0700 Subject: [PATCH 002/123] works (thanks to robin) --- scripts/base/init-bare.bro | 2 ++ src/NetVar.h | 1 + src/input.bif | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 859a69f2dc..8d1d2a312f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1508,3 +1508,5 @@ const parse_udp_tunnels = F &redef; # Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging + +@load base/input.bif diff --git a/src/NetVar.h b/src/NetVar.h index f8def230c0..957a86aeb3 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,6 +264,7 @@ extern void init_net_var(); #include "types.bif.netvar_h" #include "event.bif.netvar_h" #include "logging.bif.netvar_h" +#include "input.bif.netvar_h" #include "reporter.bif.netvar_h" #endif diff --git a/src/input.bif b/src/input.bif index edb6b4e9bb..3ff5284c63 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,7 +9,6 @@ module Input; function Input::__create_reader%(reader: Input::Reader, source: string%) : bool %{ - exit(5); InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); return new Val( the_reader != 0, TYPE_BOOL ); %} From f8be3519c7b868cd23829141b9588396df50e6ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 15:41:07 -0700 Subject: [PATCH 003/123] well, it compiles. and perhaps it sends an event. billiant. --- src/InputMgr.cc | 20 +++++++++++++++++--- src/InputMgr.h | 10 +++++++++- src/InputReader.cc | 19 +++++++++++++++++++ src/InputReader.h | 15 ++++++++++++++- src/input.bif | 6 ++++-- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index cacc512bcf..5ab602f4f3 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,17 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) { InputReaderDefinition* ir = input_readers; - exit(12); - + + RecordType* rtype = eventDescription->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + { + reporter->Error("eventDescription argument not of right type"); + return 0; + } + while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { @@ -86,5 +92,13 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source) return reader_obj; } + +void InputMgr::Error(InputReader* reader, const char* msg) +{ + reporter->Error(fmt("error with input reader for %s: %s", + reader->Source().c_str(), msg)); +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 978481afba..3d3d0f2d3f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -10,12 +10,20 @@ #include "EventHandler.h" #include "RemoteSerializer.h" +class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source); + InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + +protected: + friend class InputReader; + + // Reports an error for the given reader. + void Error(InputReader* reader, const char* msg); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index f2b3b05801..ef47bb1e10 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,5 +1,7 @@ #include "InputReader.h" +#include "EventRegistry.h" +#include "Event.h" InputReader::InputReader() { @@ -9,4 +11,21 @@ InputReader::InputReader() InputReader::~InputReader() { +} + +void InputReader::Error(const char *msg) +{ + input_mgr->Error(this, msg); +} + +bool InputReader::Init(string source, string eventName) { + EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + if ( handler == 0 ) { + reporter->Error("Event %s not found", eventName.c_str()); + return false; + } + + mgr.Dispatch(new Event(handler, 0)); + return true; } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index ce8303383a..58a56e8221 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -6,16 +6,29 @@ #ifndef INPUTREADER_H #define INPUTREADER_H +#include "InputMgr.h" +#include "BroString.h" + class InputReader { public: InputReader(); virtual ~InputReader(); + + bool Init(string source, string eventName); protected: // Methods that have to be overwritten by the individual readers - + + // Reports an error to the user. + void Error(const char *msg); + + // The following methods return the information as passed to Init(). + const string Source() const { return source; } + private: friend class InputMgr; + + string source; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/input.bif b/src/input.bif index 3ff5284c63..e2fc2ae91e 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,9 +7,11 @@ module Input; #include "NetVar.h" %%} -function Input::__create_reader%(reader: Input::Reader, source: string%) : bool +type Event: record; + +function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 9c8b0dec3b8622a592aa1c787d9bde7d53f45f5f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 16:35:34 -0700 Subject: [PATCH 004/123] event from c++ to script works (at last...) --- scripts/base/frameworks/input/__load__.bro | 1 + scripts/base/frameworks/input/main.bro | 11 +++++++++++ scripts/base/init-bare.bro | 3 ++- src/InputMgr.cc | 7 ++++--- src/InputMgr.h | 2 +- src/InputReader.cc | 19 +++++++++++-------- src/LogMgr.cc | 2 +- src/input.bif | 4 ++-- 8 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 scripts/base/frameworks/input/__load__.bro create mode 100644 scripts/base/frameworks/input/main.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/input/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro new file mode 100644 index 0000000000..cb071872ac --- /dev/null +++ b/scripts/base/frameworks/input/main.bro @@ -0,0 +1,11 @@ + +module Input; + +export { + type Event: record { + name: string; + columns: any; + }; +} + +@load base/input.bif diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8d1d2a312f..ade92bfd6e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1509,4 +1509,5 @@ const parse_udp_tunnels = F &redef; # BiFs and script-land defined types. @load base/frameworks/logging -@load base/input.bif +@load base/frameworks/input + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5ab602f4f3..f3fb2ff935 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,11 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = eventDescription->Type()->AsRecordType(); + RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) { reporter->Error("eventDescription argument not of right type"); @@ -49,7 +49,6 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); reporter->Error("unknown reader when creating reader"); return 0; } @@ -89,6 +88,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); + reader_obj->Init(source, eventName); + return reader_obj; } diff --git a/src/InputMgr.h b/src/InputMgr.h index 3d3d0f2d3f..e98f3d77b7 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -16,7 +16,7 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); protected: friend class InputReader; diff --git a/src/InputReader.cc b/src/InputReader.cc index ef47bb1e10..fc9be7f2b6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,7 +1,7 @@ #include "InputReader.h" -#include "EventRegistry.h" -#include "Event.h" +// #include "EventRegistry.h" +// #include "Event.h" InputReader::InputReader() { @@ -19,13 +19,16 @@ void InputReader::Error(const char *msg) } bool InputReader::Init(string source, string eventName) { - EventHandler* handler = event_registry->Lookup(eventName.c_str()); + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} - if ( handler == 0 ) { - reporter->Error("Event %s not found", eventName.c_str()); - return false; - } + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); - mgr.Dispatch(new Event(handler, 0)); + //mgr.Dispatch(new Event(handler, vl)); return true; } \ No newline at end of file diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9e320f8810..fc4b89b0ed 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool LogMgr::(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index e2fc2ae91e..88d4e32129 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,9 @@ module Input; type Event: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 365406024627c8c2b0b3d41263840a7fbaf029ed Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 21 Oct 2011 14:01:18 -0700 Subject: [PATCH 005/123] compiles. sill doesn't do much. --- scripts/base/frameworks/input/main.bro | 8 ++++-- src/InputMgr.cc | 35 ++++++++++++++++++++---- src/InputMgr.h | 8 +++++- src/InputReader.cc | 29 ++++++++++---------- src/InputReader.h | 10 ++++++- src/InputReaderAscii.cc | 38 +++++++++++++++++++++++++- src/InputReaderAscii.h | 10 +++++++ src/LogMgr.cc | 2 +- src/input.bif | 6 ++-- 9 files changed, 117 insertions(+), 29 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index cb071872ac..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,9 +2,11 @@ module Input; export { - type Event: record { - name: string; - columns: any; + type ReaderDescription: record { + source: string; + idx: any; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f3fb2ff935..500a325249 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,14 +35,14 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) +InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) { - reporter->Error("eventDescription argument not of right type"); + reporter->Error("readerDescription argument not of right type"); return 0; } @@ -88,7 +88,11 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* e InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); - reader_obj->Init(source, eventName); + // get the source... + const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + string source((const char*) bsource->Bytes(), bsource->Len()); + + reader_obj->Init(source, 0, NULL); return reader_obj; @@ -100,6 +104,27 @@ void InputMgr::Error(InputReader* reader, const char* msg) reader->Source().c_str(), msg)); } +/* + TODO: + +void InputMgr::SendEvent(string name) { + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} + + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); + + //mgr.Dispatch(new Event(handler, vl)); + + +} + +*/ + diff --git a/src/InputMgr.h b/src/InputMgr.h index e98f3d77b7..255d61fe5e 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -9,20 +9,26 @@ #include "Val.h" #include "EventHandler.h" #include "RemoteSerializer.h" +#include "LogMgr.h" // for the LogVal and LogType data types class InputReader; + class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); + InputReader* CreateReader(EnumVal* reader, RecordVal* description); protected: friend class InputReader; // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + +private: + // required functionality + // InputValsToRecord to convert received inputvals back to bro records / tables / whatever }; diff --git a/src/InputReader.cc b/src/InputReader.cc index fc9be7f2b6..4d29040e60 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,7 +5,7 @@ InputReader::InputReader() { - + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -18,17 +18,18 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } -bool InputReader::Init(string source, string eventName) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); - return true; +bool InputReader::Init(string arg_source, int arg_num_fields, + const LogField* const * arg_fields) +{ + source = arg_source; + num_fields = arg_num_fields; + fields = arg_fields; + + // disable if DoInit returns error. + disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + return !disabled; +} + +void InputReader::Finish() { + DoFinish(); } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index 58a56e8221..f3638c7246 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -8,16 +8,22 @@ #include "InputMgr.h" #include "BroString.h" +#include "LogMgr.h" class InputReader { public: InputReader(); virtual ~InputReader(); - bool Init(string source, string eventName); + bool Init(string arg_source, int num_fields, const LogField* const* fields); + void Finish(); + protected: // Methods that have to be overwritten by the individual readers + virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + + virtual void DoFinish() = 0; // Reports an error to the user. void Error(const char *msg); @@ -29,6 +35,8 @@ private: friend class InputMgr; string source; + int num_fields; + const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 97933c2a6e..b9aab16815 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,11 +2,47 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include + InputReaderAscii::InputReaderAscii() { - DBG_LOG(DBG_LOGGING, "input reader initialized"); + //DBG_LOG(DBG_LOGGING, "input reader initialized"); + file = 0; } InputReaderAscii::~InputReaderAscii() { +} + +void InputReaderAscii::DoFinish() +{ +} + +bool InputReaderAscii::DoInit(string path, int num_fields, + const LogField* const * fields) +{ + fname = path; + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + return false; + } + + // try to read the header line... + string line; + if ( !getline(*file, line) ) + return false; + + // split on tabs... + istringstream ss(line); + while ( ss ) { + string s; + if ( !getline(ss, s, '\t')) + break; + + + } + + + return false; } \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c471639fd4..0d2008ed7f 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -3,6 +3,9 @@ #define INPUTREADERASCII_H #include "InputReader.h" +#include +#include + class InputReaderAscii : public InputReader { public: @@ -12,8 +15,15 @@ public: static InputReader* Instantiate() { return new InputReaderAscii; } protected: + + virtual bool DoInit(string path, int num_fields, + const LogField* const * fields); + virtual void DoFinish(); private: + + ifstream* file; + string fname; }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index fc4b89b0ed..9e320f8810 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::(EnumVal* id, RecordVal* sval) +bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index 88d4e32129..3da869ea08 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,11 +7,11 @@ module Input; #include "NetVar.h" %%} -type Event: record; +type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From d7a3b85fcda42a1485ef7a53c5419100e4b5ea95 Mon Sep 17 00:00:00 2001 From: amannb Date: Tue, 25 Oct 2011 11:47:23 -0700 Subject: [PATCH 006/123] many helper functions --- src/InputMgr.cc | 71 +++++++++++++++------ src/InputMgr.h | 4 +- src/InputReader.cc | 33 +++++++++- src/InputReader.h | 12 ++++ src/InputReaderAscii.cc | 134 +++++++++++++++++++++++++++++++++++++--- src/InputReaderAscii.h | 22 +++++++ 6 files changed, 245 insertions(+), 31 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 500a325249..ec46c55813 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -100,31 +100,62 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) void InputMgr::Error(InputReader* reader, const char* msg) { - reporter->Error(fmt("error with input reader for %s: %s", - reader->Source().c_str(), msg)); + reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* - TODO: -void InputMgr::SendEvent(string name) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); +void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + val_list* vl = new val_list; + for ( int i = 0; i < num_vals; i++) { + vl->append(LogValToVal(vals[i])); + } + + mgr.Dispatch(new Event(handler, vl)); +} + +Val* InputMgr::LogValToVal(const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + return new Val(val->val.int_val, val->type); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + return new Val(val->val.uint_val, val->type); + break; + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + break; + + case TYPE_STRING: + { + BroString *s = new BroString(*(val->val.string_val)); + return new StringVal(s); + break; + } + + case TYPE_PORT: + return new PortVal(val->val.uint_val); + break; + + default: + reporter->InternalError("unsupported type for input_read"); + } + + + reporter->InternalError("Impossible error"); + return NULL; } - -*/ - - diff --git a/src/InputMgr.h b/src/InputMgr.h index 255d61fe5e..79f76a1e6f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -13,7 +13,6 @@ class InputReader; - class InputMgr { public: InputMgr(); @@ -29,6 +28,9 @@ protected: private: // required functionality // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + Val* LogValToVal(const LogVal* val); + + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 4d29040e60..d812349733 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -32,4 +32,35 @@ bool InputReader::Init(string arg_source, int arg_num_fields, void InputReader::Finish() { DoFinish(); -} \ No newline at end of file +} + +bool InputReader::Update() { + return DoUpdate(); +} + +// stolen from logwriter +const char* InputReader::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + + diff --git a/src/InputReader.h b/src/InputReader.h index f3638c7246..9d776276fa 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -18,12 +18,17 @@ public: bool Init(string arg_source, int num_fields, const LogField* const* fields); void Finish(); + + bool Update(); protected: // Methods that have to be overwritten by the individual readers virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; + + // update file contents to logmgr + virtual bool DoUpdate() = 0; // Reports an error to the user. void Error(const char *msg); @@ -31,6 +36,9 @@ protected: // The following methods return the information as passed to Init(). const string Source() const { return source; } + // A thread-safe version of fmt(). (stolen from logwriter) + const char* Fmt(const char* format, ...); + private: friend class InputMgr; @@ -44,6 +52,10 @@ private: bool disabled; bool Disabled() { return disabled; } + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; }; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index b9aab16815..4bc4b81cda 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -1,9 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReaderAscii.h" #include "DebugLogger.h" #include +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) + : name(arg_name), type(arg_type) +{ + position = arg_position; +} + +FieldMapping::FieldMapping(const FieldMapping& arg) + : name(arg.name), type(arg.type) +{ + position = arg.position; +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -18,31 +31,134 @@ void InputReaderAscii::DoFinish() { } -bool InputReaderAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) { fname = path; file = new ifstream(path.c_str()); if ( !file->is_open() ) { + Error(Fmt("cannot open %s", path.c_str())); return false; } // try to read the header line... string line; - if ( !getline(*file, line) ) + if ( !getline(*file, line) ) { + Error("could not read first line"); return false; + } // split on tabs... - istringstream ss(line); - while ( ss ) { + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { string s; - if ( !getline(ss, s, '\t')) + if ( !getline(splitstream, s, '\t')) break; - + // current found heading in s... compare if we want it + for ( int i = 0; i < num_fields; i++ ) { + const LogField* field = fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, i); + columnMap.push_back(f); + wantFields++; + break; // done with searching + } + } + + // look if we did push something... + if ( columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + columnMap.push_back(empty); + } + + // done + currTab++; } + + if ( wantFields != num_fields ) { + // we did not find all fields? + // :( + Error("wantFields != num_fields"); + return false; + } + + this->num_fields = num_fields; - return false; -} \ No newline at end of file + // well, that seems to have worked... + return true; +} + +// read the entire file and send appropriate thingies back to InputMgr +bool InputReaderAscii::DoUpdate() { + // TODO: all the stuff we need for a second reading. + // *cough* + // + + + string line; + while ( getline(*file, line ) ) { + // split on tabs + + istringstream splitstream(line); + string s; + + LogVal fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) + break; + + + if ( currTab >= columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal val(currMapping.type, true); + + switch ( currMapping.type ) { + case TYPE_STRING: + val.val.string_val = new string(s); + + default: + Error(Fmt("unsupported field format %d for %s", currMapping.type, + currMapping.name.c_str())); + return false; + } + + currField++; + } + + if ( currField != num_fields ) { + Error("curr_field != num_fields in DoUpdate"); + return false; + } + + // ok, now we have built our line. send it back to... whomever. + + } + + return true; +} diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 0d2008ed7f..551a08b02e 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #ifndef INPUTREADERASCII_H #define INPUTREADERASCII_H @@ -5,6 +6,19 @@ #include "InputReader.h" #include #include +#include + +// Description for input field mapping +struct FieldMapping { + string name; + TypeTag type; + int position; + + FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const FieldMapping& arg); + FieldMapping() { position = -1; } + bool IsEmpty() { return position == -1; } +}; class InputReaderAscii : public InputReader { @@ -19,11 +33,19 @@ protected: virtual bool DoInit(string path, int num_fields, const LogField* const * fields); virtual void DoFinish(); + + virtual bool DoUpdate(); private: ifstream* file; string fname; + + unsigned int num_fields; + + // map columns in the file to columns to send back to the manager + vector columnMap; + }; From 5b0c307f87a7213951e586f6e881501dc658f03d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 25 Oct 2011 14:11:21 -0700 Subject: [PATCH 007/123] very basic input to event working... --- scripts/base/frameworks/input/main.bro | 4 ++-- src/InputMgr.cc | 18 ++++++++++++++++-- src/InputReader.cc | 11 +++++++++++ src/InputReader.h | 3 +++ src/InputReaderAscii.cc | 14 ++++++++++---- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bea9d73d2 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any; - destination: any; + val: any &optional; + destination: any &optional; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ec46c55813..5cd82b613f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -91,8 +91,22 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) // get the source... const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - - reader_obj->Init(source, 0, NULL); + + RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + LogField** fields = new LogField*[idx->NumFields()]; + for ( int i = 0; i < idx->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = idx->FieldName(i); + field->type = idx->FieldType(i)->Tag(); + fields[i] = field; + } + + + reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Update(); return reader_obj; diff --git a/src/InputReader.cc b/src/InputReader.cc index d812349733..d512bc1699 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,6 +5,8 @@ InputReader::InputReader() { + buf = 0; + buf_len = 1024; disabled = true; // disabled will be set correcty in init. } @@ -18,6 +20,11 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } +void InputReader::Error(const string &msg) + { + input_mgr->Error(this, msg.c_str()); + } + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { @@ -38,6 +45,10 @@ bool InputReader::Update() { return DoUpdate(); } +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { + input_mgr->SendEvent(name, num_vals, vals); +} + // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) { diff --git a/src/InputReader.h b/src/InputReader.h index 9d776276fa..3bfb0adf91 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -31,6 +31,7 @@ protected: virtual bool DoUpdate() = 0; // Reports an error to the user. + void Error(const string &msg); void Error(const char *msg); // The following methods return the information as passed to Init(). @@ -39,6 +40,8 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4bc4b81cda..5d6b23416a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -37,7 +37,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const file = new ifstream(path.c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", path.c_str())); + Error(Fmt("cannot open %s", fname.c_str())); return false; } @@ -108,7 +108,7 @@ bool InputReaderAscii::DoUpdate() { istringstream splitstream(line); string s; - LogVal fields[num_fields]; + LogVal** fields = new LogVal*[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; @@ -136,11 +136,12 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal val(currMapping.type, true); + LogVal* val = new LogVal(currMapping.type, true); switch ( currMapping.type ) { case TYPE_STRING: - val.val.string_val = new string(s); + val->val.string_val = new string(s); + break; default: Error(Fmt("unsupported field format %d for %s", currMapping.type, @@ -148,6 +149,8 @@ bool InputReaderAscii::DoUpdate() { return false; } + fields[currField] = val; + currField++; } @@ -157,6 +160,9 @@ bool InputReaderAscii::DoUpdate() { } // ok, now we have built our line. send it back to... whomever. + // for testing purposes: fixed event. + + SendEvent("inputEvent", num_fields, fields); } From b245d4168a6c9d9ce9243a34f7fcc704d526a7d9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:02:57 -0700 Subject: [PATCH 008/123] yay, basic table assignment. --- scripts/base/frameworks/input/main.bro | 4 +- src/InputMgr.cc | 60 ++++++++++++++++++++++++-- src/InputMgr.h | 8 +++- src/InputReader.cc | 5 +++ src/InputReader.h | 2 + src/InputReaderAscii.cc | 16 +++++++ 6 files changed, 87 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bea9d73d2..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any &optional; - destination: any &optional; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5cd82b613f..5c30922863 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,16 @@ #include "InputReaderAscii.h" +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + unsigned int num_idx_fields; + unsigned int num_val_fields; + + TableVal* tab; + + }; struct InputReaderDefinition { bro_int_t type; // the type @@ -30,7 +40,7 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - DBG_LOG(DBG_LOGGING, "this has to happen"); + //DBG_LOG(DBG_LOGGING, "this has to happen"); } @@ -93,8 +103,10 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) string source((const char*) bsource->Bytes(), bsource->Len()); RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields()]; + LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; for ( int i = 0; i < idx->NumFields(); i++ ) { // FIXME: do type checking... @@ -103,15 +115,43 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) field->type = idx->FieldType(i)->Tag(); fields[i] = field; } + for ( int i = 0; i < val->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = val->FieldName(i); + field->type = val->FieldType(i)->Tag(); + fields[idx->NumFields() + i] = field; + } + + ReaderInfo* info = new ReaderInfo; + info->reader = reader_obj; + info->type = reader; + info->num_idx_fields = idx->NumFields(); + info->num_val_fields = val->NumFields(); + info->tab = dst; + readers.push_back(info); - reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); reader_obj->Update(); return reader_obj; } +void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); + reporter->Error("assigned"); +} + + void InputMgr::Error(InputReader* reader, const char* msg) { reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); @@ -172,4 +212,16 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return NULL; } - +InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->reader == reader ) + { + return *s; + } + } + + return 0; + } + diff --git a/src/InputMgr.h b/src/InputMgr.h index 79f76a1e6f..136be2d608 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,14 +24,18 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + + void Put(const InputReader* reader, const LogVal* const *vals); private: - // required functionality - // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + struct ReaderInfo; + Val* LogValToVal(const LogVal* val); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); + vector readers; }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index d512bc1699..6502fdb421 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -25,6 +25,11 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } +void InputReader::Put(const LogVal* const *val) +{ + input_mgr->Put(this, val); +} + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3bfb0adf91..3725c3d461 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -42,6 +42,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + void Put(const LogVal* const *val); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 5d6b23416a..d0d4a3014c 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -143,6 +143,21 @@ bool InputReaderAscii::DoUpdate() { val->val.string_val = new string(s); break; + case TYPE_BOOL: + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + val->val.uint_val = atoi(s.c_str()); + break; + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); @@ -163,6 +178,7 @@ bool InputReaderAscii::DoUpdate() { // for testing purposes: fixed event. SendEvent("inputEvent", num_fields, fields); + Put(fields); } From 86730c13dda7df06fa96b151c17ec437aa742b4f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:46:43 -0700 Subject: [PATCH 009/123] more complex types... --- scripts/base/frameworks/input/main.bro | 3 + src/InputMgr.cc | 314 +++++++++++++++++++++++-- src/InputMgr.h | 17 +- src/InputReader.cc | 11 + src/InputReader.h | 2 + src/InputReaderAscii.cc | 35 ++- src/input.bif | 17 +- src/types.bif | 4 + 8 files changed, 372 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bb7129d03 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,11 +2,14 @@ module Input; export { + const default_reader = READER_ASCII &redef; + type ReaderDescription: record { source: string; idx: any; val: any; destination: any; + reader: Reader &default=default_reader; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5c30922863..648a933a22 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -21,6 +21,8 @@ struct InputMgr::ReaderInfo { unsigned int num_val_fields; TableVal* tab; + RecordType* rtype; + RecordType* itype; }; @@ -45,7 +47,7 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) +InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; @@ -55,6 +57,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) reporter->Error("readerDescription argument not of right type"); return 0; } + + EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -106,38 +110,191 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; - for ( int i = 0; i < idx->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = idx->FieldName(i); - field->type = idx->FieldType(i)->Tag(); - fields[i] = field; + + vector fieldsV; // vector, because we don't know the length beforehands + + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return 0; } - for ( int i = 0; i < val->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = val->FieldName(i); - field->type = val->FieldType(i)->Tag(); - fields[idx->NumFields() + i] = field; + + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader; - info->num_idx_fields = idx->NumFields(); - info->num_val_fields = val->NumFields(); + Ref(reader); + info->num_idx_fields = idxfields; + info->num_val_fields = valfields; info->tab = dst; + Ref(dst); + info->rtype = val; + Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. + info->id = id; + Ref(id); // ditto... + info->itype = idx; + Ref(idx); readers.push_back(info); - reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); + reader_obj->Init(source, fieldsV.size(), fields); reader_obj->Update(); return reader_obj; +} +bool InputMgr::IsCompatibleType(BroType* t) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_RECORD: + // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. + // ^ recursive checking is done in UnrollRecordType. + return true; + + case TYPE_FILE: + case TYPE_FUNC: + return false; + + + case TYPE_TABLE: + return false; + + case TYPE_VECTOR: + { + return IsCompatibleType(t->AsVectorType()->YieldType()); + } + + default: + return false; + } + + return false; + } + + +bool InputMgr::RemoveReader(EnumVal* id) { + ReaderInfo *i = 0; + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } + } + + if ( i == 0 ) { + return false; // not found + } + + Unref(i->type); + Unref(i->tab); + Unref(i->itype); + Unref(i->rtype); + Unref(i->id); + + delete(i->reader); + delete(i); + + return true; +} + +bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( !IsCompatibleType(rec->FieldType(i)) ) { + reporter->Error("Incompatible type \"%s\" in table definition for InputReader", type_name(rec->FieldType(i)->Tag())); + return false; + } + + if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) + { + + string prep = nameprepend + rec->FieldName(i) + "."; + + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + { + return false; + } + + } else { + LogField* field = new LogField(); + field->name = nameprepend + rec->FieldName(i); + field->type = rec->FieldType(i)->Tag(); + + fields->push_back(field); + } + } + + return true; + +} + +bool InputMgr::ForceUpdate(EnumVal* id) +{ + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + i->reader->Update(); + return true; +} + +Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { + Val* idxval; + int position = 0; + + + if ( num_fields == 1 ) { + idxval = LogValToVal(vals[0]); + } else { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < type->NumFields(); j++ ) { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { + l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); + } else { + l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + position++; + } + } + idxval = l; + } + + assert ( position == num_fields ); + + return idxval; + } void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { @@ -147,10 +304,65 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { return; } - i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); - reporter->Error("assigned"); + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + i->tab->Assign(idxval, valval); } +void InputMgr::Clear(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->RemoveAll(); +} + +bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return false; + } + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + + return ( i->tab->Delete(idxval) != 0 ); +} void InputMgr::Error(InputReader* reader, const char* msg) { @@ -174,7 +386,46 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } -Val* InputMgr::LogValToVal(const LogVal* val) { + +Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + if ( position == 0 ) { + reporter->InternalError("Need position"); + return 0; + } + + /* + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("I only work with records"); + return 0; + } */ + + + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + for ( int i = 0; i < request_type->NumFields(); i++ ) { + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; + +} + +Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { + + if ( request_type != TYPE_ANY && request_type != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + return 0; + } + + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: @@ -203,6 +454,10 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return new PortVal(val->val.uint_val); break; + case TYPE_ADDR: + return new AddrVal(val->val.addr_val); + break; + default: reporter->InternalError("unsupported type for input_read"); } @@ -225,3 +480,18 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) return 0; } + +InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + return *s; + } + } + + return 0; + } + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 136be2d608..d5f732935c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -11,13 +11,17 @@ #include "RemoteSerializer.h" #include "LogMgr.h" // for the LogVal and LogType data types +#include + class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, RecordVal* description); + InputReader* CreateReader(EnumVal* id, RecordVal* description); + bool ForceUpdate(EnumVal* id); + bool RemoveReader(EnumVal* id); protected: friend class InputReader; @@ -26,14 +30,23 @@ protected: void Error(InputReader* reader, const char* msg); void Put(const InputReader* reader, const LogVal* const *vals); + void Clear(const InputReader* reader); + bool Delete(const InputReader* reader, const LogVal* const *vals); private: struct ReaderInfo; - Val* LogValToVal(const LogVal* val); + bool IsCompatibleType(BroType* t); + + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); + Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); + ReaderInfo* FindReader(const EnumVal* id); vector readers; }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 6502fdb421..1facc57c7f 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -30,6 +30,17 @@ void InputReader::Put(const LogVal* const *val) input_mgr->Put(this, val); } +void InputReader::Clear() +{ + input_mgr->Clear(this); +} + +void InputReader::Delete(const LogVal* const *val) +{ + input_mgr->Delete(this, val); +} + + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3725c3d461..0e93344e1a 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -43,6 +43,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); + void Clear(); + void Delete(const LogVal* const *val); private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index d0d4a3014c..e4d581c0d8 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -144,6 +144,13 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else { + val->val.int_val = 0; + } + break; + case TYPE_INT: val->val.int_val = atoi(s.c_str()); break; @@ -155,16 +162,37 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // fallthrough + } + case TYPE_ADDR: { + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); return false; } - fields[currField] = val; + fields[currMapping.position] = val; currField++; } @@ -174,10 +202,7 @@ bool InputReaderAscii::DoUpdate() { return false; } - // ok, now we have built our line. send it back to... whomever. - // for testing purposes: fixed event. - - SendEvent("inputEvent", num_fields, fields); + // ok, now we have built our line. send it back to the input manager Put(fields); } diff --git a/src/input.bif b/src/input.bif index 3da869ea08..4e2cbf07b5 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,22 @@ module Input; type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} + +function Input::__force_update%(id: ID%) : bool + %{ + bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_reader%(id: ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + diff --git a/src/types.bif b/src/types.bif index ee43207ddd..7b81a7f631 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,4 +174,8 @@ enum Reader %{ READER_ASCII, %} +enum ID %{ + Unknown, +%} + module GLOBAL; From f20125d22de20a7147026602f83fa1ed710700ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 28 Oct 2011 13:11:47 -0700 Subject: [PATCH 010/123] little snag with hashing functionality... --- src/InputMgr.cc | 232 +++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 11 +- src/InputReader.cc | 13 ++- src/InputReader.h | 9 +- src/InputReaderAscii.cc | 80 ++++++++++++-- src/InputReaderAscii.h | 5 +- src/Val.h | 5 +- 7 files changed, 333 insertions(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 648a933a22..5270c83d5a 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,18 @@ #include "InputReaderAscii.h" +#include "CompHash.h" + + +class InputHash { +public: + HashKey* valhash; + HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. +}; + +declare(PDict, InputHash); + + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -24,6 +36,9 @@ struct InputMgr::ReaderInfo { RecordType* rtype; RecordType* itype; + PDict(InputHash)* currDict; + PDict(InputHash)* lastDict; + }; struct InputReaderDefinition { @@ -147,10 +162,20 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx; Ref(idx); readers.push_back(info); + info->currDict = new PDict(InputHash); + info->lastDict = new PDict(InputHash); - reader_obj->Init(source, fieldsV.size(), fields); - reader_obj->Update(); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + if ( success == false ) { + RemoveReader(id); + return 0; + } + success = reader_obj->Update(); + if ( success == false ) { + RemoveReader(id); + return 0; + } return reader_obj; @@ -198,7 +223,6 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } - bool InputMgr::RemoveReader(EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) @@ -215,6 +239,9 @@ bool InputMgr::RemoveReader(EnumVal* id) { return false; // not found } + i->reader->Finish(); + + Unref(i->type); Unref(i->tab); Unref(i->itype); @@ -267,8 +294,7 @@ bool InputMgr::ForceUpdate(EnumVal* id) return false; } - i->reader->Update(); - return true; + return i->reader->Update(); } Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { @@ -297,6 +323,105 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } + +void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + + InputHash *h = i->lastDict->Lookup(idxhash); + if ( h != 0 ) { + // seen before + if ( h->valhash->Hash() == valhash->Hash() ) { + // ok, double. + i->lastDict->Remove(idxhash); + i->currDict->Insert(idxhash, h); + return; + } else { + // updated + i->lastDict->Remove(idxhash); + delete(h); + } + } + + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + //i->tab->Assign(idxval, valval); + HashKey* k = i->tab->ComputeHash(idxval); + if ( !k ) { + reporter->InternalError("could not hash"); + return; + } + + i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + ih->idxkey = k; + ih->valhash = valhash; + + i->currDict->Insert(idxhash, ih); + +} + +void InputMgr::EndCurrentSend(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + // lastdict contains all deleted entries + IterCookie *c = i->lastDict->InitForIteration(); + InputHash* ih; + while ( ( ih = i->lastDict->NextEntry(c )) ) { + i->tab->Delete(ih->idxkey); + } + + i->lastDict->Clear(); + delete(i->lastDict); + + i->lastDict = i->currDict; + i->currDict = new PDict(InputHash); +} + void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -418,6 +543,95 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } +HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { + int length = 0; + + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals"); + } + + } + + int position = 0; + char *data = (char*) malloc(length); + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + *(data+position) = val->val.int_val; + position += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + *(data+position) = val->val.uint_val; + position += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + *(data+position) = val->val.double_val; + position += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); + position += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + position += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals2"); + } + + + } + + assert(position == length); + return new HashKey(data, length); + + +} + Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { if ( request_type != TYPE_ANY && request_type != val->type ) { @@ -495,3 +709,11 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) } +string InputMgr::Hash(const string &input) { + unsigned char digest[16]; + hash_md5(input.length(), (const unsigned char*) input.c_str(), digest); + string out((const char*) digest, 16); + return out; +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index d5f732935c..d147fa262a 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -15,9 +15,10 @@ class InputReader; + class InputMgr { public: - InputMgr(); + InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); bool ForceUpdate(EnumVal* id); @@ -32,6 +33,9 @@ protected: void Put(const InputReader* reader, const LogVal* const *vals); void Clear(const InputReader* reader); bool Delete(const InputReader* reader, const LogVal* const *vals); + + void SendEntry(const InputReader* reader, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader); private: struct ReaderInfo; @@ -40,6 +44,8 @@ private: bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); @@ -49,6 +55,9 @@ private: ReaderInfo* FindReader(const EnumVal* id); vector readers; + + string Hash(const string &input); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index 1facc57c7f..494df3fb81 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -41,20 +41,22 @@ void InputReader::Delete(const LogVal* const *val) } -bool InputReader::Init(string arg_source, int arg_num_fields, +bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * arg_fields) { source = arg_source; num_fields = arg_num_fields; + index_fields = arg_idx_fields; fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); return !disabled; } void InputReader::Finish() { DoFinish(); + disabled = true; } bool InputReader::Update() { @@ -91,3 +93,10 @@ const char* InputReader::Fmt(const char* format, ...) } +void InputReader::SendEntry(const LogVal* const *vals) { + input_mgr->SendEntry(this, vals); +} + +void InputReader::EndCurrentSend() { + input_mgr->EndCurrentSend(this); +} diff --git a/src/InputReader.h b/src/InputReader.h index 0e93344e1a..b547d29506 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,7 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int num_fields, const LogField* const* fields); + bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); void Finish(); @@ -23,7 +23,7 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; @@ -46,11 +46,16 @@ protected: void Clear(); void Delete(const LogVal* const *val); + void SendEntry(const LogVal* const *vals); + void EndCurrentSend(); + + private: friend class InputMgr; string source; int num_fields; + int index_fields; const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4d581c0d8..e434f7e750 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -21,17 +21,26 @@ InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; + + //keyMap = new map(); } InputReaderAscii::~InputReaderAscii() { + DoFinish(); } void InputReaderAscii::DoFinish() { + columnMap.empty(); + if ( file != 0 ) { + file->close(); + delete(file); + file = 0; + } } -bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) { fname = path; @@ -47,6 +56,9 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const Error("could not read first line"); return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; // split on tabs... istringstream splitstream(line); @@ -83,12 +95,10 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const if ( wantFields != num_fields ) { // we did not find all fields? // :( - Error("wantFields != num_fields"); + Error("One of the requested fields could not be found in the input data file"); return false; } - - this->num_fields = num_fields; // well, that seems to have worked... return true; @@ -101,6 +111,9 @@ bool InputReaderAscii::DoUpdate() { // + // new keymap + //map *newKeyMap = new map(); + string line; while ( getline(*file, line ) ) { // split on tabs @@ -109,10 +122,12 @@ bool InputReaderAscii::DoUpdate() { string s; LogVal** fields = new LogVal*[num_fields]; + //string string_fields[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) break; @@ -146,8 +161,11 @@ bool InputReaderAscii::DoUpdate() { case TYPE_BOOL: if ( s == "T" ) { val->val.int_val = 1; - } else { + } else if ( s == "F" ) { val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; } break; @@ -173,9 +191,15 @@ bool InputReaderAscii::DoUpdate() { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // fallthrough + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + } case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 copy_addr(t, val->val.addr_val); @@ -193,19 +217,57 @@ bool InputReaderAscii::DoUpdate() { } fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; currField++; } if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate"); + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); return false; } - // ok, now we have built our line. send it back to the input manager - Put(fields); + + SendEntry(fields); + + /* + string indexstring = ""; + string valstring = ""; + for ( unsigned int i = 0; i < idx_fields; i++ ) { + indexstring.append(string_fields[i]); + } + + for ( unsigned int i = idx_fields; i < num_fields; i++ ) { + valstring.append(string_fields[i]); + } + + string valhash = Hash(valstring); + string indexhash = Hash(indexstring); + + if ( keyMap->find(indexhash) == keyMap->end() ) { + // new key + Put(fields); + } else if ( (*keyMap)[indexhash] != valhash ) { + // changed key + Put(fields); + keyMap->erase(indexhash); + } else { + // field not changed + keyMap->erase(indexhash); + } + + + (*newKeyMap)[indexhash] = valhash; + */ + + for ( unsigned int i = 0; i < num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } + + EndCurrentSend(); return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 551a08b02e..c26a139dcd 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -30,7 +30,7 @@ public: protected: - virtual bool DoInit(string path, int num_fields, + virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, const LogField* const * fields); virtual void DoFinish(); @@ -42,9 +42,12 @@ private: string fname; unsigned int num_fields; + unsigned int idx_fields; // map columns in the file to columns to send back to the manager vector columnMap; + + //map *keyMap; }; diff --git a/src/Val.h b/src/Val.h index d851be311b..3ae0bc3334 100644 --- a/src/Val.h +++ b/src/Val.h @@ -841,6 +841,9 @@ public: timer = 0; } + HashKey* ComputeHash(const Val* index) const + { return table_hash->ComputeHash(index, 1); } + protected: friend class Val; friend class StateAccess; @@ -851,8 +854,6 @@ protected: void CheckExpireAttr(attr_tag at); int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); - HashKey* ComputeHash(const Val* index) const - { return table_hash->ComputeHash(index, 1); } bool AddProperties(Properties arg_state); bool RemoveProperties(Properties arg_state); From 638976791efb3b7d9d491266d83bb30b73d4bd53 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:00:19 -0700 Subject: [PATCH 011/123] hashing seems to work _correctly_ now... --- src/InputMgr.cc | 44 ++++++++++++++++++++++++++++++----------- src/InputReaderAscii.cc | 41 +++++++++++++++++++++++++++++++++----- src/InputReaderAscii.h | 3 +++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5270c83d5a..e3acfb9505 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -290,10 +290,10 @@ bool InputMgr::ForceUpdate(EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->InternalError("Reader not found"); return false; } - + return i->reader->Update(); } @@ -302,8 +302,9 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo int position = 0; - if ( num_fields == 1 ) { + if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { idxval = LogValToVal(vals[0]); + position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < type->NumFields(); j++ ) { @@ -317,6 +318,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo idxval = l; } + //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; @@ -331,8 +333,15 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + + reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + + //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); InputHash *h = i->lastDict->Lookup(idxhash); if ( h != 0 ) { @@ -393,9 +402,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + k = i->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; + //i->tab->Delete(k); i->currDict->Insert(idxhash, ih); @@ -407,11 +419,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { reporter->InternalError("Unknown reader"); return; } - - // lastdict contains all deleted entries + // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c )) ) { + reporter->Error("ending"); + while ( ( ih = i->lastDict->NextEntry(c) ) ) { + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -582,28 +595,37 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } + //reporter->Error("Length: %d", length); + int position = 0; char *data = (char*) malloc(length); + if ( data == 0 ) { + reporter->InternalError("Could not malloc?"); + } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - *(data+position) = val->val.int_val; + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); + memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+position) = val->val.int_val; position += sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: case TYPE_PORT: - *(data+position) = val->val.uint_val; + //*(data+position) = val->val.uint_val; + memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); position += sizeof(val->val.uint_val); break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - *(data+position) = val->val.double_val; + //*(data+position) = val->val.double_val; + memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); position += sizeof(val->val.double_val); break; @@ -685,7 +707,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader == reader ) + if ( (*s)->reader && (*s)->reader == reader ) { return *s; } @@ -699,7 +721,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) + if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) { return *s; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e434f7e750..4755b2b74f 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -50,6 +50,16 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; + this->fields = fields; + + return true; +} + + +bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; if ( !getline(*file, line) ) { @@ -57,9 +67,6 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } - this->num_fields = num_fields; - this->idx_fields = idx_fields; - // split on tabs... istringstream splitstream(line); unsigned int currTab = 0; @@ -70,7 +77,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const break; // current found heading in s... compare if we want it - for ( int i = 0; i < num_fields; i++ ) { + for ( unsigned int i = 0; i < num_fields; i++ ) { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position @@ -92,7 +99,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const currTab++; } - if ( wantFields != num_fields ) { + if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( Error("One of the requested fields could not be found in the input data file"); @@ -106,9 +113,30 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { + + + // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) + if ( file && file->is_open() ) { + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + // + + // file->seekg(0, ios::beg); // do not forget clear. + + + if ( ReadHeader() == false ) { + return false; + } + // TODO: all the stuff we need for a second reading. // *cough* // + // // new keymap @@ -152,6 +180,7 @@ bool InputReaderAscii::DoUpdate() { } LogVal* val = new LogVal(currMapping.type, true); + //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { case TYPE_STRING: @@ -267,6 +296,8 @@ bool InputReaderAscii::DoUpdate() { } + //file->clear(); // remove end of file evil bits + //file->seekg(0, ios::beg); // and seek to start. EndCurrentSend(); return true; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c26a139dcd..673a2cdae2 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,6 +37,8 @@ protected: virtual bool DoUpdate(); private: + + bool ReadHeader(); ifstream* file; string fname; @@ -46,6 +48,7 @@ private: // map columns in the file to columns to send back to the manager vector columnMap; + const LogField* const * fields; // raw mapping //map *keyMap; From b5a77aa77baf69b788adb7b40f4246f0692939a0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:29:58 -0700 Subject: [PATCH 012/123] reading seems to work with all atomic types + records... --- src/InputMgr.cc | 25 ++++++++++++++++++++----- src/InputReaderAscii.cc | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e3acfb9505..bbfd354189 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -334,12 +334,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } - reporter->Error("Hashing %d index fields", i->num_idx_fields); + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); - reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); - reporter->Error("Hashing %d val fields", i->num_val_fields); + //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); - reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -422,7 +422,6 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - reporter->Error("ending"); while ( ( ih = i->lastDict->NextEntry(c) ) ) { reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); @@ -589,6 +588,11 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals length += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals"); } @@ -641,6 +645,13 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals position += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + position += sizeof(val->val.subnet_val.width); + memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + position += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals2"); } @@ -694,6 +705,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new AddrVal(val->val.addr_val); break; + case TYPE_SUBNET: + return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); + break; + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4755b2b74f..39635de407 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -216,7 +216,7 @@ bool InputReaderAscii::DoUpdate() { case TYPE_SUBNET: { int pos = s.find("/"); - string width = s.substr(pos); + string width = s.substr(pos+1); val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; From 4845c3a9a61d95dd0246f3beb6fcaa6e74c88f8c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 3 Nov 2011 14:04:13 -0700 Subject: [PATCH 013/123] send events when input entries change --- src/InputMgr.cc | 97 +++++++++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 8 +++- src/InputReader.cc | 24 +++++++----- src/input.bif | 11 ++++++ src/types.bif | 7 ++++ 5 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index bbfd354189..e987dda81e 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -38,6 +38,8 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; + + list* events; // events we fire when "something" happens }; @@ -165,6 +167,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->events = new list(); int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { @@ -223,7 +226,7 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } -bool InputMgr::RemoveReader(EnumVal* id) { +bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -254,6 +257,43 @@ bool InputMgr::RemoveReader(EnumVal* id) { return true; } +bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + i->events->push_back(eventName); + + return true; +} + +bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + bool erased = false; + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) + { + if ( *it == eventName ) { + it = i->events->erase(it); + erased = true; + } + else + ++it; + } + + + return erased; +} + + bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -286,7 +326,7 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } -bool InputMgr::ForceUpdate(EnumVal* id) +bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -333,6 +373,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + bool updated = false; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); @@ -355,6 +397,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { // updated i->lastDict->Remove(idxhash); delete(h); + updated = true; } } @@ -411,8 +454,25 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + EnumVal* ev; + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + + Ref(idxval); + Ref(valval); + SendEvent(*it, ev, idxval, valval); + ++it; + } + } + void InputMgr::EndCurrentSend(const InputReader* reader) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -423,6 +483,23 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events->size() > 0 ) { + ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + Val *val = i->tab->Lookup(idx); + assert(val != 0); + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -523,6 +600,22 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } +void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + + val_list* vl = new val_list; + vl->append(event); + vl->append(left); + vl->append(right); + + mgr.Dispatch(new Event(handler, vl)); +} + Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d147fa262a..378838c2cf 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -21,8 +21,11 @@ public: InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); - bool ForceUpdate(EnumVal* id); - bool RemoveReader(EnumVal* id); + bool ForceUpdate(const EnumVal* id); + bool RemoveReader(const EnumVal* id); + bool RegisterEvent(const EnumVal* id, string eventName); + bool UnregisterEvent(const EnumVal* id, string eventName); + protected: friend class InputReader; @@ -43,6 +46,7 @@ private: bool IsCompatibleType(BroType* t); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 494df3fb81..994f8b9b97 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,13 +1,12 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReader.h" -// #include "EventRegistry.h" -// #include "Event.h" InputReader::InputReader() { buf = 0; buf_len = 1024; - disabled = true; // disabled will be set correcty in init. + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -21,9 +20,9 @@ void InputReader::Error(const char *msg) } void InputReader::Error(const string &msg) - { +{ input_mgr->Error(this, msg.c_str()); - } +} void InputReader::Put(const LogVal* const *val) { @@ -54,16 +53,19 @@ bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields return !disabled; } -void InputReader::Finish() { +void InputReader::Finish() +{ DoFinish(); disabled = true; } -bool InputReader::Update() { +bool InputReader::Update() +{ return DoUpdate(); } -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ input_mgr->SendEvent(name, num_vals, vals); } @@ -93,10 +95,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) { +void InputReader::SendEntry(const LogVal* const *vals) +{ input_mgr->SendEntry(this, vals); } -void InputReader::EndCurrentSend() { +void InputReader::EndCurrentSend() +{ input_mgr->EndCurrentSend(this); } diff --git a/src/input.bif b/src/input.bif index 4e2cbf07b5..90dd2386b1 100644 --- a/src/input.bif +++ b/src/input.bif @@ -21,6 +21,17 @@ function Input::__force_update%(id: ID%) : bool return new Val( res, TYPE_BOOL ); %} +function Input::__add_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} function Input::__remove_reader%(id: ID%) : bool %{ diff --git a/src/types.bif b/src/types.bif index 7b81a7f631..f90a954224 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,6 +174,13 @@ enum Reader %{ READER_ASCII, %} +enum Event %{ + EVENT_NEW, + EVENT_CHANGED, + EVENT_REMOVED, +%} + + enum ID %{ Unknown, %} From 2e3874331d0540724a47e91c8b0146b7b3fa6846 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 12:41:10 -0700 Subject: [PATCH 014/123] support for filters and little event fix --- scripts/base/frameworks/input/main.bro | 10 ++ src/InputMgr.cc | 204 +++++++++++++++++++++---- src/InputMgr.h | 5 + src/input.bif | 22 ++- 4 files changed, 210 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bb7129d03..35ca6dfa4e 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,16 @@ export { destination: any; reader: Reader &default=default_reader; }; + + type Filter: record { + name: string; + ## descriptive name. for later removal + + + pred: function(typ: Input::Event, left: any, right: any): bool &optional; + ## decision function, that decides if an inserton, update or removal should really be executed + }; + } @load base/input.bif diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e987dda81e..0b87a561ad 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -24,6 +24,11 @@ public: declare(PDict, InputHash); +struct InputMgr::Filter { + EnumVal* id; + string name; + Func* pred; +}; struct InputMgr::ReaderInfo { EnumVal* id; @@ -39,10 +44,17 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - list* events; // events we fire when "something" happens + list events; // events we fire when "something" happens + list filters; // events we fire when "something" happens + +// ~ReaderInfo(); }; +//void InputMgr::~ReaderInfo() { +// +//} + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -167,8 +179,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); - info->events = new list(); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { RemoveReader(id); @@ -264,7 +274,7 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return false; } - i->events->push_back(eventName); + i->events.push_back(eventName); return true; } @@ -276,21 +286,23 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - bool erased = false; + //bool erased = false; - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { if ( *it == eventName ) { - it = i->events->erase(it); - erased = true; + it = i->events.erase(it); + return true; + // erased = true; } else ++it; } - return erased; + return false; + //return erased; } @@ -337,6 +349,59 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + + i->filters.push_back(filter); + + return true; +} + +bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) + { + if ( (*it).name == name ) { + it = i->filters.erase(it); + return true; + break; + } + else + ++it; + } + + return false;; +} + + + Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -398,6 +463,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->lastDict->Remove(idxhash); delete(h); updated = true; + } } @@ -437,6 +503,48 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { valval = r; } + + Val* oldval = 0; + if ( updated == true ) { + // in that case, we need the old value to send the event (if we send an event). + oldval = i->tab->Lookup(idxval); + } + + + // call filters first do determine if we really add / change the entry + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev; + Ref(idxval); + Ref(valval); + + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + val_list vl(3); + vl.append(ev); + vl.append(idxval); + vl.append(valval); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit. + return; + } + + ++it; + } + + //i->tab->Assign(idxval, valval); HashKey* k = i->tab->ComputeHash(idxval); if ( !k ) { @@ -454,21 +562,28 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { + // send events now that we are kind of finished. + std::list::iterator filter_iterator = i->events.begin(); + while ( filter_iterator != i->events.end() ) { EnumVal* ev; - if ( updated ) { + Ref(idxval); + + if ( updated ) { // in case of update send back the old value. ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + Ref(oldval); + SendEvent(*filter_iterator, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + Ref(valval); + SendEvent(*filter_iterator, ev, idxval, valval); } - Ref(idxval); - Ref(valval); - SendEvent(*it, ev, idxval, valval); - ++it; + ++filter_iterator; } + + } @@ -483,24 +598,61 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event + { - if ( i->events->size() > 0 ) { ListVal *idx = i->tab->RecoverIndex(ih->idxkey); assert(idx != 0); Val *val = i->tab->Lookup(idx); assert(val != 0); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; + + { + // ask filter, if we want to expire this element... + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); + + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit and simply go to the next entry of lastDict + continue; + } + + ++it; + } } + + // + + { + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + } - reporter->Error("Expiring element"); + //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 378838c2cf..74914e65ad 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -26,6 +26,9 @@ public: bool RegisterEvent(const EnumVal* id, string eventName); bool UnregisterEvent(const EnumVal* id, string eventName); + bool AddFilter(EnumVal *id, RecordVal* filter); + bool RemoveFilter(EnumVal* id, const string &name); + protected: friend class InputReader; @@ -62,6 +65,8 @@ private: string Hash(const string &input); + struct Filter; + }; extern InputMgr* input_mgr; diff --git a/src/input.bif b/src/input.bif index 90dd2386b1..2301482506 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,34 +8,46 @@ module Input; %%} type ReaderDescription: record; +type Filter: record; -function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__force_update%(id: ID%) : bool +function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: ID, name: string%) : bool +function Input::__add_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_event%(id: ID, name: string%) : bool +function Input::__remove_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: ID%) : bool +function Input::__remove_reader%(id: Log::ID%) : bool %{ bool res = input_mgr->RemoveReader(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} +function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool + %{ + bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_filter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} From 5f37040c962a9f3d524ddcccfaeba982acb0d500 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 13:59:43 -0700 Subject: [PATCH 015/123] filters really working as intented (though probably still memleaky) --- src/InputMgr.cc | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0b87a561ad..7ca1e1b485 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -537,8 +537,14 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Unref(v); if ( result == false ) { - // throw away. Hence - we quit. - return; + if ( !updated ) { + // throw away. Hence - we quit. + return; + } else { + // keep old one + i->currDict->Insert(idxhash, h); + return; + } } ++it; @@ -596,8 +602,11 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); + i->lastDict->MakeRobustCookie(c); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c) ) ) { + HashKey *lastDictIdxKey; + //while ( ( ih = i->lastDict->NextEntry(c) ) ) { + while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event { @@ -609,6 +618,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { { + bool doBreak = false; // ask filter, if we want to expire this element... std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -627,13 +637,21 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { Val* v = (*it).pred->Call(&vl); bool result = v->AsBool(); Unref(v); + + ++it; if ( result == false ) { - // throw away. Hence - we quit and simply go to the next entry of lastDict + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); + doBreak = true; continue; } - ++it; + } + + if ( doBreak ) { + continue; } } @@ -654,6 +672,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); + i->lastDict->Remove(lastDictIdxKey); + delete(ih); } i->lastDict->Clear(); From 2aa0f6da5795aa90c83c9fa1872776737f1e7416 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 14:33:34 -0700 Subject: [PATCH 016/123] beautify script calls, track filters --- scripts/base/frameworks/input/main.bro | 63 +++++++++++++++++++++++++- src/input.bif | 12 ++--- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 35ca6dfa4e..e0c41f19be 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -20,7 +20,68 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; - + + const no_filter: Filter = [$name=""]; # Sentinel. + + global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_reader: function(id: Log::ID) : bool; + global force_update: function(id: Log::ID) : bool; + global add_event: function(id: Log::ID, name: string) : bool; + global remove_event: function(id: Log::ID, name: string) : bool; + global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; + global remove_filter: function(id: Log::ID, name: string) : bool; + global get_filter: function(id: ID, name: string) : Filter; + } @load base/input.bif + + +module Input; + +global filters: table[ID, string] of Filter; + +function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool + { + return __create_reader(id, description); + } + +function remove_reader(id: Log::ID) : bool + { + return __remove_reader(id); + } + +function force_update(id: Log::ID) : bool + { + return __force_update(id); + } + +function add_event(id: Log::ID, name: string) : bool + { + return __add_event(id, name); + } + +function remove_event(id: Log::ID, name: string) : bool + { + return __remove_event(id, name); + } + +function add_filter(id: Log::ID, filter: Input::Filter) : bool + { + filters[id, filter$name] = filter; + return __add_filter(id, filter); + } + +function remove_filter(id: Log::ID, name: string) : bool + { + delete filters[id, name]; + return __remove_filter(id, name); + } + +function get_filter(id: ID, name: string) : Filter + { + if ( [id, name] in filters ) + return filters[id, name]; + + return no_filter; + } diff --git a/src/input.bif b/src/input.bif index 2301482506..c4bf14e3ed 100644 --- a/src/input.bif +++ b/src/input.bif @@ -16,6 +16,12 @@ function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescript return new Val( the_reader != 0, TYPE_BOOL ); %} +function Input::__remove_reader%(id: Log::ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); @@ -34,12 +40,6 @@ function Input::__remove_event%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool - %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From 1d39eaf32dff51ef779188f3123e3d63a28f71f8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 15:03:40 -0700 Subject: [PATCH 017/123] small fixes, less leakiness --- scripts/base/frameworks/input/main.bro | 1 - src/InputMgr.cc | 98 +++++++++++--------------- 2 files changed, 41 insertions(+), 58 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e0c41f19be..6e19452f7a 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -16,7 +16,6 @@ export { name: string; ## descriptive name. for later removal - pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ca1e1b485..18b17c7576 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,8 +28,14 @@ struct InputMgr::Filter { EnumVal* id; string name; Func* pred; + + ~Filter(); }; +InputMgr::Filter::~Filter() { + Unref(id); +} + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -45,15 +51,20 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* lastDict; list events; // events we fire when "something" happens - list filters; // events we fire when "something" happens - -// ~ReaderInfo(); + list filters; // filters that can prevent our actions + ~ReaderInfo(); }; -//void InputMgr::~ReaderInfo() { -// -//} +InputMgr::ReaderInfo::~ReaderInfo() { + Unref(type); + Unref(tab); + Unref(itype); + Unref(rtype); + Unref(id); + + delete(reader); +} struct InputReaderDefinition { bro_int_t type; // the type @@ -71,10 +82,8 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - //DBG_LOG(DBG_LOGGING, "this has to happen"); } - // create a new input reader object to be used at whomevers leisure lateron. InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { @@ -163,30 +172,26 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader; - Ref(reader); + info->type = reader->Ref()->AsEnumVal(); info->num_idx_fields = idxfields; info->num_val_fields = valfields; - info->tab = dst; - Ref(dst); - info->rtype = val; - Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. - info->id = id; - Ref(id); // ditto... - info->itype = idx; - Ref(idx); - readers.push_back(info); + info->tab = dst->Ref()->AsTableVal(); + info->rtype = val->Ref()->AsRecordType(); + info->id = id->Ref()->AsEnumVal(); + info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + readers.push_back(info); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { - RemoveReader(id); + assert( RemoveReader(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - RemoveReader(id); + assert ( RemoveReader(id) ); return 0; } @@ -240,12 +245,12 @@ bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) - { - i = (*s); - readers.erase(s); // remove from vector - break; - } + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } } if ( i == 0 ) { @@ -254,14 +259,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { i->reader->Finish(); - - Unref(i->type); - Unref(i->tab); - Unref(i->itype); - Unref(i->rtype); - Unref(i->id); - - delete(i->reader); delete(i); return true; @@ -279,6 +276,8 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return true; } +// remove first event with name eventName +// (though there shouldn't really be several events with the same name... bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -286,23 +285,18 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - //bool erased = false; - std::list::iterator it = i->events.begin(); while ( it != i->events.end() ) { if ( *it == eventName ) { it = i->events.erase(it); return true; - // erased = true; } else ++it; } - return false; - //return erased; } @@ -335,14 +329,13 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } return true; - } bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -352,7 +345,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -380,7 +373,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -427,7 +420,6 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo assert ( position == num_fields ); return idxval; - } @@ -538,7 +530,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( result == false ) { if ( !updated ) { - // throw away. Hence - we quit. + // throw away. Hence - we quit. And remove the entry from the current dictionary... + delete(i->currDict->RemoveEntry(idxhash)); return; } else { // keep old one @@ -588,9 +581,6 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { ++filter_iterator; } - - - } @@ -670,13 +660,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } - //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); + i->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); + i->lastDict->Clear(); // should be empty... but... well... who knows... delete(i->lastDict); i->lastDict = i->currDict; @@ -699,11 +688,6 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { } else { RecordVal * r = new RecordVal(i->rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - for ( int j = 0; j < i->rtype->NumFields(); j++) { Val* val = 0; From 5983d44d950d2b93d91c36846668b35fb24d437a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 8 Nov 2011 15:33:32 -0800 Subject: [PATCH 018/123] read header line in bro logfile format --- src/InputReaderAscii.cc | 19 +++++++++++++++++-- src/InputReaderAscii.h | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 39635de407..e4770421c6 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -62,7 +62,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; - if ( !getline(*file, line) ) { + if ( !GetLine(line) ) { Error("could not read first line"); return false; } @@ -111,6 +111,21 @@ bool InputReaderAscii::ReadHeader() { return true; } +bool InputReaderAscii::GetLine(string& str) { + while ( getline(*file, str) ) { + if ( str[0] != '#' ) { + return true; + } + + if ( str.compare(0,8, "#fields\t") == 0 ) { + str = str.substr(8); + return true; + } + } + + return false; +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -143,7 +158,7 @@ bool InputReaderAscii::DoUpdate() { //map *newKeyMap = new map(); string line; - while ( getline(*file, line ) ) { + while ( GetLine(line ) ) { // split on tabs istringstream splitstream(line); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 673a2cdae2..f86cfd0062 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,8 +37,9 @@ protected: virtual bool DoUpdate(); private: - bool ReadHeader(); + + bool GetLine(string& str); ifstream* file; string fname; From 1a642f3568c3c8c445f76a5b7438f385b880bbe7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 14 Nov 2011 17:18:28 -0800 Subject: [PATCH 019/123] tried enum support - doesn't yet work due to internal bro interface problems... --- src/InputMgr.cc | 6 ++++++ src/InputReaderAscii.cc | 2 +- src/input.bif | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 18b17c7576..8db5a70c66 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -828,6 +828,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { length += val->val.string_val->size(); break; @@ -883,6 +884,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); position += val->val.string_val->size(); @@ -958,6 +960,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_ENUM: + reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4770421c6..f91c55b666 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -128,7 +128,6 @@ bool InputReaderAscii::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { @@ -198,6 +197,7 @@ bool InputReaderAscii::DoUpdate() { //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { + case TYPE_ENUM: case TYPE_STRING: val->val.string_val = new string(s); break; diff --git a/src/input.bif b/src/input.bif index c4bf14e3ed..7b051fba16 100644 --- a/src/input.bif +++ b/src/input.bif @@ -51,3 +51,4 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} + From cde8153c1868f6bd3984a2c35aeb645b1023968d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 08:36:03 -0800 Subject: [PATCH 020/123] switch to set if record or simple value is desired. --- scripts/base/frameworks/input/main.bro | 1 + src/InputMgr.cc | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 6e19452f7a..2b87ac980c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,6 +9,7 @@ export { idx: any; val: any; destination: any; + want_record: bool &default=T; reader: Reader &default=default_reader; }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8db5a70c66..5f99583c10 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -42,6 +42,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; + bool want_record; TableVal* tab; RecordType* rtype; @@ -147,6 +148,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -181,6 +183,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->want_record = ( want_record->InternalInt() == 1 ); + + if ( valfields > 1 ) { + assert(info->want_record); + } readers.push_back(info); @@ -311,7 +318,6 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) { - string prep = nameprepend + rec->FieldName(i) + "."; if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) @@ -464,7 +470,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -683,7 +689,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -961,7 +967,7 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; case TYPE_ENUM: - reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); default: From 4a3c9923253c8c26973a24c537a3ca231961db5e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 10:57:45 -0800 Subject: [PATCH 021/123] InputReader can read Sets. --- src/InputMgr.cc | 244 +++++++++++++++++++++++++--------------- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 211 ++++++++++++++++++++++------------ src/InputReaderAscii.h | 5 + src/LogMgr.cc | 6 +- src/LogMgr.h | 4 +- 6 files changed, 310 insertions(+), 162 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5f99583c10..ba17d8448f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -234,11 +234,14 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_TABLE: - return false; + { + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } case TYPE_VECTOR: { - return IsCompatibleType(t->AsVectorType()->YieldType()); + return false; // do me... + //return IsCompatibleType(t->AsVectorType()->YieldType()); } default: @@ -329,6 +332,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec LogField* field = new LogField(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); + if ( field->type == TYPE_TABLE ) { + field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } fields->push_back(field); } @@ -810,49 +816,133 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } + +int InputMgr::GetLogValLength(const LogVal* val) { + int length = 0; + + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + + case TYPE_TABLE: { + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += GetLogValLength(val->val.set_val.vals[i]); + } + break; + } + + default: + reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + } + + return length; + +} + +int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); + memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+startpos) = val->val.int_val; + return sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + //*(data+startpos) = val->val.uint_val; + memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); + return sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + //*(data+startpos) = val->val.double_val; + memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + return sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); + return val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+startpos, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + return NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: { + int length = 0; + memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + length += sizeof(val->val.subnet_val.width); + memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + length += sizeof(val->val.subnet_val.net); + return length; + break; + } + + case TYPE_TABLE: { + int length = 0; + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + } + return length; + break; + } + + default: + reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + return 0; + } + + reporter->InternalError("internal error"); + return 0; + +} + HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch (val->type) { - case TYPE_BOOL: - case TYPE_INT: - length += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - length += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - length += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - length += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - length += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - length += sizeof(val->val.subnet_val.width); - length += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals"); - } - + length += GetLogValLength(val); } //reporter->Error("Length: %d", length); @@ -864,56 +954,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch ( val->type ) { - case TYPE_BOOL: - case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); - memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+position) = val->val.int_val; - position += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - //*(data+position) = val->val.uint_val; - memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); - position += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - //*(data+position) = val->val.double_val; - memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); - position += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); - position += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); - position += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); - position += sizeof(val->val.subnet_val.width); - memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); - position += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals2"); - } - - + position += CopyLogVal(data, position, val); } assert(position == length); @@ -966,6 +1007,29 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_TABLE: { + if ( val->val.set_val.size == 0 ) { + // empty table + TypeList* set_index = new TypeList(base_type(TYPE_ANY)); + // iim quite sure this does not work... we probably need the internal set type for this... + reporter->InternalError("Implement me."); + return new TableVal(new SetType(set_index, 0)); + } else { + // all entries have to have the same type... + TypeTag type = val->val.set_val.vals[0]->type; + TypeList* set_index = new TypeList(base_type(type)); + set_index->Append(base_type(type)); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + assert( val->val.set_val.vals[i]->type == type); + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; + } + break; + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 74914e65ad..d4bfa5c355 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -52,6 +52,8 @@ private: void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + int GetLogValLength(const LogVal* val); + int CopyLogVal(char *data, const int startpos, const LogVal* val); Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index f91c55b666..60a8c5685a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,12 +11,22 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) + : name(arg_name), type(arg_type), set_type(arg_set_type) +{ + position = arg_position; +} + FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type) + : name(arg.name), type(arg.type), set_type(arg.set_type) { position = arg.position; } +FieldMapping FieldMapping::setType() { + return FieldMapping(name, set_type, position); +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -81,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, i); + FieldMapping f(field->name, field->type, field->set_type, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -126,6 +136,132 @@ bool InputReaderAscii::GetLine(string& str) { return false; } + +LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { + + LogVal* val = new LogVal(field.type, true); + //bzero(val, sizeof(LogVal)); + + switch ( field.type ) { + case TYPE_ENUM: + case TYPE_STRING: + val->val.string_val = new string(s); + break; + + case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else if ( s == "F" ) { + val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; + } + break; + + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + val->val.uint_val = atoi(s.c_str()); + break; + + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos+1); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + + } + case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + case TYPE_TABLE: { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } + + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + if ( s[i] == ',') length++; + + unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + + istringstream splitstream(s); + while ( splitstream ) { + string element; + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); + break; + } + + if ( !getline(splitstream, element, ',') ) + break; + + + LogVal* newval = EntryToVal(element, field.setType()); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + + pos++; + + } + + if ( pos != length ) { + Error("Internal error while parsing set: did not find all elements"); + return 0; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d for %s", field.type, + field.name.c_str())); + return 0; + } + + return val; + +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -161,7 +297,6 @@ bool InputReaderAscii::DoUpdate() { // split on tabs istringstream splitstream(line); - string s; LogVal** fields = new LogVal*[num_fields]; //string string_fields[num_fields]; @@ -170,6 +305,7 @@ bool InputReaderAscii::DoUpdate() { unsigned int currField = 0; while ( splitstream ) { + string s; if ( !getline(splitstream, s, '\t') ) break; @@ -193,73 +329,10 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = new LogVal(currMapping.type, true); - //bzero(val, sizeof(LogVal)); - - switch ( currMapping.type ) { - case TYPE_ENUM: - case TYPE_STRING: - val->val.string_val = new string(s); - break; - - case TYPE_BOOL: - if ( s == "T" ) { - val->val.int_val = 1; - } else if ( s == "F" ) { - val->val.int_val = 0; - } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); - return false; - } - break; - - case TYPE_INT: - val->val.int_val = atoi(s.c_str()); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - val->val.uint_val = atoi(s.c_str()); - break; - - case TYPE_SUBNET: { - int pos = s.find("/"); - string width = s.substr(pos+1); - val->val.subnet_val.width = atoi(width.c_str()); - string addr = s.substr(0, pos); - s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - val->val.subnet_val.net = dotted_to_addr(s.c_str()); - break; - - } - case TYPE_ADDR: { - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); -#ifdef BROv6 - copy_addr(t, val->val.addr_val); -#else - copy_addr(&t, val->val.addr_val); -#endif - break; - } - - - default: - Error(Fmt("unsupported field format %d for %s", currMapping.type, - currMapping.name.c_str())); + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { return false; - } - + } fields[currMapping.position] = val; //string_fields[currMapping.position] = s; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index f86cfd0062..ab2b89339c 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,11 +12,15 @@ struct FieldMapping { string name; TypeTag type; + TypeTag set_type; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } + + FieldMapping setType(); bool IsEmpty() { return position == -1; } }; @@ -38,6 +42,7 @@ protected: private: bool ReadHeader(); + LogVal* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f78a2a19e0..9818d9cdfb 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); type = (TypeTag) t; + set_type = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..40dab8677b 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // needed by input framework. otherwise it cannot determine the inner type of a set. + TypeTag set_type; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), set_type(other.set_type) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From 3d0162bcdc2e2a3e4923ac22ec5597cb4d167304 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 11:18:48 -0800 Subject: [PATCH 022/123] isCompatibleType works correctly for tables. --- src/InputMgr.cc | 27 +++++++++++++++++---------- src/InputMgr.h | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ba17d8448f..4506501c94 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -205,7 +205,8 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return reader_obj; } -bool InputMgr::IsCompatibleType(BroType* t) + +bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) return false; @@ -223,24 +224,29 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_INTERVAL: case TYPE_ENUM: case TYPE_STRING: - case TYPE_RECORD: - // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. - // ^ recursive checking is done in UnrollRecordType. return true; - - case TYPE_FILE: - case TYPE_FUNC: - return false; + case TYPE_RECORD: + return ! atomic_only; case TYPE_TABLE: { - return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); } case TYPE_VECTOR: { return false; // do me... + + //if ( atomic_only ) + // return false; + // //return IsCompatibleType(t->AsVectorType()->YieldType()); } @@ -249,7 +255,8 @@ bool InputMgr::IsCompatibleType(BroType* t) } return false; - } +} + bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index d4bfa5c355..93c6447467 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -46,7 +46,7 @@ protected: private: struct ReaderInfo; - bool IsCompatibleType(BroType* t); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); From fb5f26e7fcf5d3bda5c25472e2933e7ed975ddb1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 15:23:46 -0800 Subject: [PATCH 023/123] make default values work (thanks to robin) --- src/InputMgr.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4506501c94..a0a9d6a35c 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -97,7 +97,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return 0; } - EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -148,7 +148,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -163,9 +162,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) if ( status ) { reporter->Error("Problem unrolling"); + Unref(reader); return 0; } + Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { @@ -174,7 +175,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader->Ref()->AsEnumVal(); + info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault info->num_idx_fields = idxfields; info->num_val_fields = valfields; info->tab = dst->Ref()->AsTableVal(); @@ -184,6 +185,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); info->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { assert(info->want_record); From 821878835a800edb794c25c32a59ef67958a87da Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 16:32:35 -0800 Subject: [PATCH 024/123] read vector. still missing: enums, empty fields for optional parameters. --- src/InputMgr.cc | 45 +++++++++++++++++++++++++++------ src/InputReaderAscii.cc | 56 ++++++++++++++++++++++++++--------------- src/InputReaderAscii.h | 6 ++--- src/LogMgr.cc | 6 ++--- src/LogMgr.h | 6 ++--- 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index a0a9d6a35c..4c21268c84 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -244,12 +244,10 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) case TYPE_VECTOR: { - return false; // do me... - - //if ( atomic_only ) - // return false; - // - //return IsCompatibleType(t->AsVectorType()->YieldType()); + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: @@ -342,7 +340,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); if ( field->type == TYPE_TABLE ) { - field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } else if ( field->type == TYPE_VECTOR ) { + field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); } fields->push_back(field); @@ -870,6 +870,13 @@ int InputMgr::GetLogValLength(const LogVal* val) { break; } + case TYPE_VECTOR: { + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += GetLogValLength(val->val.vector_val.vals[i]); + } + break; + } + default: reporter->InternalError("unsupported type %d for GetLogValLength", val->type); } @@ -936,6 +943,15 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { break; } + case TYPE_VECTOR: { + int length = 0; + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); + } + return length; + break; + } + default: reporter->InternalError("unsupported type %d for CopyLogVal", val->type); return 0; @@ -1039,6 +1055,21 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; } + case TYPE_VECTOR: { + assert ( val->val.vector_val.size > 1 ); // implement empty vector... + + // all entries have to have the same type... + TypeTag type = val->val.vector_val.vals[0]->type; + VectorType* vt = new VectorType(base_type(type)); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + assert( val->val.vector_val.vals[i]->type == type); + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; + + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 60a8c5685a..3b4409e652 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,20 +11,20 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) - : name(arg_name), type(arg_type), set_type(arg_set_type) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) + : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), set_type(arg.set_type) + : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; } -FieldMapping FieldMapping::setType() { - return FieldMapping(name, set_type, position); +FieldMapping FieldMapping::subType() { + return FieldMapping(name, subtype, position); } InputReaderAscii::InputReaderAscii() @@ -91,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, field->set_type, i); + FieldMapping f(field->name, field->type, field->subtype, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -112,7 +112,7 @@ bool InputReaderAscii::ReadHeader() { if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( - Error("One of the requested fields could not be found in the input data file"); + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); return false; } @@ -199,25 +199,40 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - case TYPE_TABLE: { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } - + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) if ( s[i] == ',') length++; unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; - val->val.set_val.vals = lvals; - val->val.set_val.size = length; + + if ( field.type == TYPE_TABLE ) { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + /* Fix support for emtyp tables if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } */ + + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } else if ( field.type == TYPE_VECTOR ) { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } else { + assert(false); + } istringstream splitstream(s); while ( splitstream ) { @@ -232,7 +247,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; - LogVal* newval = EntryToVal(element, field.setType()); + LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -243,6 +258,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { } + if ( pos != length ) { Error("Internal error while parsing set: did not find all elements"); return 0; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index ab2b89339c..56c1001acb 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,15 +12,15 @@ struct FieldMapping { string name; TypeTag type; - TypeTag set_type; + TypeTag subtype; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); - FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } - FieldMapping setType(); + FieldMapping subType(); bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9818d9cdfb..6eaace3893 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,16 +83,16 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; - set_type = (TypeTag) it; + subtype = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 40dab8677b..b8530d29ab 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,12 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set. - TypeTag set_type; + // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), set_type(other.set_type) { } + : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From ab68d8400789156a23b1e6bbaf57828e7eac7c83 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:13:36 -0800 Subject: [PATCH 025/123] reading of enum types (thanks, Seth) --- src/InputMgr.cc | 50 +++++++++++++++++++++++++++------------------- src/InputMgr.h | 4 ++-- src/InputReader.cc | 3 ++- src/InputReader.h | 2 +- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4c21268c84..64f3e2eb9b 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -422,7 +422,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { - idxval = LogValToVal(vals[0]); + idxval = LogValToVal(vals[0], type->FieldType(0)); position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); @@ -430,7 +430,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); } else { - l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + l->Append(LogValToVal(vals[position], type->FieldType(j))); position++; } } @@ -486,7 +486,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -501,7 +501,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -705,7 +705,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -715,7 +715,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -760,7 +760,7 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } - +/* Does not work atm, because LogValToVal needs BroType void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -775,7 +775,7 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c } mgr.Dispatch(new Event(handler, vl)); -} +} */ void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) { @@ -814,7 +814,7 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); } else { - fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)); (*position)++; } @@ -988,10 +988,10 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } -Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { +Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { - if ( request_type != TYPE_ANY && request_type != val->type ) { - reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } @@ -1041,13 +1041,12 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new TableVal(new SetType(set_index, 0)); } else { // all entries have to have the same type... - TypeTag type = val->val.set_val.vals[0]->type; - TypeList* set_index = new TypeList(base_type(type)); - set_index->Append(base_type(type)); + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - assert( val->val.set_val.vals[i]->type == type); t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); } return t; @@ -1059,19 +1058,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { assert ( val->val.vector_val.size > 1 ); // implement empty vector... // all entries have to have the same type... - TypeTag type = val->val.vector_val.vals[0]->type; - VectorType* vt = new VectorType(base_type(type)); + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) { - assert( val->val.vector_val.vals[i]->type == type); v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); } return v; } - case TYPE_ENUM: - reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); + case TYPE_ENUM: { + // well, this is kind of stupid, because EnumType just mangles the module name and the var name together again... + // but well + string module = extract_module_name(val->val.string_val->c_str()); + string var = extract_var_name(val->val.string_val->c_str()); + bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); + if ( index == -1 ) { + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); + } + return new EnumVal(index, request_type->Ref()->AsEnumType() ); + break; + } default: diff --git a/src/InputMgr.h b/src/InputMgr.h index 93c6447467..17b7e2e804 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -55,11 +55,11 @@ private: int GetLogValLength(const LogVal* val); int CopyLogVal(char *data, const int startpos, const LogVal* val); - Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 994f8b9b97..7403d1f989 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -64,10 +64,11 @@ bool InputReader::Update() return DoUpdate(); } +/* void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { input_mgr->SendEvent(name, num_vals, vals); -} +} */ // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index b547d29506..41000e4c0c 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -40,7 +40,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); void Clear(); From 4fef1e3f8c5fe7005f23f91d6001cd1d79ffcdd2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:47:28 -0800 Subject: [PATCH 026/123] set & entry separator configuration (with the restriction that they have to be exactly one character long) --- .../base/frameworks/input/readers/ascii.bro | 19 +++++++++ src/InputReaderAscii.cc | 40 +++++++++++++++++-- src/InputReaderAscii.h | 15 ++++++- src/input.bif | 9 +++++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/ascii.bro diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro new file mode 100644 index 0000000000..9f630975a2 --- /dev/null +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -0,0 +1,19 @@ +##! Interface for the ascii input reader. + +module InputAscii; + +export { + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + const set_separator = "," &redef; + + ## String to use for empty fields. + const empty_field = "-" &redef; + + ## String to use for an unset &optional field. + const unset_field = "-" &redef; +} diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 3b4409e652..22cdcfdcf0 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,6 +2,7 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include "NetVar.h" #include @@ -29,15 +30,46 @@ FieldMapping FieldMapping::subType() { InputReaderAscii::InputReaderAscii() { - //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; //keyMap = new map(); + + separator_len = BifConst::LogAscii::separator->Len(); + separator = new char[separator_len]; + memcpy(separator, BifConst::LogAscii::separator->Bytes(), + separator_len); + if ( separator_len != 1 ) { + Error("separator length has to be 1. Separator will be truncated."); + } + + set_separator_len = BifConst::LogAscii::set_separator->Len(); + set_separator = new char[set_separator_len]; + memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), + set_separator_len); + if ( set_separator_len != 1 ) { + Error("set_separator length has to be 1. Separator will be truncated."); + } + + empty_field_len = BifConst::LogAscii::empty_field->Len(); + empty_field = new char[empty_field_len]; + memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), + empty_field_len); + + unset_field_len = BifConst::LogAscii::unset_field->Len(); + unset_field = new char[unset_field_len]; + memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), + unset_field_len); + } InputReaderAscii::~InputReaderAscii() { DoFinish(); + + delete [] separator; + delete [] set_separator; + delete [] empty_field; + delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -83,7 +115,7 @@ bool InputReaderAscii::ReadHeader() { int wantFields = 0; while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t')) + if ( !getline(splitstream, s, separator[0])) break; // current found heading in s... compare if we want it @@ -243,7 +275,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - if ( !getline(splitstream, element, ',') ) + if ( !getline(splitstream, element, set_separator[0]) ) break; @@ -322,7 +354,7 @@ bool InputReaderAscii::DoUpdate() { while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t') ) + if ( !getline(splitstream, s, separator[0]) ) break; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 56c1001acb..d69b8c04bc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -57,7 +57,20 @@ private: const LogField* const * fields; // raw mapping //map *keyMap; - + // + // Options set from the script-level. + char* separator; + int separator_len; + + char* set_separator; + int set_separator_len; + + char* empty_field; + int empty_field_len; + + char* unset_field; + int unset_field_len; + }; diff --git a/src/input.bif b/src/input.bif index 7b051fba16..aaef25dcc3 100644 --- a/src/input.bif +++ b/src/input.bif @@ -52,3 +52,12 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +# Options for Ascii Reader + +module InputAscii; + +const separator: string; +const set_separator: string; +const empty_field: string; +const unset_field: string; + From 4dd95fcf3cf1efc83ec4d2c53fee765d2a800674 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 23:51:51 -0800 Subject: [PATCH 027/123] support for uninitialized fields & empty sets and tables. The only snag is... with the default output format of the log-file writer, the input reader cannot tell if a table or set is empty or uninitialized (both cases use the same character by default). In this case, by default it is assumed that the field/vector is uninitalized. --- scripts/base/frameworks/input/__load__.bro | 2 + src/InputMgr.cc | 61 ++++++++++------------ src/InputReaderAscii.cc | 61 ++++++++-------------- src/InputReaderAscii.h | 12 ++--- 4 files changed, 57 insertions(+), 79 deletions(-) diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index a10fe855df..a3315186d5 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1 +1,3 @@ @load ./main +@load ./readers/ascii + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 64f3e2eb9b..9c0e9c12b5 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -505,10 +505,10 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { position++; } - if ( val == 0 ) { + /* if ( val == 0 ) { reporter->InternalError("conversion error"); return; - } + } */ r->Assign(j,val); @@ -871,7 +871,9 @@ int InputMgr::GetLogValLength(const LogVal* val) { } case TYPE_VECTOR: { - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { + reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; @@ -945,7 +947,8 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_VECTOR: { int length = 0; - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); } return length; @@ -994,6 +997,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } + + if ( !val->present ) { + return 0; // unset field + } switch ( val->type ) { @@ -1033,38 +1040,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { break; case TYPE_TABLE: { - if ( val->val.set_val.size == 0 ) { - // empty table - TypeList* set_index = new TypeList(base_type(TYPE_ANY)); - // iim quite sure this does not work... we probably need the internal set type for this... - reporter->InternalError("Implement me."); - return new TableVal(new SetType(set_index, 0)); - } else { - // all entries have to have the same type... - BroType* type = request_type->AsTableType()->Indices()->PureType(); - TypeList* set_index = new TypeList(type->Ref()); - set_index->Append(type->Ref()); - SetType* s = new SetType(set_index, 0); - TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return t; - } + // all entries have to have the same type... + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; break; } case TYPE_VECTOR: { - assert ( val->val.vector_val.size > 1 ); // implement empty vector... - - // all entries have to have the same type... - BroType* type = request_type->AsVectorType()->YieldType(); - VectorType* vt = new VectorType(type->Ref()); - VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) { - v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return v; + // all entries have to have the same type... + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 22cdcfdcf0..4a0d4157bc 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -34,31 +34,19 @@ InputReaderAscii::InputReaderAscii() //keyMap = new map(); - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); - if ( separator_len != 1 ) { + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); + if ( separator.size() != 1 ) { Error("separator length has to be 1. Separator will be truncated."); } - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); - if ( set_separator_len != 1 ) { + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); + if ( set_separator.size() != 1 ) { Error("set_separator length has to be 1. Separator will be truncated."); } - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); - - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); + + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); } @@ -66,10 +54,6 @@ InputReaderAscii::~InputReaderAscii() { DoFinish(); - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -172,7 +156,10 @@ bool InputReaderAscii::GetLine(string& str) { LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { LogVal* val = new LogVal(field.type, true); - //bzero(val, sizeof(LogVal)); + + if ( s.compare(unset_field) == 0 ) { // field is not set... + return new LogVal(field.type, false); + } switch ( field.type ) { case TYPE_ENUM: @@ -244,19 +231,13 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { if ( s[i] == ',') length++; unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; LogVal** lvals = new LogVal* [length]; if ( field.type == TYPE_TABLE ) { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - /* Fix support for emtyp tables if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } */ - val->val.set_val.vals = lvals; val->val.set_val.size = length; } else if ( field.type == TYPE_VECTOR ) { @@ -266,18 +247,20 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { assert(false); } + if ( length == 0 ) + break; //empty + istringstream splitstream(s); while ( splitstream ) { string element; - if ( pos >= length ) { - Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); - break; - } - if ( !getline(splitstream, element, set_separator[0]) ) break; - + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d >= length %d. Element: %s", pos, length, element.c_str())); + break; + } LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index d69b8c04bc..c848c17110 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -59,17 +59,13 @@ private: //map *keyMap; // // Options set from the script-level. - char* separator; - int separator_len; + string separator; - char* set_separator; - int set_separator_len; + string set_separator; - char* empty_field; - int empty_field_len; + string empty_field; - char* unset_field; - int unset_field_len; + string unset_field; }; From e2c521fc4e6ef9eb66eb6cd431f9186468b14b70 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 18 Nov 2011 10:49:20 -0800 Subject: [PATCH 028/123] start reworking input framework... does not compile at the moment, but there are a few uncommitted changes that will be reverted in the next commit. --- scripts/base/frameworks/input/main.bro | 46 ++--- src/InputMgr.cc | 263 ++++++++++++------------- src/InputMgr.h | 18 +- src/InputReader.cc | 39 ++-- src/InputReader.h | 37 ++-- src/input.bif | 22 +-- 6 files changed, 208 insertions(+), 217 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 2b87ac980c..d9c0812498 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,30 +4,36 @@ module Input; export { const default_reader = READER_ASCII &redef; - type ReaderDescription: record { + type StreamDescription: record { source: string; - idx: any; - val: any; - destination: any; - want_record: bool &default=T; reader: Reader &default=default_reader; }; type Filter: record { - name: string; ## descriptive name. for later removal + name: string; + ## for tables + idx: any &optional; + val: any &optional; + destination: any &optional; + want_record: bool &default=T; + table_ev: any &optional; # event containing idx, val as values. + + ## decision function, that decides if an insertion, update or removal should really be executed. + ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; - ## decision function, that decides if an inserton, update or removal should really be executed + + ## for "normalized" events + ev: any &optional; + ev_description: any &optional; }; const no_filter: Filter = [$name=""]; # Sentinel. - global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; - global remove_reader: function(id: Log::ID) : bool; + global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_event: function(id: Log::ID, name: string) : bool; - global remove_event: function(id: Log::ID, name: string) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; global remove_filter: function(id: Log::ID, name: string) : bool; global get_filter: function(id: ID, name: string) : Filter; @@ -41,14 +47,14 @@ module Input; global filters: table[ID, string] of Filter; -function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool { - return __create_reader(id, description); + return __create_stream(id, description); } -function remove_reader(id: Log::ID) : bool +function remove_stream(id: Log::ID) : bool { - return __remove_reader(id); + return __remove_stream(id); } function force_update(id: Log::ID) : bool @@ -56,16 +62,6 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_event(id: Log::ID, name: string) : bool - { - return __add_event(id, name); - } - -function remove_event(id: Log::ID, name: string) : bool - { - return __remove_event(id, name); - } - function add_filter(id: Log::ID, filter: Input::Filter) : bool { filters[id, filter$name] = filter; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 9c0e9c12b5..f9250f6f0f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -27,22 +27,11 @@ declare(PDict, InputHash); struct InputMgr::Filter { EnumVal* id; string name; - Func* pred; - ~Filter(); -}; - -InputMgr::Filter::~Filter() { - Unref(id); -} - -struct InputMgr::ReaderInfo { - EnumVal* id; - EnumVal* type; - InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; + EventHandlerPtr table_event; TableVal* tab; RecordType* rtype; @@ -50,18 +39,42 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - - list events; // events we fire when "something" happens - list filters; // filters that can prevent our actions + + Func* pred; + + EventHandlerPtr event; + RecordType* event_type; + + ~Filter(); +}; + +InputMgr::Filter::~Filter() { + Unref(id); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); + if ( rtype ) + Unref(rtype); + if ( event_type) + Unref(event_type); +} + +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + + //list events; // events we fire when "something" happens + map filters; // filters that can prevent our actions ~ReaderInfo(); }; InputMgr::ReaderInfo::~ReaderInfo() { + // all the contents of filters should delete themselves automatically... + Unref(type); - Unref(tab); - Unref(itype); - Unref(rtype); Unref(id); delete(reader); @@ -86,14 +99,14 @@ InputMgr::InputMgr() } // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) +InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) { - reporter->Error("readerDescription argument not of right type"); + reporter->Error("Streamdescription argument not of right type"); return 0; } @@ -145,55 +158,15 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); - TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - - - vector fieldsV; // vector, because we don't know the length beforehands - - - bool status = !UnrollRecordType(&fieldsV, idx, ""); - - int idxfields = fieldsV.size(); - - status = status || !UnrollRecordType(&fieldsV, val, ""); - int valfields = fieldsV.size() - idxfields; - - if ( status ) { - reporter->Error("Problem unrolling"); - Unref(reader); - return 0; - } - - Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); - - LogField** fields = new LogField*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { - fields[i] = fieldsV[i]; - } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->num_idx_fields = idxfields; - info->num_val_fields = valfields; - info->tab = dst->Ref()->AsTableVal(); - info->rtype = val->Ref()->AsRecordType(); info->id = id->Ref()->AsEnumVal(); - info->itype = idx->Ref()->AsRecordType(); - info->currDict = new PDict(InputHash); - info->lastDict = new PDict(InputHash); - info->want_record = ( want_record->InternalInt() == 1 ); - Unref(want_record); // ref'd by lookupwithdefault - - if ( valfields > 1 ) { - assert(info->want_record); - } readers.push_back(info); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + int success = reader_obj->Init(source); if ( success == false ) { assert( RemoveReader(id) ); return 0; @@ -208,6 +181,86 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + + vector fieldsV; // vector, because we don't know the length beforehands + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; + } + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + filter.num_idx_fields = idxfields; + filter.num_val_fields = valfields; + filter.tab = dst ? dst->Ref()->AsTableVal() : 0; + filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; + filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.currDict = new PDict(InputHash); + filter.lastDict = new PDict(InputHash); + filter.want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + if ( valfields > 1 ) { + assert(info->want_record); + } + + i->filters[id->InternalInt()] = filter; + + // ok, now we have to alert the reader of our new filter with our funky new fields + // the id is handled in a ... well, to be honest, a little bit sneaky manner. + // the "problem" is, that we can have several filters in the reader for one filter in the log manager. + // this is due to the fact, that a filter can either output it's result as a table, as an event... + // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets + // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want + // to build a union of these values and figure it out later. + // hence -> filter id is multiplicated with 2. + // filterId*2 -> results for table + // filterId*2+1 -> results for event + i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + + return true; +} + + bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) @@ -258,7 +311,7 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) } -bool InputMgr::RemoveReader(const EnumVal* id) { +bool InputMgr::RemoveStream(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -281,42 +334,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { return true; } -bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - i->events.push_back(eventName); - - return true; -} - -// remove first event with name eventName -// (though there shouldn't really be several events with the same name... -bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) - { - if ( *it == eventName ) { - it = i->events.erase(it); - return true; - } - else - ++it; - } - - return false; -} - - bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -363,34 +380,6 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) - { - reporter->Error("filter argument not of right type"); - return false; - } - - - Val* name = fval->Lookup(rtype->FieldOffset("name")); - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); - - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - - i->filters.push_back(filter); - - return true; -} - bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -398,7 +387,7 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } - +/* std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -410,8 +399,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { else ++it; } + */ - return false;; + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + it->filters.erase(it); + return true; } @@ -444,7 +440,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } -void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -605,7 +601,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } -void InputMgr::EndCurrentSend(const InputReader* reader) { +void InputMgr::EndCurrentSend(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -693,7 +689,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { i->currDict = new PDict(InputHash); } -void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -733,7 +729,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { i->tab->Assign(idxval, valval); } -void InputMgr::Clear(const InputReader* reader) { +void InputMgr::Clear(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -873,7 +869,6 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_VECTOR: { int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; diff --git a/src/InputMgr.h b/src/InputMgr.h index 17b7e2e804..1cacf89143 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -20,11 +20,9 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* id, RecordVal* description); + InputReader* CreateStream(EnumVal* id, RecordVal* description); bool ForceUpdate(const EnumVal* id); - bool RemoveReader(const EnumVal* id); - bool RegisterEvent(const EnumVal* id, string eventName); - bool UnregisterEvent(const EnumVal* id, string eventName); + bool RemoveStream(const EnumVal* id); bool AddFilter(EnumVal *id, RecordVal* filter); bool RemoveFilter(EnumVal* id, const string &name); @@ -36,12 +34,14 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); - void Put(const InputReader* reader, const LogVal* const *vals); - void Clear(const InputReader* reader); - bool Delete(const InputReader* reader, const LogVal* const *vals); + // for readers to write to input stream in direct mode (reporting new/deleted values directly) + void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Clear(const InputReader* reader, int id); + bool Delete(const InputReader* reader, int id, const LogVal* const *vals); - void SendEntry(const InputReader* reader, const LogVal* const *vals); - void EndCurrentSend(const InputReader* reader); + // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) + void SendEntry(const InputReader* reader, int id, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader, int id); private: struct ReaderInfo; diff --git a/src/InputReader.cc b/src/InputReader.cc index 7403d1f989..1008cf1b67 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -24,35 +24,42 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } -void InputReader::Put(const LogVal* const *val) +void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, val); + input_mgr->Put(this, int id, val); } -void InputReader::Clear() +void InputReader::Clear(int id) { - input_mgr->Clear(this); + input_mgr->Clear(this, int id); } -void InputReader::Delete(const LogVal* const *val) +void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, val); + input_mgr->Delete(this, int id, val); } -bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, - const LogField* const * arg_fields) +bool InputReader::Init(string arg_source) { source = arg_source; - num_fields = arg_num_fields; - index_fields = arg_idx_fields; - fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); + disabled = !DoInit(arg_source); return !disabled; } +bool InputReader::AddFilter(int id, int arg_num_fields, + const LogField* const * arg_fields) +{ + return DoAddFilter(int id, arg_num_fields, arg_fields); +} + +bool InputReader::RemoveFilter(int id) +{ + return DoRemoveFilter(int id); +} + void InputReader::Finish() { DoFinish(); @@ -96,12 +103,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) +void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, vals); + input_mgr->SendEntry(this, int id, vals); } -void InputReader::EndCurrentSend() +void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this); + input_mgr->EndCurrentSend(this, int id); } diff --git a/src/InputReader.h b/src/InputReader.h index 41000e4c0c..12f0bc9db4 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,11 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); + bool Init(string arg_source); + + bool AddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + bool RemoveFilter ( int id ); void Finish(); @@ -23,8 +27,11 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; - + virtual bool DoInit(string arg_sources) = 0; + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; + virtual bool DoRemoveFilter( int id ); + virtual void DoFinish() = 0; // update file contents to logmgr @@ -42,28 +49,26 @@ protected: //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); - void Put(const LogVal* const *val); - void Clear(); - void Delete(const LogVal* const *val); + // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + void Put(int id, const LogVal* const *val); + void Delete(int id, const LogVal* const *val); + void Clear(int id); - void SendEntry(const LogVal* const *vals); - void EndCurrentSend(); + // Table-functions (tracking mode): Only changed lines are propagated. + void SendEntry(int id, const LogVal* const *vals); + void EndCurrentSend(int id); private: friend class InputMgr; string source; - int num_fields; - int index_fields; - const LogField* const * fields; - // When an error occurs, this method is called to set a flag marking the - // writer as disabled. + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. - bool disabled; - - bool Disabled() { return disabled; } + bool disabled; + bool Disabled() { return disabled; } // For implementing Fmt(). char* buf; diff --git a/src/input.bif b/src/input.bif index aaef25dcc3..ef069316ab 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,18 +7,18 @@ module Input; #include "NetVar.h" %%} -type ReaderDescription: record; +type StreamDescription: record; type Filter: record; -function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool +function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Log::ID%) : bool %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); + bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} @@ -28,18 +28,6 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - -function Input::__remove_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From b3f01915fbca99ddda434ae791b143150d27fcf7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:07:50 -0800 Subject: [PATCH 029/123] compiles with basic new filter framework - but crashes on use. --- scripts/base/frameworks/input/main.bro | 16 +- src/InputMgr.cc | 255 +++++++++++------------- src/InputMgr.h | 2 +- src/InputReader.cc | 14 +- src/InputReader.h | 3 +- src/InputReaderAscii.cc | 265 ++++++++++++------------- src/InputReaderAscii.h | 30 ++- 7 files changed, 283 insertions(+), 302 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index d9c0812498..9d83d73ec6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -14,9 +14,9 @@ export { name: string; ## for tables - idx: any &optional; - val: any &optional; - destination: any &optional; + idx: any; + val: any; + destination: any; want_record: bool &default=T; table_ev: any &optional; # event containing idx, val as values. @@ -25,13 +25,13 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## for "normalized" events - ev: any &optional; - ev_description: any &optional; + # ev: any &optional; + # ev_description: any &optional; }; - const no_filter: Filter = [$name=""]; # Sentinel. + const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; @@ -47,7 +47,7 @@ module Input; global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f9250f6f0f..403d656140 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -16,18 +16,20 @@ #include "CompHash.h" -class InputHash { -public: +struct InputHash { HashKey* valhash; HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. }; declare(PDict, InputHash); -struct InputMgr::Filter { +class InputMgr::Filter { +public: EnumVal* id; string name; + //int filter_type; // to distinguish between event and table filters + unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; @@ -68,6 +70,8 @@ struct InputMgr::ReaderInfo { //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions + bool HasFilter(int id); + ~ReaderInfo(); }; @@ -80,6 +84,15 @@ InputMgr::ReaderInfo::~ReaderInfo() { delete(reader); } +bool InputMgr::ReaderInfo::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } + return true; +} + + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -168,12 +181,12 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) int success = reader_obj->Init(source); if ( success == false ) { - assert( RemoveReader(id) ); + assert( RemoveStream(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - assert ( RemoveReader(id) ); + assert ( RemoveStream(id) ); return 0; } @@ -224,6 +237,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } + // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); @@ -231,8 +245,8 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; - filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + filter.rtype = val ? val->Ref()->AsRecordType() : 0; + filter.itype = idx ? idx->Ref()->AsRecordType() : 0; // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); @@ -240,22 +254,11 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(info->want_record); + assert(filter.want_record); } i->filters[id->InternalInt()] = filter; - - // ok, now we have to alert the reader of our new filter with our funky new fields - // the id is handled in a ... well, to be honest, a little bit sneaky manner. - // the "problem" is, that we can have several filters in the reader for one filter in the log manager. - // this is due to the fact, that a filter can either output it's result as a table, as an event... - // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets - // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want - // to build a union of these values and figure it out later. - // hence -> filter id is multiplicated with 2. - // filterId*2 -> results for table - // filterId*2+1 -> results for event - i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); return true; } @@ -387,31 +390,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } -/* - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) - { - if ( (*it).name == name ) { - it = i->filters.erase(it); - return true; - break; - } - else - ++it; - } - */ - map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - it->filters.erase(it); + i->filters.erase(it); return true; } - - Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -449,27 +436,28 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const bool updated = false; + assert(i->HasFilter(id)); //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->lastDict->Lookup(idxhash); + InputHash *h = i->filters[id].lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->lastDict->Remove(idxhash); - i->currDict->Insert(idxhash, h); + i->filters[id].lastDict->Remove(idxhash); + i->filters[id].currDict->Insert(idxhash, h); return; } else { // updated - i->lastDict->Remove(idxhash); + i->filters[id].lastDict->Remove(idxhash); delete(h); updated = true; @@ -477,27 +465,22 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -516,17 +499,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->tab->Lookup(idxval); + oldval = i->filters[id].tab->Lookup(idxval); } - // call filters first do determine if we really add / change the entry - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } - + // call filter first to determine if we really add / change the entry + if ( i->filters[id].pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -541,44 +519,45 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = (*it).pred->Call(&vl); + Val* v = i->filters[id].pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->currDict->RemoveEntry(idxhash)); + delete(i->filters[id].currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->currDict->Insert(idxhash, h); + i->filters[id].currDict->Insert(idxhash, h); return; } } - ++it; } //i->tab->Assign(idxval, valval); - HashKey* k = i->tab->ComputeHash(idxval); + HashKey* k = i->filters[id].tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - i->tab->Assign(idxval, k, valval); + i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->tab->ComputeHash(idxval); + k = i->filters[id].tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->currDict->Insert(idxhash, ih); + i->filters[id].currDict->Insert(idxhash, ih); // send events now that we are kind of finished. + + /* FIXME: fix me. std::list::iterator filter_iterator = i->events.begin(); while ( filter_iterator != i->events.end() ) { EnumVal* ev; @@ -597,7 +576,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ++filter_iterator; - } + } */ } @@ -607,86 +586,74 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } + + assert(i->HasFilter(id)); + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->lastDict->InitForIteration(); - i->lastDict->MakeRobustCookie(c); + IterCookie *c = i->filters[id].lastDict->InitForIteration(); + i->filters[id].lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - - if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event - { + while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + if ( i->filters[id].pred ) { + ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); assert(idx != 0); - Val *val = i->tab->Lookup(idx); + Val *val = i->filters[id].tab->Lookup(idx); assert(val != 0); - { - bool doBreak = false; - // ask filter, if we want to expire this element... - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } + bool doBreak = false; + // ask predicate, if we want to expire this element... - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); - Ref(val); + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(idx); - vl.append(val); - Val* v = (*it).pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - - ++it; - - if ( result == false ) { - // Keep it. Hence - we quit and simply go to the next entry of lastDict - // ah well - and we have to add the entry to currDict... - i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); - doBreak = true; - continue; - } - - } - - if ( doBreak ) { - continue; - } + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = i->filters[id].pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + continue; } - + + // { - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) { + /* FIXME: events + std::list::iterator it = i->filters[id].events.begin(); + while ( it != i->filters[id].events.end() ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(*it, ev, idx, val); ++it; } + */ } } - i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); // deletex in next line + i->filters[id].tab->Delete(ih->idxkey); + i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->lastDict); + i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... + delete(i->filters[id].lastDict); - i->lastDict = i->currDict; - i->currDict = new PDict(InputHash); + i->filters[id].lastDict = i->filters[id].currDict; + i->filters[id].currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -696,22 +663,24 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -726,7 +695,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->tab->Assign(idxval, valval); + i->filters[id].tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -735,20 +704,24 @@ void InputMgr::Clear(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } - - i->tab->RemoveAll(); + + assert(i->HasFilter(id)); + + i->filters[id].tab->RemoveAll(); } -bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { +bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); return false; } - - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); - return ( i->tab->Delete(idxval) != 0 ); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + + return ( i->filters[id].tab->Delete(idxval) != 0 ); } void InputMgr::Error(InputReader* reader, const char* msg) diff --git a/src/InputMgr.h b/src/InputMgr.h index 1cacf89143..5d531cd6fc 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -35,7 +35,7 @@ protected: void Error(InputReader* reader, const char* msg); // for readers to write to input stream in direct mode (reporting new/deleted values directly) - void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Put(const InputReader* reader, int id, const LogVal* const *vals); void Clear(const InputReader* reader, int id); bool Delete(const InputReader* reader, int id, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1008cf1b67..1c65985fd6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -26,17 +26,17 @@ void InputReader::Error(const string &msg) void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, int id, val); + input_mgr->Put(this, id, val); } void InputReader::Clear(int id) { - input_mgr->Clear(this, int id); + input_mgr->Clear(this, id); } void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, int id, val); + input_mgr->Delete(this, id, val); } @@ -52,12 +52,12 @@ bool InputReader::Init(string arg_source) bool InputReader::AddFilter(int id, int arg_num_fields, const LogField* const * arg_fields) { - return DoAddFilter(int id, arg_num_fields, arg_fields); + return DoAddFilter(id, arg_num_fields, arg_fields); } bool InputReader::RemoveFilter(int id) { - return DoRemoveFilter(int id); + return DoRemoveFilter(id); } void InputReader::Finish() @@ -105,10 +105,10 @@ const char* InputReader::Fmt(const char* format, ...) void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, int id, vals); + input_mgr->SendEntry(this, id, vals); } void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this, int id); + input_mgr->EndCurrentSend(this, id); } diff --git a/src/InputReader.h b/src/InputReader.h index 12f0bc9db4..6e3d689750 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -30,7 +30,8 @@ protected: virtual bool DoInit(string arg_sources) = 0; virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; - virtual bool DoRemoveFilter( int id ); + + virtual bool DoRemoveFilter( int id ) = 0; virtual void DoFinish() = 0; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4a0d4157bc..84feb74e61 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -28,6 +28,7 @@ FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } + InputReaderAscii::InputReaderAscii() { file = 0; @@ -58,7 +59,7 @@ InputReaderAscii::~InputReaderAscii() void InputReaderAscii::DoFinish() { - columnMap.empty(); + filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -66,7 +67,7 @@ void InputReaderAscii::DoFinish() } } -bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path) { fname = path; @@ -76,11 +77,39 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + return true; +} - this->num_fields = num_fields; - this->idx_fields = idx_fields; - this->fields = fields; +bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) { + if ( HasFilter(id) ) { + return false; // no, we don't want to add this a second time + } + Filter f; + f.num_fields = arg_num_fields; + f.fields = fields; + + filters[id] = f; + + return true; +} + +bool InputReaderAscii::DoRemoveFilter ( int id ) { + if (!HasFilter(id) ) { + return false; + } + + assert ( filters.erase(id) == 1 ); + + return true; +} + + +bool InputReaderAscii::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } return true; } @@ -93,46 +122,47 @@ bool InputReaderAscii::ReadHeader() { return false; } - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < num_fields; i++ ) { - const LogField* field = fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + // split on tabs... + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + // current found heading in s... compare if we want it + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, field->subtype, i); + (*it).second.columnMap.push_back(f); + wantFields++; + break; // done with searching + } } + + // look if we did push something... + if ( (*it).second.columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + (*it).second.columnMap.push_back(empty); + } + + // done + currTab++; + } + + if ( wantFields != (int) (*it).second.num_fields ) { + // we did not find all fields? + // :( + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); + return false; } - - // look if we did push something... - if ( columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); - return false; } - // well, that seems to have worked... return true; } @@ -314,110 +344,77 @@ bool InputReaderAscii::DoUpdate() { return false; } - // TODO: all the stuff we need for a second reading. - // *cough* - // - // - - - // new keymap - //map *newKeyMap = new map(); - string line; while ( GetLine(line ) ) { - // split on tabs + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - istringstream splitstream(line); - - LogVal** fields = new LogVal*[num_fields]; - //string string_fields[num_fields]; - - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { - - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - + // split on tabs - if ( currTab >= columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; - return false; - } - - FieldMapping currMapping = columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); - if ( val == 0 ) { - return false; - } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; - - currField++; - } - - if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; - } - - - SendEntry(fields); - - /* - string indexstring = ""; - string valstring = ""; - for ( unsigned int i = 0; i < idx_fields; i++ ) { - indexstring.append(string_fields[i]); - } - - for ( unsigned int i = idx_fields; i < num_fields; i++ ) { - valstring.append(string_fields[i]); - } - - string valhash = Hash(valstring); - string indexhash = Hash(indexstring); - - if ( keyMap->find(indexhash) == keyMap->end() ) { - // new key - Put(fields); - } else if ( (*keyMap)[indexhash] != valhash ) { - // changed key - Put(fields); - keyMap->erase(indexhash); - } else { - // field not changed - keyMap->erase(indexhash); - } - - - (*newKeyMap)[indexhash] = valhash; - */ + istringstream splitstream(line); - for ( unsigned int i = 0; i < num_fields; i++ ) { - delete fields[i]; + LogVal** fields = new LogVal*[(*it).second.num_fields]; + //string string_fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + + if ( currTab >= (*it).second.columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = (*it).second.columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= (*it).second.num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { + return false; + } + fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; + + currField++; + } + + if ( currField != (*it).second.num_fields ) { + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); + return false; + } + + + SendEntry((*it).first, fields); + + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } - delete [] fields; } //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. - EndCurrentSend(); + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + EndCurrentSend((*it).first); + } return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c848c17110..01169a3cfc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -34,13 +34,30 @@ public: protected: - virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, - const LogField* const * fields); + virtual bool DoInit(string path); + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + virtual bool DoRemoveFilter ( int id ); + virtual void DoFinish(); virtual bool DoUpdate(); private: + + struct Filter { + unsigned int num_fields; + + const LogField* const * fields; // raw mapping + + // map columns in the file to columns to send back to the manager + vector columnMap; + + }; + + bool HasFilter(int id); + bool ReadHeader(); LogVal* EntryToVal(string s, FieldMapping type); @@ -49,15 +66,8 @@ private: ifstream* file; string fname; - unsigned int num_fields; - unsigned int idx_fields; + map filters; - // map columns in the file to columns to send back to the manager - vector columnMap; - const LogField* const * fields; // raw mapping - - //map *keyMap; - // // Options set from the script-level. string separator; From 7eb4d9934168fa0d37e01e3dbdd6feb60a88c485 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:27:34 -0800 Subject: [PATCH 030/123] very basic functionality kind of works again --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 403d656140..b51cd6fd27 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -47,10 +47,50 @@ public: EventHandlerPtr event; RecordType* event_type; - ~Filter(); + // ~Filter(); + // Filter(); + // Filter(const InputMgr::Filter& filter); + + void DoCleanup(); }; -InputMgr::Filter::~Filter() { +/* +InputMgr::Filter::Filter() { + tab = 0; + itype = 0; + rtype = 0; + event_type = 0; +} + +InputMgr::Filter::Filter(const InputMgr::Filter& f) { + id = f.id; + id->Ref(); + + tab = f.tab; + if ( tab ) + tab->Ref(); + + itype = f.itype; + if ( itype ) + itype->Ref(); + + rtype = f.rtype; + if ( rtype ) + Ref(rtype); + + event_type = f.event_type; + if ( event_type ) + Ref(event_type); + + name = f.name; + num_idx_fields = f.num_idx_fields; + num_val_fields = f.num_val_fields; + want_record = f.want_record; + + +} */ + +void InputMgr::Filter::DoCleanup() { Unref(id); if ( tab ) Unref(tab); @@ -60,7 +100,10 @@ InputMgr::Filter::~Filter() { Unref(rtype); if ( event_type) Unref(event_type); -} + + delete currDict; + delete lastDict; +} struct InputMgr::ReaderInfo { EnumVal* id; @@ -395,6 +438,8 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } + i->filters[id->InternalInt()].DoCleanup(); + i->filters.erase(it); return true; } @@ -545,6 +590,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + reporter->Error("assigning"); i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); From 029871e48c4c63add07d63a52127bb7b50f47189 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 13:42:02 -0800 Subject: [PATCH 031/123] first test. --- .../scripts.base.frameworks.input.basic/out | 14 +++++ .../scripts/base/frameworks/input/basic.bro | 52 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.basic/out create mode 100644 testing/btest/scripts/base/frameworks/input/basic.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out new file mode 100644 index 0000000000..ebac1866b6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -0,0 +1,14 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro new file mode 100644 index 0000000000..5e0c7be12e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -0,0 +1,52 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From f0e5303330d9842a7454001276ef5857c252bba9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:09:00 -0800 Subject: [PATCH 032/123] make want_record field for tablefilter work... --- src/InputMgr.cc | 2 +- .../out | 3 ++ .../out | 3 ++ .../frameworks/input/onecolumn-norecord.bro | 38 +++++++++++++++++++ .../frameworks/input/onecolumn-record.bro | 38 +++++++++++++++++++ 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-record.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index b51cd6fd27..c4180f4f8d 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -515,7 +515,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const int position = i->filters[id].num_idx_fields; if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(i->filters[id].rtype); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out new file mode 100644 index 0000000000..bbce48f4f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out @@ -0,0 +1,3 @@ +{ +[-42] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out new file mode 100644 index 0000000000..3f9af35c59 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out @@ -0,0 +1,3 @@ +{ +[-42] = [b=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro new file mode 100644 index 0000000000..74fd477e28 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::force_update(A::LOG); + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro new file mode 100644 index 0000000000..3cc7090462 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From 18591b53d422e5b97dc8595a5171e50a64feec78 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:20:52 -0800 Subject: [PATCH 033/123] rename filter to tablefilter in preparation of event filters... --- scripts/base/frameworks/input/main.bro | 38 +++++++++---------- src/InputMgr.cc | 6 +-- src/InputMgr.h | 4 +- src/input.bif | 10 ++--- testing/btest/btest.cfg | 2 +- .../scripts/base/frameworks/input/basic.bro | 2 +- .../frameworks/input/onecolumn-norecord.bro | 2 +- .../frameworks/input/onecolumn-record.bro | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 9d83d73ec6..4560421ecc 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,7 +9,7 @@ export { reader: Reader &default=default_reader; }; - type Filter: record { + type TableFilter: record { ## descriptive name. for later removal name: string; @@ -29,14 +29,14 @@ export { # ev_description: any &optional; }; - const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. + #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; - global remove_filter: function(id: Log::ID, name: string) : bool; - global get_filter: function(id: ID, name: string) : Filter; + global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; + global remove_tablefilter: function(id: Log::ID, name: string) : bool; + #global get_filter: function(id: ID, name: string) : Filter; } @@ -45,7 +45,7 @@ export { module Input; -global filters: table[ID, string] of Filter; +#global filters: table[ID, string] of Filter; function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { @@ -62,22 +62,22 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_filter(id: Log::ID, filter: Input::Filter) : bool +function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool { - filters[id, filter$name] = filter; - return __add_filter(id, filter); +# filters[id, filter$name] = filter; + return __add_tablefilter(id, filter); } -function remove_filter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Log::ID, name: string) : bool { - delete filters[id, name]; - return __remove_filter(id, name); +# delete filters[id, name]; + return __remove_tablefilter(id, name); } -function get_filter(id: ID, name: string) : Filter - { - if ( [id, name] in filters ) - return filters[id, name]; - - return no_filter; - } +#function get_filter(id: ID, name: string) : Filter +# { +# if ( [id, name] in filters ) +# return filters[id, name]; +# +# return no_filter; +# } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index c4180f4f8d..8fda5d506f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -237,7 +237,7 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { +bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Stream not found"); @@ -245,7 +245,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { } RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) { reporter->Error("filter argument not of right type"); return false; @@ -426,7 +426,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { +bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Reader not found"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 5d531cd6fc..4280ba1d81 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,8 +24,8 @@ public: bool ForceUpdate(const EnumVal* id); bool RemoveStream(const EnumVal* id); - bool AddFilter(EnumVal *id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, const string &name); + bool AddTableFilter(EnumVal *id, RecordVal* filter); + bool RemoveTableFilter(EnumVal* id, const string &name); protected: diff --git a/src/input.bif b/src/input.bif index ef069316ab..1300f91bea 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,7 +8,7 @@ module Input; %%} type StreamDescription: record; -type Filter: record; +type TableFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -28,15 +28,15 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool +function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool %{ - bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_filter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool %{ - bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7d8283587c..739d0b2ad4 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -3,7 +3,7 @@ TestDirs = doc bifs language core scripts istate coverage TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* *.trace +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 5e0c7be12e..139888fa7c 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -46,7 +46,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 74fd477e28..12dbdd42aa 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 3cc7090462..4eef12d752 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } From 92b3723b0947739fb4aff9a3d7a87662f65011c7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:36:03 -0800 Subject: [PATCH 034/123] add very basic predicate test. --- .../out | 7 ++ .../base/frameworks/input/predicate.bro | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicate/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicate.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro new file mode 100644 index 0000000000..e82ded6fd0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -0,0 +1,66 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + ]); + Input::force_update(A::LOG); + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} From 77a517f2b5e90d94c9de144605c24247c620e4af Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:45:27 -0800 Subject: [PATCH 035/123] camel-casing for types --- testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 139888fa7c..3ad45eac69 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -18,11 +18,11 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; e: Log::ID; c: count; @@ -40,13 +40,13 @@ type val: record { ve: vector of int; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 12dbdd42aa..134ceb49e6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 4eef12d752..c07c9c826c 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index e82ded6fd0..769536d2a6 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -24,22 +24,22 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, - $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); Input::force_update(A::LOG); if ( 1 in servers ) { From 53af0544ccc9a60254228ab702ec39639b5b577d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:01:07 -0800 Subject: [PATCH 036/123] re-enable table events --- scripts/base/frameworks/input/main.bro | 2 +- src/InputMgr.cc | 120 +++++++++++------- src/InputMgr.h | 2 +- .../scripts.base.frameworks.input.event/out | 21 +++ .../scripts/base/frameworks/input/event.bro | 48 +++++++ 5 files changed, 147 insertions(+), 46 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.event/out create mode 100644 testing/btest/scripts/base/frameworks/input/event.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4560421ecc..69b4d41ebb 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -18,7 +18,7 @@ export { val: any; destination: any; want_record: bool &default=T; - table_ev: any &optional; # event containing idx, val as values. + ev: any &optional; # event containing idx, val as values. ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8fda5d506f..de9ef158b7 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -214,7 +214,6 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault @@ -259,6 +258,50 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + + if ( event ) { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 3 ) + { + reporter->Error("Table event must take 3 arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events first attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[1], idx) ) + { + reporter->Error("table events index attributes do not match"); + return false; + } + + if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + { + reporter->Error("table events value attributes do not match"); + return false; + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + reporter->Error("table events value attribute does not match"); + return false; + } + + } + vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); @@ -273,24 +316,22 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { return false; } - Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { fields[i] = fieldsV[i]; } - // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); filter.pred = pred ? pred->AsFunc() : 0; filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; - filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = val ? val->Ref()->AsRecordType() : 0; - filter.itype = idx ? idx->Ref()->AsRecordType() : 0; - // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.tab = dst->Ref()->AsTableVal(); + filter.rtype = val->Ref()->AsRecordType(); + filter.itype = idx->Ref()->AsRecordType(); + filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); filter.want_record = ( want_record->InternalInt() == 1 ); @@ -601,11 +642,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const i->filters[id].currDict->Insert(idxhash, ih); - // send events now that we are kind of finished. - - /* FIXME: fix me. - std::list::iterator filter_iterator = i->events.begin(); - while ( filter_iterator != i->events.end() ) { + if ( i->filters[id].event ) { EnumVal* ev; Ref(idxval); @@ -613,16 +650,13 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(*filter_iterator, ev, idxval, oldval); + SendEvent(i->filters[id].event, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(*filter_iterator, ev, idxval, valval); + SendEvent(i->filters[id].event, ev, idxval, valval); } - - - ++filter_iterator; - } */ + } } @@ -643,12 +677,17 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - if ( i->filters[id].pred ) { - ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); - assert(idx != 0); - Val *val = i->filters[id].tab->Lookup(idx); - assert(val != 0); + ListVal * idx; + Val *val; + if ( i->filters[id].pred || i->filters[id].event ) { + idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + val = i->filters[id].tab->Lookup(idx); + assert(val != 0); + } + + if ( i->filters[id].pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -673,21 +712,13 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { } - // - - { - /* FIXME: events - std::list::iterator it = i->filters[id].events.begin(); - while ( it != i->filters[id].events.end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; - } - */ - } + } + if ( i->filters[id].event ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(i->filters[id].event, ev, idx, val); } i->filters[id].tab->Delete(ih->idxkey); @@ -792,20 +823,21 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } */ -void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { - EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) { - reporter->Error("Event %s not found", name.c_str()); - return; - } + //EventHandler* handler = event_registry->Lookup(name.c_str()); + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", name.c_str()); + // return; + //} val_list* vl = new val_list; vl->append(event); vl->append(left); vl->append(right); - mgr.Dispatch(new Event(handler, vl)); + //mgr.Dispatch(new Event(handler, vl)); + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 4280ba1d81..d04b7c9a2c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -49,7 +49,7 @@ private: bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); + void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From 3035eb2b219946144fe67097781226d77f34176e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:30:16 -0800 Subject: [PATCH 037/123] fix a little bug that prevented several simultaneous filters from working. --- src/InputMgr.cc | 8 +- .../out | 15 +++ .../base/frameworks/input/twofilters.bro | 95 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out create mode 100644 testing/btest/scripts/base/frameworks/input/twofilters.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index de9ef158b7..2ec3c649be 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -341,8 +341,12 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { assert(filter.want_record); } - i->filters[id->InternalInt()] = filter; - i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), fields ); return true; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out new file mode 100644 index 0000000000..5b1ee5e983 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out @@ -0,0 +1,15 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID +MARK +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro new file mode 100644 index 0000000000..575665e6e5 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -0,0 +1,95 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination1: table[int] of Val = table(); +global destination2: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); + Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + + Input::force_update(A::LOG); + if ( 1 in destination1 ) { + print "VALID"; + } + if ( 2 in destination1 ) { + print "VALID"; + } + if ( !(3 in destination1) ) { + print "VALID"; + } + if ( !(4 in destination1) ) { + print "VALID"; + } + if ( !(5 in destination1) ) { + print "VALID"; + } + if ( !(6 in destination1) ) { + print "VALID"; + } + if ( 7 in destination1 ) { + print "VALID"; + } + + print "MARK"; + + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 3 in destination2 ) { + print "VALID"; + } + if ( 4 in destination2 ) { + print "VALID"; + } + if ( 5 in destination2 ) { + print "VALID"; + } + if ( 6 in destination2 ) { + print "VALID"; + } + if ( 7 in destination2 ) { + print "VALID"; + } + + +} From f82bf3f35fb5f1e298366f9af6d9ec0372ac3d58 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:09:06 -0800 Subject: [PATCH 038/123] re-enable direct event sending from input readers --- src/InputMgr.cc | 18 +++++++++++++----- src/InputMgr.h | 3 ++- src/InputReader.cc | 7 +++---- src/InputReader.h | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 2ec3c649be..e6d739d26f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -810,22 +810,30 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* Does not work atm, because LogValToVal needs BroType -void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { reporter->Error("Event %s not found", name.c_str()); - return; + return false; + } + + RecordType *type = handler->FType()->Args(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) { + reporter->Error("Wrong number of values for event %s", name.c_str()); + return false; } val_list* vl = new val_list; for ( int i = 0; i < num_vals; i++) { - vl->append(LogValToVal(vals[i])); + vl->append(LogValToVal(vals[i], type->FieldType(i))); } mgr.Dispatch(new Event(handler, vl)); -} */ + + return true; +} void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d04b7c9a2c..ba1ddafc92 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -59,7 +59,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1c65985fd6..3f296dc0aa 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -71,11 +71,10 @@ bool InputReader::Update() return DoUpdate(); } -/* -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { - input_mgr->SendEvent(name, num_vals, vals); -} */ + return input_mgr->SendEvent(name, num_vals, vals); +} // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index 6e3d689750..34d549308e 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -48,7 +48,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table void Put(int id, const LogVal* const *val); From 3c40f00a539e5d3d2945ffd1df71e19e93c7133d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:39:27 -0800 Subject: [PATCH 039/123] make filters pointers (for inheritance) --- scripts/base/frameworks/input/main.bro | 29 +- src/InputMgr.cc | 519 +++++++++++++----- src/InputMgr.h | 21 +- src/input.bif | 13 + .../out | 21 + .../scripts/base/frameworks/input/event.bro | 16 +- .../base/frameworks/input/tableevent.bro | 48 ++ 7 files changed, 507 insertions(+), 160 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out create mode 100644 testing/btest/scripts/base/frameworks/input/tableevent.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 69b4d41ebb..a036eeeebd 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -23,10 +23,19 @@ export { ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; + }; - ## for "normalized" events - # ev: any &optional; - # ev_description: any &optional; + type EventFilter: record { + ## descriptive name. for later removal + name: string; + + # the event + ev: any; + # record describing the fields + fields: any; + + # does the event want the field unrolled (default) or as a simple record value? + want_record: bool &default=F; }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. @@ -36,6 +45,8 @@ export { global force_update: function(id: Log::ID) : bool; global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; global remove_tablefilter: function(id: Log::ID, name: string) : bool; + global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; + global remove_eventfilter: function(id: Log::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -74,6 +85,18 @@ function remove_tablefilter(id: Log::ID, name: string) : bool return __remove_tablefilter(id, name); } +function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool + { +# filters[id, filter$name] = filter; + return __add_eventfilter(id, filter); + } + +function remove_eventfilter(id: Log::ID, name: string) : bool + { +# delete filters[id, name]; + return __remove_eventfilter(id, name); + } + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e6d739d26f..0df11ea359 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,7 +28,13 @@ public: EnumVal* id; string name; - //int filter_type; // to distinguish between event and table filters + FilterType filter_type; // to distinguish between event and table filters + + virtual ~Filter(); +}; + +class InputMgr::TableFilter: public InputMgr::Filter { +public: unsigned int num_idx_fields; unsigned int num_val_fields; @@ -45,61 +51,42 @@ public: Func* pred; EventHandlerPtr event; - RecordType* event_type; - // ~Filter(); - // Filter(); - // Filter(const InputMgr::Filter& filter); - - void DoCleanup(); + TableFilter(); + ~TableFilter(); }; -/* -InputMgr::Filter::Filter() { +class InputMgr::EventFilter: public InputMgr::Filter { +public: + EventHandlerPtr event; + + RecordType* fields; + unsigned int num_fields; + + bool want_record; + EventFilter(); +}; + +InputMgr::TableFilter::TableFilter() { + filter_type = TABLE_FILTER; + tab = 0; itype = 0; rtype = 0; - event_type = 0; } -InputMgr::Filter::Filter(const InputMgr::Filter& f) { - id = f.id; - id->Ref(); +InputMgr::EventFilter::EventFilter() { + filter_type = EVENT_FILTER; +} - tab = f.tab; - if ( tab ) - tab->Ref(); - - itype = f.itype; - if ( itype ) - itype->Ref(); - - rtype = f.rtype; - if ( rtype ) - Ref(rtype); - - event_type = f.event_type; - if ( event_type ) - Ref(event_type); - - name = f.name; - num_idx_fields = f.num_idx_fields; - num_val_fields = f.num_val_fields; - want_record = f.want_record; - - -} */ - -void InputMgr::Filter::DoCleanup() { +InputMgr::Filter::~Filter() { Unref(id); - if ( tab ) - Unref(tab); - if ( itype ) - Unref(itype); - if ( rtype ) - Unref(rtype); - if ( event_type) - Unref(event_type); +} + +InputMgr::TableFilter::~TableFilter() { + Unref(tab); + Unref(itype); + Unref(rtype); delete currDict; delete lastDict; @@ -111,7 +98,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; //list events; // events we fire when "something" happens - map filters; // filters that can prevent our actions + map filters; // filters that can prevent our actions bool HasFilter(int id); @@ -119,7 +106,11 @@ struct InputMgr::ReaderInfo { }; InputMgr::ReaderInfo::~ReaderInfo() { - // all the contents of filters should delete themselves automatically... + map::iterator it = filters.begin(); + + while ( it != filters.end() ) { + delete (*it).second; + } Unref(type); Unref(id); @@ -128,7 +119,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { } bool InputMgr::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); + map::iterator it = filters.find(id); if ( it == filters.end() ) { return false; } @@ -236,6 +227,114 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } +bool InputMgr::AddEventFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val->AsFunc(); + + { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() < 2 ) { + reporter->Error("event takes not enough arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("events first attribute must be of type Input::Event"); + return false; + } + + if ( want_record->InternalInt() == 0 ) { + if ( args->length() != fields->NumFields() + 1 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + for ( int i = 0; i < fields->NumFields(); i++ ) { + if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + } + + } else if ( want_record->InternalInt() == 1 ) { + if ( args->length() != 2 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + if ( !same_type((*args)[1], fields ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + + } else { + assert(false); + } + + } + + + vector fieldsV; // vector, because UnrollRecordType needs it + + bool status = !UnrollRecordType(&fieldsV, fields, ""); + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + + LogField** logf = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + logf[i] = fieldsV[i]; + } + + EventFilter* filter = new EventFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->num_fields = fieldsV.size(); + filter->fields = fields->Ref()->AsRecordType(); + filter->event = event_registry->Lookup(event->GetID()->Name()); + filter->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), logf ); + + return true; +} + bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -299,6 +398,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { reporter->Error("table events value attribute does not match"); return false; } + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); } @@ -322,28 +422,28 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - filter.num_idx_fields = idxfields; - filter.num_val_fields = valfields; - filter.tab = dst->Ref()->AsTableVal(); - filter.rtype = val->Ref()->AsRecordType(); - filter.itype = idx->Ref()->AsRecordType(); - filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; - filter.currDict = new PDict(InputHash); - filter.lastDict = new PDict(InputHash); - filter.want_record = ( want_record->InternalInt() == 1 ); + TableFilter* filter = new TableFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->num_idx_fields = idxfields; + filter->num_val_fields = valfields; + filter->tab = dst->Ref()->AsTableVal(); + filter->rtype = val->Ref()->AsRecordType(); + filter->itype = idx->Ref()->AsRecordType(); + filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + filter->currDict = new PDict(InputHash); + filter->lastDict = new PDict(InputHash); + filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(filter.want_record); + assert(filter->want_record); } int filterid = 0; if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> } i->filters[filterid] = filter; i->reader->AddFilter( filterid, fieldsV.size(), fields ); @@ -478,13 +578,39 @@ bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); + map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - i->filters[id->InternalInt()].DoCleanup(); + if ( i->filters[id->InternalInt()]->filter_type != TABLE_FILTER ) { + // wrong type; + return false; + } + delete (*it).second; + i->filters.erase(it); + return true; +} + +bool InputMgr::RemoveEventFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + if ( i->filters[id->InternalInt()]->filter_type != EVENT_FILTER ) { + // wrong type; + return false; + } + + delete (*it).second; i->filters.erase(it); return true; } @@ -524,30 +650,53 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + SendEntryTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + bool updated = false; + assert(i); assert(i->HasFilter(id)); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); + HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); + HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->filters[id].lastDict->Lookup(idxhash); + InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->filters[id].lastDict->Remove(idxhash); - i->filters[id].currDict->Insert(idxhash, h); + filter->lastDict->Remove(idxhash); + filter->currDict->Insert(idxhash, h); return; } else { // updated - i->filters[id].lastDict->Remove(idxhash); + filter->lastDict->Remove(idxhash); delete(h); updated = true; @@ -555,30 +704,25 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } - /* if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } */ - r->Assign(j,val); } @@ -589,12 +733,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->filters[id].tab->Lookup(idxval); + oldval = filter->tab->Lookup(idxval); } // call filter first to determine if we really add / change the entry - if ( i->filters[id].pred ) { + if ( filter->pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -609,18 +753,18 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->filters[id].currDict->RemoveEntry(idxhash)); + delete(filter->currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->filters[id].currDict->Insert(idxhash, h); + filter->currDict->Insert(idxhash, h); return; } } @@ -629,24 +773,23 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const //i->tab->Assign(idxval, valval); - HashKey* k = i->filters[id].tab->ComputeHash(idxval); + HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - reporter->Error("assigning"); - i->filters[id].tab->Assign(idxval, k, valval); + filter->tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->filters[id].tab->ComputeHash(idxval); + k = filter->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->filters[id].currDict->Insert(idxhash, ih); + filter->currDict->Insert(idxhash, ih); - if ( i->filters[id].event ) { + if ( filter->event ) { EnumVal* ev; Ref(idxval); @@ -654,11 +797,11 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(i->filters[id].event, ev, idxval, oldval); + SendEvent(filter->event, 3, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(i->filters[id].event, ev, idxval, valval); + SendEvent(filter->event, 3, ev, idxval, valval); } } } @@ -673,25 +816,33 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { assert(i->HasFilter(id)); + if ( i->filters[id]->filter_type == EVENT_FILTER ) { + // nothing to do.. + return; + } + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->filters[id].lastDict->InitForIteration(); - i->filters[id].lastDict->MakeRobustCookie(c); + IterCookie *c = filter->lastDict->InitForIteration(); + filter->lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { + while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { ListVal * idx; Val *val; - if ( i->filters[id].pred || i->filters[id].event ) { - idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + if ( filter->pred || filter->event ) { + idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); - val = i->filters[id].tab->Lookup(idx); + val = filter->tab->Lookup(idx); assert(val != 0); } - if ( i->filters[id].pred ) { + if ( filter->pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -704,37 +855,37 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { vl.append(ev); vl.append(idx); vl.append(val); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... - i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); continue; } } - if ( i->filters[id].event ) { + if ( filter->event ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(i->filters[id].event, ev, idx, val); + SendEvent(filter->event, 3, ev, idx, val); } - i->filters[id].tab->Delete(ih->idxkey); - i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line + filter->tab->Delete(ih->idxkey); + filter->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->filters[id].lastDict); + filter->lastDict->Clear(); // should be empty->->-> but->->-> well->->-> who knows->->-> + delete(filter->lastDict); - i->filters[id].lastDict = i->filters[id].currDict; - i->filters[id].currDict = new PDict(InputHash); + filter->lastDict = filter->currDict; + filter->currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -744,24 +895,86 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + PutTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + bool updated = false; + + assert(i); assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + assert(i->filters[id]->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i->filters[id]; + + Val *val; + list out_vals; + // no tracking, send everything with a new event... + //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); + out_vals.push_back(type); + + int position = 0; + if ( filter->want_record ) { + RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); + out_vals.push_back(r); + + } else { + for ( int j = 0; j < filter->fields->NumFields(); j++) { + Val* val = 0; + if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], filter->fields->FieldType(j)); + position++; + } + out_vals.push_back(val); + } + } + + SendEvent(filter->event, out_vals); + +} + +void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + assert(i); + assert(i->HasFilter(id)); + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } @@ -776,7 +989,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->filters[id].tab->Assign(idxval, valval); + filter->tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -788,7 +1001,10 @@ void InputMgr::Clear(const InputReader* reader, int id) { assert(i->HasFilter(id)); - i->filters[id].tab->RemoveAll(); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + filter->tab->RemoveAll(); } bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { @@ -800,9 +1016,18 @@ bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *va assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); - - return ( i->filters[id].tab->Delete(idxval) != 0 ); + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i->filters[id]; + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + return( filter->tab->Delete(idxval) != 0 ); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + return true; + } else { + assert(false); + return false; + } } void InputMgr::Error(InputReader* reader, const char* msg) @@ -835,25 +1060,35 @@ bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c return true; } -void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, const int numvals, ...) { - //EventHandler* handler = event_registry->Lookup(name.c_str()); - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", name.c_str()); - // return; - //} - val_list* vl = new val_list; - vl->append(event); - vl->append(left); - vl->append(right); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl->append( va_arg(lP, Val*) ); + } + va_end(lP); + + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); +} + +void InputMgr::SendEvent(EventHandlerPtr ev, list events) +{ + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) { + vl->append( *i ); + } - //mgr.Dispatch(new Event(handler, vl)); mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } -Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + +RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { reporter->InternalError("Need position"); return 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index ba1ddafc92..cebed231e4 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -27,6 +27,8 @@ public: bool AddTableFilter(EnumVal *id, RecordVal* filter); bool RemoveTableFilter(EnumVal* id, const string &name); + bool AddEventFilter(EnumVal *id, RecordVal* filter); + bool RemoveEventFilter(EnumVal* id, const string &name); protected: friend class InputReader; @@ -46,10 +48,17 @@ protected: private: struct ReaderInfo; + void SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals); + void PutTable(const InputReader* reader, int id, const LogVal* const *vals); + void SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); + + void SendEvent(EventHandlerPtr ev, const int numvals, ...); + void SendEvent(EventHandlerPtr ev, list events); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); @@ -57,9 +66,8 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); - Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); @@ -68,7 +76,12 @@ private: string Hash(const string &input); - struct Filter; + class Filter; + class TableFilter; + class EventFilter; + + enum FilterType { TABLE_FILTER, EVENT_FILTER }; + }; diff --git a/src/input.bif b/src/input.bif index 1300f91bea..b1d57b1df6 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,6 +9,7 @@ module Input; type StreamDescription: record; type TableFilter: record; +type EventFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -40,6 +41,18 @@ function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool + %{ + bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} + # Options for Ascii Reader module InputAscii; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 36e8171689..d9be733a1b 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -16,7 +16,6 @@ 7 T @TEST-END-FILE -redef InputAscii::empty_field = "EMPTY"; module A; @@ -24,25 +23,20 @@ export { redef enum Log::ID += { LOG }; } -type Idx: record { - i: int; -}; - type Val: record { + i: int; b: bool; }; -global destination: table[int] of Val = table(); - -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(tpe: Input::Event, i: int, b: bool) { print tpe; - print left; - print right; + print i; + print b; } event bro_init() { Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); Input::force_update(A::LOG); } diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From be1b3ce5e1f239861013517b8d8c5a5fafbdf0bd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 13:29:02 -0600 Subject: [PATCH 040/123] Add note about independent component releases to Broxygen index. --- doc/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index ba3df81e7d..0a6b225431 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -46,6 +46,12 @@ Script Reference Other Bro Components -------------------- +The following are snapshots of documentation for components that come +with this version of Bro (|version|). Since they can also be used +independently, see the `download page +`_ for documentation of any +current, independent component releases. + .. toctree:: :maxdepth: 1 From 4975584e01af095cf6a6d10a4c09eb88a3f198e0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 13:28:44 -0800 Subject: [PATCH 041/123] change Log enum to Input enum. --- scripts/base/frameworks/input/main.bro | 10 +++++----- src/input.bif | 10 +++++----- src/types.bif | 1 - testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- testing/btest/scripts/base/frameworks/input/event.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/twofilters.bro | 10 +++++----- 9 files changed, 35 insertions(+), 36 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index a036eeeebd..1a9e8b885c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -58,28 +58,28 @@ module Input; #global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::StreamDescription) : bool +function create_stream(id: Input::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } -function remove_stream(id: Log::ID) : bool +function remove_stream(id: Input::ID) : bool { return __remove_stream(id); } -function force_update(id: Log::ID) : bool +function force_update(id: Input::ID) : bool { return __force_update(id); } -function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool +function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool { # filters[id, filter$name] = filter; return __add_tablefilter(id, filter); } -function remove_tablefilter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_tablefilter(id, name); diff --git a/src/input.bif b/src/input.bif index b1d57b1df6..a7d561c060 100644 --- a/src/input.bif +++ b/src/input.bif @@ -11,31 +11,31 @@ type StreamDescription: record; type TableFilter: record; type EventFilter: record; -function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool +function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_stream%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Input::ID%) : bool %{ bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Log::ID%) : bool +function Input::__force_update%(id: Input::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool +function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool %{ bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool %{ bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); diff --git a/src/types.bif b/src/types.bif index f90a954224..15fab1a7b1 100644 --- a/src/types.bif +++ b/src/types.bif @@ -180,7 +180,6 @@ enum Event %{ EVENT_REMOVED, %} - enum ID %{ Unknown, %} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 3ad45eac69..10cc7376a8 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -45,8 +45,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index d9be733a1b..a07f0934a0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -20,7 +20,7 @@ module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Val: record { @@ -36,7 +36,7 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::force_update(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 134ceb49e6..88838cc8d6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index c07c9c826c..fc4d862cd3 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 769536d2a6..5e6bae7b62 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -37,11 +37,11 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 575665e6e5..5af664e0e9 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -38,13 +38,13 @@ global destination2: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in destination1 ) { print "VALID"; } From 1abb1424b8f1e050f24ffb9a9a7829fd8e10c3f0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 14:15:04 -0800 Subject: [PATCH 042/123] begin documenting... --- doc/index.rst | 1 + doc/input.rst | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 doc/input.rst diff --git a/doc/index.rst b/doc/index.rst index 0a6b225431..975ee7def4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -20,6 +20,7 @@ Frameworks notice logging + input cluster signatures diff --git a/doc/input.rst b/doc/input.rst new file mode 100644 index 0000000000..78e96fe06e --- /dev/null +++ b/doc/input.rst @@ -0,0 +1,190 @@ +===================== +Loading Data into Bro +===================== + +.. rst-class:: opening + + Bro comes with a flexible input interface that allows to read + previously stored data. Data is either read into bro tables or + sent to scripts using events. + This document describes how the input framework can be used. + +.. contents:: + +Terminology +=========== + +Bro's input framework is built around three main abstracts, that are +very similar to the abstracts used in the logging framework: + + Input Streams + An input stream corresponds to a single input source + (usually a textfile). It defined the information necessary + to find the source (e.g. the filename) + + Filters + Each input stream has a set of filters attached to it, that + determine exaclty what kind of information is read. + There are two different kind of streams, event streams and table + streams. + By default, event streams generate an event for each line read + from the input source. + Table streams on the other hand read the input source in a bro + table for easy later access. + + Readers + A reader defines the input format for the specific input stream. + At the moment, Bro comes with only one type of reader, which can + read the tab seperated ASCII logfiles that were generated by the + logging framework. + + +Basics +====== + +For examples, please look at the unit tests in +``testing/btest/scripts/base/frameworks/input/``. + +A very basic example to open an input stream is: + +.. code:: bro + + module Foo; + + export { + # Create an ID for our new stream + redef enum Input::ID += { INPUT }; + } + + event bro_init() { + Input::create_stream(FOO::INPUT, [$source="input.log"]); + } + +The fields that can be set when creating a stream are: + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + +Filters +======= + +Each filter defines the data fields that it wants to receive from the respective +input file. Depending on the type of filter, events or a table are created from +the data in the source file. + +Event Filters +------------- + +Event filters are filters that generate an event for each line in of the input source. + +For example, a simple filter retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: + +.. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(tpe: Input::Event, i: int, b: bool) { + # work with event data + } + + event bro_init { + # Input stream definition, etc + ... + + Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The fields that can be set for an event filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``fields`` + Name of a record type containing the fields, which should be retrieved from + the input stream. + + ``ev`` + The event which is fired, after a line has been read from the input source. + The first argument that is passed to the event is an Input::Event structure, + followed by the data, either inside of a record (if ``want_record is set``) or as + individual fields. + The Input::Event structure can contain information, if the received line is ``NEW``, has + been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information + for event filters, the value is always ``NEW`` at the moment. + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + +Table Filters +------------- + +Table filters are the second, more complex type of filter. + +Table filters store the information they read from an input source in a bro table. For example, +when reading a file that contains ip addresses and connection attemt information one could use +an approach similar to this: + +.. code:: bro + + type Idx: record { + a: addr; + }; + + type Val: record { + tries: count; + }; + + global conn_attempts: table[addr] of count = table(); + + event bro_init { + # Input stream definitions, etc. + ... + + Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The table conn_attempts will then contain the information about connection attemps. + +The possible fields that can be set for an table filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``idx`` + Record type that defines the index of the table + + ``val`` + Record type that defines the values of the table + + ``want_record`` + Defines if the values of the table should be stored as a record (default), + or as a simple value. Has to be set if Val contains more than one element. + + ``destination`` + The destination table + + ``ev`` + Optional event that is raised, when values are added to, changed in or deleted from the table. + Events are passed an Input::Event description as the first argument, the index record as the second argument + and the values as the third argument. + + ``pred`` + Optional predicate, that can prevent entries from being added to the table and events from being sent. From 2a6387129ceb68941beccb866822958c661ab2c4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 11:25:11 -0800 Subject: [PATCH 043/123] documentation --- doc/scripts/DocSourcesList.cmake | 3 + scripts/base/frameworks/input/main.bro | 96 ++++++++++++++++++++------ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 9d99effc02..bd41d301c0 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro) @@ -31,6 +32,8 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) +rest_target(${psd} base/frameworks/input/main.bro) +rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 1a9e8b885c..66b13743b8 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -1,52 +1,106 @@ +##! The input framework provides a way to read previously stored data either +##! as an event stream or into a bro table. module Input; export { + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + ## Stream decription type used for the `create_stream` method type StreamDescription: record { + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. source: string; + + ## Reader to use for this steam reader: Reader &default=default_reader; }; + ## TableFilter description type used for the `add_tablefilter` method. type TableFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - ## for tables - idx: any; - val: any; + ## Table which will contain the data read by the input framework destination: any; + ## Record that defines the values used as the index of the table + idx: any; + ## Record that defines the values used as the values of the table + val: any; + ## Defines if the value of the table is a record (default), or a single value. + ## Val can only contain one element when this is set to false. want_record: bool &default=T; + + ## The event that is raised each time a value is added to, changed in or removed from the table. + ## The event will receive an Input::Event enum as the first argument, the idx record as the second argument + ## and the value (record) as the third argument. ev: any &optional; # event containing idx, val as values. - ## decision function, that decides if an insertion, update or removal should really be executed. - ## or events should be thought + ## Predicate function, that can decide if an insertion, update or removal should really be executed. + ## Parameters are the same as for the event. If true is returned, the update is performed. If false + ## is returned, it is skipped pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; + ## EventFilter description type used for the `add_eventfilter` method. type EventFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - # the event - ev: any; - # record describing the fields + ## Record describing the fields to be retrieved from the source input. fields: any; - - # does the event want the field unrolled (default) or as a simple record value? + ## If want_record if false (default), the event receives each value in fields as a seperate argument. + ## If it is set to true, the event receives all fields in a signle record value. want_record: bool &default=F; + + ## The event that is rised each time a new line is received from the reader. + ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. + ev: any; + }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; - global remove_stream: function(id: Log::ID) : bool; - global force_update: function(id: Log::ID) : bool; - global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; - global remove_tablefilter: function(id: Log::ID, name: string) : bool; - global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; - global remove_eventfilter: function(id: Log::ID, name: string) : bool; + ## Create a new input stream from a given source. Returns true on success. + ## + ## id: `Input::ID` enum value identifying this stream + ## description: `StreamDescription` record describing the source. + global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; + + ## Remove a current input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream to be removed + global remove_stream: function(id: Input::ID) : bool; + + ## Forces the current input to be checked for changes. + ## + ## id: `Input::ID` enum value identifying the stream + global force_update: function(id: Input::ID) : bool; + + ## Adds a table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `TableFilter` record describing the filter. + global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; + + ## Removes a named table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_tablefilter: function(id: Input::ID, name: string) : bool; + + ## Adds an event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `EventFilter` record describing the filter. + global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; + + ## Removes a named event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -85,13 +139,13 @@ function remove_tablefilter(id: Input::ID, name: string) : bool return __remove_tablefilter(id, name); } -function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool +function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool { # filters[id, filter$name] = filter; return __add_eventfilter(id, filter); } -function remove_eventfilter(id: Log::ID, name: string) : bool +function remove_eventfilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_eventfilter(id, name); From a68e6b9fa49211a7c51c45f83ec0c610f03a956d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 14:32:53 -0800 Subject: [PATCH 044/123] allow sets to be read from files, convenience function for reading a file once, bug in destructor that could lead to a segfault. --- scripts/base/frameworks/input/main.bro | 34 +++++++++++- src/InputMgr.cc | 54 ++++++++++++++----- .../scripts/base/frameworks/input/basic.bro | 2 + 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 66b13743b8..c76eba80b9 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,6 +4,8 @@ module Input; export { + redef enum Input::ID += { TABLE_READ }; + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -27,7 +29,8 @@ export { ## Record that defines the values used as the index of the table idx: any; ## Record that defines the values used as the values of the table - val: any; + ## If val is undefined, destination has to be a set. + val: any &optional; ## Defines if the value of the table is a record (default), or a single value. ## Val can only contain one element when this is set to false. want_record: bool &default=T; @@ -102,6 +105,14 @@ export { ## name: the name of the filter to be removed. global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; + + ## Convenience function for reading a specific input source exactly once using + ## exactly one tablefilter + ## + ## id: `Input::ID` enum value identifying the stream + ## description: `StreamDescription` record describing the source. + ## filter: the `TableFilter` record describing the filter. + global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; } @@ -151,6 +162,27 @@ function remove_eventfilter(id: Input::ID, name: string) : bool return __remove_eventfilter(id, name); } +function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { + local ok: bool = T; + # since we create and delete it ourselves this should be ok... at least for singlethreaded operation + local id: Input::ID = Input::TABLE_READ; + + ok = create_stream(id, description); + if ( ok ) { + ok = add_tablefilter(id, filter); + } + if ( ok ) { + ok = force_update(id); + } + if ( ok ) { + ok = remove_stream(id); + } else { + remove_stream(id); + } + + return ok; +} + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0df11ea359..7ec5974199 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -86,7 +86,8 @@ InputMgr::Filter::~Filter() { InputMgr::TableFilter::~TableFilter() { Unref(tab); Unref(itype); - Unref(rtype); + if ( rtype ) // can be 0 for sets + Unref(rtype); delete currDict; delete lastDict; @@ -110,6 +111,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { while ( it != filters.end() ) { delete (*it).second; + ++it; } Unref(type); @@ -354,7 +356,10 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { Val* pred = fval->Lookup(rtype->FieldOffset("pred")); RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = 0; + if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + } TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -408,9 +413,14 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { int idxfields = fieldsV.size(); - status = status || !UnrollRecordType(&fieldsV, val, ""); + if ( val ) // if we are not a set + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + if ( !val ) + assert(valfields == 0); + if ( status ) { reporter->Error("Problem unrolling"); return false; @@ -429,7 +439,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; filter->tab = dst->Ref()->AsTableVal(); - filter->rtype = val->Ref()->AsRecordType(); + filter->rtype = val ? val->Ref()->AsRecordType() : 0; filter->itype = idx->Ref()->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); @@ -681,7 +691,10 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + HashKey* valhash = 0; + if ( filter->num_val_fields > 0 ) + HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -689,12 +702,13 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( h->valhash->Hash() == valhash->Hash() ) { - // ok, double. + if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { + // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); return; } else { + assert( filter->num_val_fields > 0 ); // updated filter->lastDict->Remove(idxhash); delete(h); @@ -708,7 +722,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(filter->rtype); @@ -732,8 +748,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* oldval = 0; if ( updated == true ) { - // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); + assert(filter->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval); } @@ -749,10 +766,12 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } - val_list vl(3); + val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); vl.append(idxval); - vl.append(valval); + if ( filter->num_val_fields > 0 ) + vl.append(valval); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); @@ -794,6 +813,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Ref(idxval); if ( updated ) { // in case of update send back the old value. + assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); @@ -801,7 +821,11 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(filter->event, 3, ev, idxval, valval); + if ( filter->num_val_fields == 0 ) { + SendEvent(filter->event, 3, ev, idxval); + } else { + SendEvent(filter->event, 3, ev, idxval, valval); + } } } } @@ -963,7 +987,9 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { RecordVal * r = new RecordVal(filter->rtype); diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 10cc7376a8..d1b6659eb6 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -49,4 +49,6 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); } From 78b24da7e4f84de7f499cc9365a30e7c966a7d14 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 15:02:03 -0800 Subject: [PATCH 045/123] start support for annotation for log field types. commit before rolling part of it back... --- src/Attr.h | 1 + src/InputMgr.cc | 10 ++++++++++ src/InputReaderAscii.h | 3 +++ src/LogMgr.cc | 8 +++++--- src/LogMgr.h | 6 ++++-- src/parse.y | 5 ++++- src/scan.l | 1 + 7 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/Attr.h b/src/Attr.h index 6c835dc61c..471acfe4ba 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_GROUP, ATTR_LOG, ATTR_ERROR_HANDLER, + ATTR_TYPE_COLUMN, // for input framework ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ec5974199..0cfb59ee90 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -561,6 +561,16 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); } else if ( field->type == TYPE_VECTOR ) { field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); + } else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { + // we have an annotation for the second column + + Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); + + assert(c); + assert(c->Type()->Tag() == TYPE_STRING); + + field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } fields->push_back(field); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 01169a3cfc..c174248454 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,8 +12,11 @@ struct FieldMapping { string name; TypeTag type; + // internal type for sets and vectors TypeTag subtype; int position; + // for ports: pos of the second field + int secondary_position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 7fc8d4ef86..dd4f9c8e5a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,7 +83,8 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && + fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; subtype = (TypeTag) it; @@ -92,7 +93,8 @@ bool LogField::Read(SerializationFormat* fmt) bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && + fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() @@ -151,7 +153,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t->IsSet() ) return false; - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); } case TYPE_VECTOR: diff --git a/src/LogMgr.h b/src/LogMgr.h index b8530d29ab..4ccdeb793c 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -14,13 +14,15 @@ class SerializationFormat; // Description of a log field. struct LogField { string name; + // needed by input framework. port fields have two names (one for the port, one for the type) - this specifies the secondary name. + string secondary_name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + // needed by input framework. otherwise it cannot determine the inner type of a set. TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), secondary_name(other.secondary_name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); diff --git a/src/parse.y b/src/parse.y index 495931aae0..988a19714b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 88 +%expect 91 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,6 +24,7 @@ %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER +%token TOK_ATTR_TYPE_COLUMN %token TOK_DEBUG @@ -1313,6 +1314,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_TYPE_COLUMN '=' expr + { $$ = new Attr(ATTR_TYPE_COLUMN, $3); } | TOK_ATTR_LOG { $$ = new Attr(ATTR_LOG); } | TOK_ATTR_ERROR_HANDLER diff --git a/src/scan.l b/src/scan.l index 7ebd7894e1..6a85e89780 100644 --- a/src/scan.l +++ b/src/scan.l @@ -308,6 +308,7 @@ when return TOK_WHEN; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; &priority return TOK_ATTR_PRIORITY; +&type_column return TOK_ATTR_TYPE_COLUMN; &read_expire return TOK_ATTR_EXPIRE_READ; &redef return TOK_ATTR_REDEF; &rotate_interval return TOK_ATTR_ROTATE_INTERVAL; From aecbbdd966c896b475591a3415578adc1629273e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 046/123] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index dd4f9c8e5a..ed32e4e40b 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -118,6 +118,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -190,9 +194,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -305,9 +312,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1066,6 +1075,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1097,7 +1122,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 4ccdeb793c..d9d61236e0 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -38,10 +38,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -136,6 +138,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From 4a690484ecaf5ea086090d2b6ef855cc6c913cad Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:42:37 -0800 Subject: [PATCH 047/123] make port annotation work and ascii input reader way more rebust with better error messages. --- src/Attr.cc | 22 ++- src/InputMgr.cc | 44 ++++- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 150 +++++++++--------- src/InputReaderAscii.h | 4 +- src/LogMgr.cc | 1 - .../scripts.base.frameworks.input.port/out | 6 + .../scripts/base/frameworks/input/port.bro | 39 +++++ 8 files changed, 187 insertions(+), 81 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.port/out create mode 100644 testing/btest/scripts/base/frameworks/input/port.bro diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..1d610f7cb4 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -17,7 +17,7 @@ const char* attr_name(attr_tag t) "&persistent", "&synchronized", "&postprocessor", "&encrypt", "&match", "&disable_print_hook", "&raw_output", "&mergeable", "&priority", - "&group", "&log", "&error_handler", "(&tracked)", + "&group", "&log", "&error_handler", "&type_column", "(&tracked)", }; return attr_names[int(t)]; @@ -417,6 +417,26 @@ void Attributes::CheckAttr(Attr* a) Error("&log applied to a type that cannot be logged"); break; + case ATTR_TYPE_COLUMN: + { + if ( type->Tag() != TYPE_PORT ) + { + Error("type_column tag only applicable to ports"); + break; + } + + BroType* atype = a->AttrExpr()->Type(); + + if ( atype->Tag() != TYPE_STRING ) { + Error("type column needs to have a string argument"); + break; + } + + + break; + } + + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0cfb59ee90..612461bae8 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -1167,9 +1167,14 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: length += sizeof(val->val.uint_val); break; + + case TYPE_PORT: + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) + length += val->val.port_val.proto->size(); + break; case TYPE_DOUBLE: case TYPE_TIME: @@ -1228,12 +1233,24 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; + case TYPE_PORT: { + int length = 0; + memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) { + memcpy(data+startpos, val->val.port_val.proto->c_str(), val->val.port_val.proto->length()); + length += val->val.port_val.proto->size(); + } + return length; + break; + } + + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: @@ -1320,6 +1337,24 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } +TransportProto InputMgr::StringToProto(const string &proto) { + if ( proto == "unknown" ) { + return TRANSPORT_UNKNOWN; + } else if ( proto == "tcp" ) { + return TRANSPORT_TCP; + } else if ( proto == "udp" ) { + return TRANSPORT_UDP; + } else if ( proto == "icmp" ) { + return TRANSPORT_ICMP; + } + + //assert(false); + + reporter->Error("Tried to parse invalid/unknown protocol: %s", proto.c_str()); + + return TRANSPORT_UNKNOWN; +} + Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1357,7 +1392,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { } case TYPE_PORT: - return new PortVal(val->val.uint_val); + if ( val->val.port_val.proto == 0 ) + return new PortVal(val->val.port_val.port); + else + return new PortVal(val->val.port_val.port, StringToProto(*val->val.port_val.proto)); break; case TYPE_ADDR: diff --git a/src/InputMgr.h b/src/InputMgr.h index cebed231e4..ba6e208fd1 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -68,6 +68,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + TransportProto StringToProto(const string &proto); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 84feb74e61..501022d58e 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -10,25 +10,27 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int : name(arg_name), type(arg_type) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; + secondary_position = arg.secondary_position; } FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } - InputReaderAscii::InputReaderAscii() { file = 0; @@ -122,45 +124,45 @@ bool InputReaderAscii::ReadHeader() { return false; } + map fields; + + // construcr list of field names. + istringstream splitstream(line); + int pos=0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + fields[s] = pos; + pos++; + } + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const LogField* field = (*it).second.fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - (*it).second.columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + + map::iterator fit = fields.find(field->name); + if ( fit == fields.end() ) { + Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + return false; + } + + + FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = fields.find(field->secondary_name); + if ( fit2 == fields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; } + f.secondary_position = fields[field->secondary_name]; } - - // look if we did push something... - if ( (*it).second.columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - (*it).second.columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) (*it).second.num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); - return false; + (*it).second.columnMap.push_back(f); } + } // well, that seems to have worked... @@ -220,10 +222,14 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_PORT: + val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.proto = 0; + break; + case TYPE_SUBNET: { int pos = s.find("/"); string width = s.substr(pos+1); @@ -346,59 +352,55 @@ bool InputReaderAscii::DoUpdate() { string line; while ( GetLine(line ) ) { + // split on tabs + istringstream splitstream(line); + + map stringfields; + int pos = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + stringfields[pos] = s; + pos++; + } + + pos--; // for easy comparisons of max element. for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs - - istringstream splitstream(line); - LogVal** fields = new LogVal*[(*it).second.num_fields]; - //string string_fields[num_fields]; - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { + int fpos = 0; + for ( vector::iterator fit = (*it).second.columnMap.begin(); + fit != (*it).second.columnMap.end(); + fit++ ){ - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - - - if ( currTab >= (*it).second.columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; } - FieldMapping currMapping = (*it).second.columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= (*it).second.num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); + LogVal* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { return false; } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - currField++; - } - - if ( currField != (*it).second.num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; + val->val.port_val.proto = new string(stringfields[(*fit).secondary_position]); + } + + fields[fpos] = val; + + fpos++; } + assert ( (unsigned int) fpos == (*it).second.num_fields ); SendEntry((*it).first, fields); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c174248454..2670d785d5 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -21,10 +21,10 @@ struct FieldMapping { FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); - FieldMapping() { position = -1; } + FieldMapping() { position = -1; secondary_position = -1; } FieldMapping subType(); - bool IsEmpty() { return position == -1; } + //bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ed32e4e40b..307a2e24e8 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -1088,7 +1088,6 @@ string LogMgr::TransportProtoToString(TransportProto p) { } assert(false); - return ""; } LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out new file mode 100644 index 0000000000..6f2bd3271b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -0,0 +1,6 @@ +Trying to find field: t +{ +[1.2.3.4] = [p=80/tcp], +[1.2.4.6] = [p=30/unknown], +[1.2.3.5] = [p=52/udp] +} diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro new file mode 100644 index 0000000000..6f98c363c7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -0,0 +1,39 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#fields i p t +1.2.3.4 80 tcp +1.2.3.5 52 udp +1.2.4.6 30 unknown +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: addr; +}; + +type Val: record { + p: port &type_column="t"; +}; + +global servers: table[addr] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); + print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); +} From 9f32f68a13f290f01747f02b05dbe4b8b63d2790 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:50:36 -0800 Subject: [PATCH 048/123] make test more robust. --- .../Baseline/scripts.base.frameworks.input.port/out | 9 +++------ testing/btest/scripts/base/frameworks/input/port.bro | 10 ++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out index 6f2bd3271b..858551aa2f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.port/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -1,6 +1,3 @@ -Trying to find field: t -{ -[1.2.3.4] = [p=80/tcp], -[1.2.4.6] = [p=30/unknown], -[1.2.3.5] = [p=52/udp] -} +[p=80/tcp] +[p=52/udp] +[p=30/unknown] diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 6f98c363c7..c14892ae36 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -6,7 +6,7 @@ #fields i p t 1.2.3.4 80 tcp 1.2.3.5 52 udp -1.2.4.6 30 unknown +1.2.3.6 30 unknown @TEST-END-FILE redef InputAscii::empty_field = "EMPTY"; @@ -31,9 +31,11 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); - print servers; - Input::remove_tablefilter(A::INPUT, "ssh"); + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; + Input::remove_tablefilter(A::INPUT, "input"); Input::remove_stream(A::INPUT); } From eb64eeedcd27c543becf330d7da48db9aa541ea8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 049/123] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 307a2e24e8..dd5bc4b66a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -119,8 +119,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From e0b7dc04512a228b332e2b9aaddd39300646d5cc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 14:12:59 -0800 Subject: [PATCH 050/123] fix compile warnings --- src/InputMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 00a1a26311..1fe3a82abe 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -866,8 +866,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal * idx; - Val *val; + ListVal * idx = 0; + Val *val = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); From a14ec02d3b75788966db7dccb1e8a533e627300a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:25 -0800 Subject: [PATCH 051/123] change empty field defenition like in logging framework --- scripts/base/frameworks/input/readers/ascii.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 9f630975a2..14c04757f7 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -12,7 +12,7 @@ export { const set_separator = "," &redef; ## String to use for empty fields. - const empty_field = "-" &redef; + const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. const unset_field = "-" &redef; From 70a2cf67324c607014b067af13fc9c6887e571e3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:51 -0800 Subject: [PATCH 052/123] update baseline to include input framework --- .../coverage.bare-load-baseline/canonified_loaded_scripts.log | 4 ++++ .../canonified_loaded_scripts.log | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fab67304e..5a0de92a6e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -18,4 +18,8 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 3f77797df8..e727928579 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -18,6 +18,10 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/./patterns.bro From eb53a3d1c8bb9b7a590fc4f5ba70a5ea5017b6ee Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 12 Jan 2012 11:51:12 -0800 Subject: [PATCH 053/123] make input framework compile with brov6 --- src/InputReaderAscii.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 6da161a2bf..257cb4cf71 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -253,19 +253,37 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // NOTE: dotted_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... +#ifdef BROv6 + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.subnet_val.net, addr); + delete addr; + } else { + val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = 0; + val->val.subnet_val.net[3] = dotted_to_addr(s.c_str()); + } +#else val->val.subnet_val.net = dotted_to_addr(s.c_str()); +#endif break; } case TYPE_ADDR: { // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 - copy_addr(t, val->val.addr_val); + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.addr_val, addr); + delete addr; + } else { + val->val.addr_val[0] = val->val.addr_val[1] = val->val.addr_val[2] = 0; + val->val.addr_val[3] = dotted_to_addr(s.c_str()); + } #else + uint32 t = dotted_to_addr(s.c_str()); copy_addr(&t, val->val.addr_val); #endif break; From ac1708f8436126d1be5b522461dc88c8e1fa6158 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 20 Jan 2012 12:33:48 -0800 Subject: [PATCH 054/123] fix handling of predicates - now the second argument that is sent to the predicate really is a recordVal and not a ListVal. --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++------------------- src/InputMgr.h | 1 + 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 1fe3a82abe..0e01bd6333 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -737,22 +737,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(filter->rtype); - - for ( int j = 0; j < filter->rtype->NumFields(); j++) { - - Val* val = 0; - if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); - } else { - val = LogValToVal(vals[position], filter->rtype->FieldType(j)); - position++; - } - - r->Assign(j,val); - - } - valval = r; + valval = LogValToRecordVal(vals, filter->rtype, &position); } @@ -767,7 +752,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c // call filter first to determine if we really add / change the entry if ( filter->pred ) { EnumVal* ev; - Ref(idxval); + //Ref(idxval); + int startpos = 0; + Val* predidx = LogValToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -778,7 +765,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); - vl.append(idxval); + vl.append(predidx); if ( filter->num_val_fields > 0 ) vl.append(valval); @@ -882,12 +869,14 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { // ask predicate, if we want to expire this element... EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); + //Ref(idx); + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); Ref(val); val_list vl(3); vl.append(ev); - vl.append(idx); + vl.append(predidx); vl.append(val); Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); @@ -1123,6 +1112,29 @@ void InputMgr::SendEvent(EventHandlerPtr ev, list events) } +RecordVal* InputMgr::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + + int maxpos = list->Length(); + + for ( int i = 0; i < request_type->NumFields(); i++ ) { + assert ( (*position) <= maxpos ); + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = list->Index(*position); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; +} + + RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index 21af018a47..5623c15338 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -67,6 +67,7 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); From f24c50b49a88b22db612d098489eeac042122a76 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 20 Jan 2012 12:42:23 -0800 Subject: [PATCH 055/123] remove unnecessary stuff from function. --- src/InputMgr.cc | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0e01bd6333..89530febe6 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -955,7 +955,6 @@ void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, in if ( filter->want_record ) { RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); - } else { for ( int j = 0; j < filter->fields->NumFields(); j++) { Val* val = 0; @@ -991,27 +990,7 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(filter->rtype); - - for ( int j = 0; j < filter->rtype->NumFields(); j++) { - - Val* val = 0; - if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); - } else { - val = LogValToVal(vals[position], filter->rtype->FieldType(j)); - position++; - } - - if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } - - r->Assign(j,val); - - } - valval = r; + valval = LogValToRecordVal(vals, filter->rtype, &position); } filter->tab->Assign(idxval, valval); From 833e7244000c1a1d13b67c9087ada12686bf4b98 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 16:14:39 -0800 Subject: [PATCH 056/123] way less compile errors. --- src/input/Manager.cc | 191 +++++++++++++++------------- src/input/Manager.h | 6 +- src/input/ReaderBackend.cc | 95 +++++++++----- src/input/ReaderBackend.h | 23 ++-- src/input/ReaderFrontend.h | 54 ++++++++ src/input/readers/Ascii.h | 29 +++-- src/logging/Manager.cc | 2 +- src/threading/SerializationTypes.cc | 6 +- src/threading/SerializationTypes.h | 2 + 9 files changed, 257 insertions(+), 151 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 24bc464daf..a97286162d 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -3,18 +3,20 @@ #include #include "Manager.h" +#include "ReaderFrontend.h" +#include "ReaderBackend.h" +#include "readers/Ascii.h" + #include "Event.h" #include "EventHandler.h" #include "NetVar.h" #include "Net.h" -#include "InputReader.h" - -#include "InputReaderAscii.h" - #include "CompHash.h" +#include "../threading/SerializationTypes.h" + using namespace input; using threading::Value; using threading::Field; @@ -99,7 +101,7 @@ Manager::TableFilter::~TableFilter() { struct Manager::ReaderInfo { EnumVal* id; EnumVal* type; - InputReader* reader; + ReaderFrontend* reader; //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions @@ -132,38 +134,27 @@ bool Manager::ReaderInfo::HasFilter(int id) { } -struct InputReaderDefinition { +struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages bool (*init)(); // optional one-time inifializing function - InputReader* (*factory)(); // factory function for creating instances + ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances }; -InputReaderDefinition input_readers[] = { - { BifEnum::Input::READER_ASCII, "Ascii", 0, InputReaderAscii::Instantiate }, +ReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, // End marker - { BifEnum::Input::READER_DEFAULT, "None", 0, (InputReader* (*)())0 } + { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } }; Manager::Manager() { } -// create a new input reader object to be used at whomevers leisure lateron. -InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) -{ - InputReaderDefinition* ir = input_readers; - - RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) - { - reporter->Error("Streamdescription argument not of right type"); - return 0; - } +ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { + ReaderDefinition* ir = input_readers; - EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { @@ -171,7 +162,7 @@ InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) return 0; } - if ( ir->type != reader->AsEnum() ) { + if ( ir->type != type ) { // no, didn't find the right one... ++ir; continue; @@ -201,9 +192,30 @@ InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) // all done. break. break; } - assert(ir->factory); - InputReader* reader_obj = (*ir->factory)(); + + ReaderBackend* backend = (*ir->factory)(frontend); + assert(backend); + + frontend->ty_name = ir->name; + return backend; +} + +// create a new input reader object to be used at whomevers leisure lateron. +ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) +{ + ReaderDefinition* ir = input_readers; + + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) + { + reporter->Error("Streamdescription argument not of right type"); + return 0; + } + + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); + + ReaderFrontend* reader_obj = new ReaderFrontend(id->AsEnum()); assert(reader_obj); // get the source... @@ -217,16 +229,16 @@ InputReader* Manager::CreateStream(EnumVal* id, RecordVal* description) readers.push_back(info); - int success = reader_obj->Init(source); - if ( success == false ) { + reader_obj->Init(source); + /* if ( success == false ) { assert( RemoveStream(id) ); return 0; - } - success = reader_obj->Update(); - if ( success == false ) { + } */ + reader_obj->Update(); + /* if ( success == false ) { assert ( RemoveStream(id) ); return 0; - } + } */ return reader_obj; @@ -306,7 +318,7 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { } - vector fieldsV; // vector, because UnrollRecordType needs it + vector fieldsV; // vector, because UnrollRecordType needs it bool status = !UnrollRecordType(&fieldsV, fields, ""); @@ -316,7 +328,7 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { } - LogField** logf = new LogField*[fieldsV.size()]; + Field** logf = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { logf[i] = fieldsV[i]; } @@ -410,7 +422,7 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } - vector fieldsV; // vector, because we don't know the length beforehands + vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); @@ -430,7 +442,7 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } - LogField** fields = new LogField*[fieldsV.size()]; + Field** fields = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { fields[i] = fieldsV[i]; } @@ -538,12 +550,12 @@ bool Manager::RemoveStream(const EnumVal* id) { return true; } -bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { +bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { if ( !IsCompatibleType(rec->FieldType(i)) ) { - reporter->Error("Incompatible type \"%s\" in table definition for InputReader", type_name(rec->FieldType(i)->Tag())); + reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; } @@ -557,7 +569,7 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, } } else { - LogField* field = new LogField(); + Field* field = new Field(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); if ( field->type == TYPE_TABLE ) { @@ -591,7 +603,9 @@ bool Manager::ForceUpdate(const EnumVal* id) return false; } - return i->reader->Update(); + i->reader->Update(); + + return true; // update is async :( } bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { @@ -638,21 +652,21 @@ bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { return true; } -Val* Manager::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { +Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { - idxval = LogValToVal(vals[0], type->FieldType(0)); + idxval = ValueToVal(vals[0], type->FieldType(0)); position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < type->NumFields(); j++ ) { if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { - l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); + l->Append(ValueToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); } else { - l->Append(LogValToVal(vals[position], type->FieldType(j))); + l->Append(ValueToVal(vals[position], type->FieldType(j))); position++; } } @@ -666,7 +680,7 @@ Val* Manager::LogValToIndexVal(int num_fields, const RecordType *type, const Log } -void Manager::SendEntry(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::SendEntry(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -689,7 +703,7 @@ void Manager::SendEntry(const InputReader* reader, int id, const LogVal* const * } -void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::SendEntryTable(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -701,12 +715,12 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co TableFilter* filter = (TableFilter*) i->filters[id]; //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); + HashKey* idxhash = HashValues(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = 0; if ( filter->num_val_fields > 0 ) - HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + HashValues(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -731,16 +745,16 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co } - Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { valval = 0; } else if ( filter->num_val_fields == 1 && !filter->want_record ) { - valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); + valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); } else { - valval = LogValToRecordVal(vals, filter->rtype, &position); + valval = ValueToRecordVal(vals, filter->rtype, &position); } @@ -757,7 +771,7 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co EnumVal* ev; //Ref(idxval); int startpos = 0; - Val* predidx = LogValToRecordVal(vals, filter->itype, &startpos); + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -831,7 +845,7 @@ void Manager::SendEntryTable(const InputReader* reader, int id, const LogVal* co } -void Manager::EndCurrentSend(const InputReader* reader, int id) { +void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -914,7 +928,7 @@ void Manager::EndCurrentSend(const InputReader* reader, int id) { filter->currDict = new PDict(InputHash); } -void Manager::Put(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::Put(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -937,7 +951,7 @@ void Manager::Put(const InputReader* reader, int id, const LogVal* const *vals) } -void Manager::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals) { +void Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -956,15 +970,15 @@ void Manager::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int int position = 0; if ( filter->want_record ) { - RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); + RecordVal * r = ValueToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); } else { for ( int j = 0; j < filter->fields->NumFields(); j++) { Val* val = 0; if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); + val = ValueToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], filter->fields->FieldType(j)); + val = ValueToVal(vals[position], filter->fields->FieldType(j)); position++; } out_vals.push_back(val); @@ -975,7 +989,7 @@ void Manager::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int } -void Manager::PutTable(const InputReader* reader, int id, const LogVal* const *vals) { +void Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); assert(i); @@ -984,22 +998,22 @@ void Manager::PutTable(const InputReader* reader, int id, const LogVal* const *v assert(i->filters[id]->filter_type == TABLE_FILTER); TableFilter* filter = (TableFilter*) i->filters[id]; - Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { valval = 0; } else if ( filter->num_val_fields == 1 && !filter->want_record ) { - valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); + valval = ValueToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - valval = LogValToRecordVal(vals, filter->rtype, &position); + valval = ValueToRecordVal(vals, filter->rtype, &position); } filter->tab->Assign(idxval, valval); } -void Manager::Clear(const InputReader* reader, int id) { +void Manager::Clear(const ReaderFrontend* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -1014,7 +1028,7 @@ void Manager::Clear(const InputReader* reader, int id) { filter->tab->RemoveAll(); } -bool Manager::Delete(const InputReader* reader, int id, const LogVal* const *vals) { +bool Manager::Delete(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -1025,7 +1039,7 @@ bool Manager::Delete(const InputReader* reader, int id, const LogVal* const *val if ( i->filters[id]->filter_type == TABLE_FILTER ) { TableFilter* filter = (TableFilter*) i->filters[id]; - Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); return( filter->tab->Delete(idxval) != 0 ); } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); @@ -1037,12 +1051,12 @@ bool Manager::Delete(const InputReader* reader, int id, const LogVal* const *val } } -void Manager::Error(InputReader* reader, const char* msg) +void Manager::Error(ReaderFrontend* reader, const char* msg) { reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -bool Manager::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool Manager::SendEvent(const string& name, const int num_vals, const Value* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { @@ -1059,7 +1073,7 @@ bool Manager::SendEvent(const string& name, const int num_vals, const LogVal* co val_list* vl = new val_list; for ( int i = 0; i < num_vals; i++) { - vl->append(LogValToVal(vals[i], type->FieldType(i))); + vl->append(ValueToVal(vals[i], type->FieldType(i))); } mgr.Dispatch(new Event(handler, vl)); @@ -1118,7 +1132,7 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, -RecordVal* Manager::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { +RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { reporter->InternalError("Need position"); return 0; @@ -1136,9 +1150,9 @@ RecordVal* Manager::LogValToRecordVal(const LogVal* const *vals, RecordType *req Val* fieldVal = 0; if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { - fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); } else { - fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)); + fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); (*position)++; } @@ -1150,7 +1164,7 @@ RecordVal* Manager::LogValToRecordVal(const LogVal* const *vals, RecordType *req } -int Manager::GetLogValLength(const LogVal* val) { +int Manager::GetValueLength(const Value* val) { int length = 0; switch (val->type) { @@ -1193,7 +1207,7 @@ int Manager::GetLogValLength(const LogVal* val) { case TYPE_TABLE: { for ( int i = 0; i < val->val.set_val.size; i++ ) { - length += GetLogValLength(val->val.set_val.vals[i]); + length += GetValueLength(val->val.set_val.vals[i]); } break; } @@ -1201,20 +1215,20 @@ int Manager::GetLogValLength(const LogVal* val) { case TYPE_VECTOR: { int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - length += GetLogValLength(val->val.vector_val.vals[i]); + length += GetValueLength(val->val.vector_val.vals[i]); } break; } default: - reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + reporter->InternalError("unsupported type %d for GetValueLength", val->type); } return length; } -int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { +int Manager::CopyValue(char *data, const int startpos, const Value* val) { switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: @@ -1276,7 +1290,7 @@ int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_TABLE: { int length = 0; for ( int i = 0; i < val->val.set_val.size; i++ ) { - length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); } return length; break; @@ -1286,14 +1300,14 @@ int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { int length = 0; int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); + length += CopyValue(data, startpos+length, val->val.vector_val.vals[i]); } return length; break; } default: - reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + reporter->InternalError("unsupported type %d for CopyValue", val->type); return 0; } @@ -1302,12 +1316,12 @@ int Manager::CopyLogVal(char *data, const int startpos, const LogVal* val) { } -HashKey* Manager::HashLogVals(const int num_elements, const LogVal* const *vals) { +HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { - const LogVal* val = vals[i]; - length += GetLogValLength(val); + const Value* val = vals[i]; + length += GetValueLength(val); } //reporter->Error("Length: %d", length); @@ -1318,8 +1332,8 @@ HashKey* Manager::HashLogVals(const int num_elements, const LogVal* const *vals) reporter->InternalError("Could not malloc?"); } for ( int i = 0; i < num_elements; i++ ) { - const LogVal* val = vals[i]; - position += CopyLogVal(data, position, val); + const Value* val = vals[i]; + position += CopyValue(data, position, val); } assert(position == length); @@ -1328,7 +1342,7 @@ HashKey* Manager::HashLogVals(const int num_elements, const LogVal* const *vals) } -Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { +Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); @@ -1384,7 +1398,7 @@ Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + t->Assign(ValueToVal( val->val.set_val.vals[i], type ), 0); } return t; break; @@ -1396,7 +1410,7 @@ Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) { - v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); } return v; @@ -1425,7 +1439,7 @@ Val* Manager::LogValToVal(const LogVal* val, BroType* request_type) { return NULL; } -Manager::ReaderInfo* Manager::FindReader(const InputReader* reader) +Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -1460,4 +1474,3 @@ string Manager::Hash(const string &input) { return out; } - diff --git a/src/input/Manager.h b/src/input/Manager.h index fe37efa08b..507af6468f 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -14,6 +14,7 @@ namespace input { class ReaderFrontend; +class ReaderBackend; class Manager { public: @@ -30,7 +31,8 @@ public: bool RemoveEventFilter(EnumVal* id, const string &name); protected: - + friend class ReaderFrontend; + // Reports an error for the given reader. void Error(ReaderFrontend* reader, const char* msg); @@ -42,6 +44,8 @@ protected: // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) void SendEntry(const ReaderFrontend* reader, int id, const threading::Value* const *vals); void EndCurrentSend(const ReaderFrontend* reader, int id); + + ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); private: struct ReaderInfo; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index b2bcedb2ad..8c996db4a1 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -1,13 +1,44 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "InputReader.h" +#include "ReaderBackend.h" +#include "ReaderFrontend.h" using threading::Value; using threading::Field; -namespace logging { +namespace input { -InputReader::InputReader(ReaderFrontend *arg_frontend) :MsgThread() +class ErrorMessage : public threading::OutputMessage { +public: + ErrorMessage(ReaderFrontend* reader, string message) + : threading::OutputMessage("Error", reader), + message(message) {} + + virtual bool Process() { + input_mgr->Error(object, message.c_str()); + return true; + } + +private: + string message; +} + +class PutMessage : public threading::OutputMessage { +public: + PutMessage(ReaderFrontend* reader, int id, const Value* const *val) + : threading::OutputMessage("Error", reader), + id(id), val(val) {} + + virtual bool Process() { + return input_mgr->Put(object, id, val); + } + +private: + int id; + Value* val; +} + +ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { buf = 0; buf_len = 1024; @@ -18,38 +49,47 @@ InputReader::InputReader(ReaderFrontend *arg_frontend) :MsgThread() SetName(frontend->Name()); } -InputReader::~InputReader() +ReaderBackend::~ReaderBackend() { } -void InputReader::Error(const char *msg) +void ReaderBackend::Error(const string &msg) { - input_mgr->Error(this, msg); + SendOut(new ErrorMessage(frontend, msg); } -void InputReader::Error(const string &msg) +void ReaderBackend::Put(int id, const Value* const *val) { - input_mgr->Error(this, msg.c_str()); + SendOut(new PutMessage(frontend, id, val); } -void InputReader::Put(int id, const LogVal* const *val) +void ReaderBackend::Delete(int id, const Value* const *val) { - input_mgr->Put(this, id, val); + SendOut(new DeleteMessage(frontend, id, val); } -void InputReader::Clear(int id) +void ReaderBackend::Clear(int id) { - input_mgr->Clear(this, id); + SendOut(new ClearMessage(frontend, id); } -void InputReader::Delete(int id, const LogVal* const *val) +bool ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) { - input_mgr->Delete(this, id, val); + SendOut(new SendEventMessage(frontend, name, num_vals, vals); +} + +void ReaderBackend::EndCurrentSend(int id) +{ + SendOut(new EndCurrentSendMessage(frontent, id); } +void ReaderBackend::SendEntry(int id, const Value* const *vals) +{ + SendOut(new SendEntryMessage(frontend, id, vals); +} -bool InputReader::Init(string arg_source) +bool ReaderBackend::Init(string arg_source) { source = arg_source; @@ -58,35 +98,31 @@ bool InputReader::Init(string arg_source) return !disabled; } -bool InputReader::AddFilter(int id, int arg_num_fields, - const LogField* const * arg_fields) +bool ReaderBackend::AddFilter(int id, int arg_num_fields, + const Field* const * arg_fields) { return DoAddFilter(id, arg_num_fields, arg_fields); } -bool InputReader::RemoveFilter(int id) +bool ReaderBackend::RemoveFilter(int id) { return DoRemoveFilter(id); } -void InputReader::Finish() +void ReaderBackend::Finish() { DoFinish(); disabled = true; } -bool InputReader::Update() +bool ReaderBackend::Update() { return DoUpdate(); } -bool InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) -{ - return input_mgr->SendEvent(name, num_vals, vals); -} // stolen from logwriter -const char* InputReader::Fmt(const char* format, ...) +const char* ReaderBackend::Fmt(const char* format, ...) { if ( ! buf ) buf = (char*) malloc(buf_len); @@ -111,14 +147,5 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(int id, const LogVal* const *vals) -{ - input_mgr->SendEntry(this, id, vals); -} - -void InputReader::EndCurrentSend(int id) -{ - input_mgr->EndCurrentSend(this, id); -} } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 7d2640b4fd..1fe44a09b2 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -1,26 +1,25 @@ // See the file "COPYING" in the main distribution directory for copyright. -// -// Same notes about thread safety as in LogWriter.h apply. - #ifndef INPUT_READERBACKEND_H #define INPUT_READERBACKEND_H -#include "InputMgr.h" #include "BroString.h" -#include "LogMgr.h" +#include "../threading/SerializationTypes.h" +#include "threading/MsgThread.h" namespace input { +class ReaderFrontend; + class ReaderBackend : public threading::MsgThread { public: - ReaderBackend(ReaderFrontend *frontend); + ReaderBackend(ReaderFrontend* frontend); virtual ~ReaderBackend(); bool Init(string arg_source); - bool AddFilter( int id, int arg_num_fields, const LogField* const* fields ); + bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); bool RemoveFilter ( int id ); @@ -32,7 +31,7 @@ protected: // Methods that have to be overwritten by the individual readers virtual bool DoInit(string arg_sources) = 0; - virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; + virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; virtual bool DoRemoveFilter( int id ) = 0; @@ -51,15 +50,15 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table - void Put(int id, const LogVal* const *val); - void Delete(int id, const LogVal* const *val); + void Put(int id, const threading::Value* const *val); + void Delete(int id, const threading::Value* const *val); void Clear(int id); // Table-functions (tracking mode): Only changed lines are propagated. - void SendEntry(int id, const LogVal* const *vals); + void SendEntry(int id, const threading::Value* const *vals); void EndCurrentSend(int id); diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index e69de29bb2..984ba30794 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -0,0 +1,54 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERFRONTEND_H +#define INPUT_READERFRONTEND_H + +#include "Manager.h" + +#include "threading/MsgThread.h" + +namespace input { + +class ReaderBackend; + +class ReaderFrontend { +public: + ReaderFrontend(bro_int_t type); + + virtual ~ReaderFrontend(); + + void Init(string arg_source); + + void Update(); + + void AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + + void Finish(); + + /** + * Returns a descriptive name for the reader, including the type of + * the backend and the source used. + * + * This method is safe to call from any thread. + */ + string Name() const; + + +protected: + friend class Manager; + + const string Source() const { return source; } + + string ty_name; // Name of the backend type. Set by the manager. + +private: + string source; + +}; + +} + + +#endif /* INPUT_READERFRONTEND_H */ + + diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 1747f983e4..a3bf5c21a6 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -1,13 +1,15 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef INPUTREADERASCII_H -#define INPUTREADERASCII_H +#ifndef INPUT_READERS_ASCII_H +#define INPUT_READERS_ASCII_H -#include "InputReader.h" -#include #include #include +#include "../ReaderBackend.h" + +namespace input { namespace reader { + // Description for input field mapping struct FieldMapping { string name; @@ -28,18 +30,18 @@ struct FieldMapping { }; -class InputReaderAscii : public InputReader { +class Ascii : public ReaderBackend { public: - InputReaderAscii(); - ~InputReaderAscii(); + Ascii(ReaderFrontend* frontend); + ~Ascii(); - static InputReader* Instantiate() { return new InputReaderAscii; } + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Ascii(frontend); } protected: virtual bool DoInit(string path); - virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ); + virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); virtual bool DoRemoveFilter ( int id ); @@ -52,7 +54,7 @@ private: struct Filter { unsigned int num_fields; - const LogField* const * fields; // raw mapping + const threading::Field* const * fields; // raw mapping // map columns in the file to columns to send back to the manager vector columnMap; @@ -64,7 +66,7 @@ private: TransportProto StringToProto(const string &proto); bool ReadHeader(); - LogVal* EntryToVal(string s, FieldMapping type); + threading::Value* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); @@ -85,4 +87,7 @@ private: }; -#endif /* INPUTREADERASCII_H */ +} +} + +#endif /* INPUT_READERS_ASCII_H */ diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index add10b3f10..65a55dee02 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -14,7 +14,7 @@ #include "writers/Ascii.h" #include "writers/None.h" -#include "threading/SerializationTypes.h" +#include "../threading/SerializationTypes.h" using namespace logging; using threading::Value; diff --git a/src/threading/SerializationTypes.cc b/src/threading/SerializationTypes.cc index f74de6ce57..dc5a1a14f9 100644 --- a/src/threading/SerializationTypes.cc +++ b/src/threading/SerializationTypes.cc @@ -12,7 +12,8 @@ bool Field::Read(SerializationFormat* fmt) int t; int st; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && + fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); type = (TypeTag) t; subtype = (TypeTag) st; @@ -21,7 +22,8 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && + fmt->Write((int)subtype, "subtype")); } Value::~Value() diff --git a/src/threading/SerializationTypes.h b/src/threading/SerializationTypes.h index 11ceda929c..ffcf774842 100644 --- a/src/threading/SerializationTypes.h +++ b/src/threading/SerializationTypes.h @@ -13,6 +13,8 @@ namespace threading { */ struct Field { string name; //! Name of the field. + // needed by input framework. port fields have two names (one for the port, one for the type) - this specifies the secondary name. + string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. From 8385d5bb2dedab25da82b67213a6051bd52e1746 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 6 Feb 2012 17:37:02 -0800 Subject: [PATCH 057/123] it compiles :) But that's all, not tested, don't expect it to do anything but crash. --- src/input/Manager.cc | 4 +- src/input/Manager.h | 17 +++-- src/input/ReaderBackend.cc | 114 ++++++++++++++++++++++++++++----- src/input/ReaderBackend.h | 4 +- src/input/ReaderFrontend.cc | 121 +++++++++++++++++++++++++++++++----- src/input/ReaderFrontend.h | 17 ++--- src/input/readers/Ascii.cc | 48 +++++++------- 7 files changed, 258 insertions(+), 67 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a97286162d..189a034b0f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -680,7 +680,7 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(const ReaderFrontend* reader, int id, const Value* const *vals) { +void Manager::SendEntry(const ReaderFrontend* reader, const int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -703,7 +703,7 @@ void Manager::SendEntry(const ReaderFrontend* reader, int id, const Value* const } -void Manager::SendEntryTable(const ReaderFrontend* reader, int id, const Value* const *vals) { +void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; diff --git a/src/input/Manager.h b/src/input/Manager.h index 507af6468f..a0b98294ca 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -14,7 +14,7 @@ namespace input { class ReaderFrontend; -class ReaderBackend; +class ReaderBackend; class Manager { public: @@ -32,6 +32,13 @@ public: protected: friend class ReaderFrontend; + friend class ErrorMessage; + friend class PutMessage; + friend class DeleteMessage; + friend class ClearMessage; + friend class SendEventMessage; + friend class SendEntryMessage; + friend class EndCurrentSendMessage; // Reports an error for the given reader. void Error(ReaderFrontend* reader, const char* msg); @@ -42,11 +49,14 @@ protected: bool Delete(const ReaderFrontend* reader, int id, const threading::Value* const *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) - void SendEntry(const ReaderFrontend* reader, int id, const threading::Value* const *vals); - void EndCurrentSend(const ReaderFrontend* reader, int id); + void SendEntry(const ReaderFrontend* reader, const int id, const threading::Value* const *vals); + void EndCurrentSend(const ReaderFrontend* reader, const int id); + + bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + private: struct ReaderInfo; @@ -60,7 +70,6 @@ private: void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); - bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); HashKey* HashValues(const int num_elements, const threading::Value* const *vals); int GetValueLength(const threading::Value* val); diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 8c996db4a1..72c8f95d8e 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -2,6 +2,7 @@ #include "ReaderBackend.h" #include "ReaderFrontend.h" +#include "Manager.h" using threading::Value; using threading::Field; @@ -15,28 +16,106 @@ public: message(message) {} virtual bool Process() { - input_mgr->Error(object, message.c_str()); + input_mgr->Error(Object(), message.c_str()); return true; } private: string message; -} +}; class PutMessage : public threading::OutputMessage { public: PutMessage(ReaderFrontend* reader, int id, const Value* const *val) - : threading::OutputMessage("Error", reader), + : threading::OutputMessage("Put", reader), id(id), val(val) {} virtual bool Process() { - return input_mgr->Put(object, id, val); + input_mgr->Put(Object(), id, val); + return true; } private: int id; - Value* val; -} + const Value* const *val; +}; + +class DeleteMessage : public threading::OutputMessage { +public: + DeleteMessage(ReaderFrontend* reader, int id, const Value* const *val) + : threading::OutputMessage("Delete", reader), + id(id), val(val) {} + + virtual bool Process() { + return input_mgr->Delete(Object(), id, val); + } + +private: + int id; + const Value* const *val; +}; + +class ClearMessage : public threading::OutputMessage { +public: + ClearMessage(ReaderFrontend* reader, int id) + : threading::OutputMessage("Clear", reader), + id(id) {} + + virtual bool Process() { + input_mgr->Clear(Object(), id); + return true; + } + +private: + int id; +}; + +class SendEventMessage : public threading::OutputMessage { +public: + SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, const Value* const *val) + : threading::OutputMessage("SendEvent", reader), + name(name), num_vals(num_vals), val(val) {} + + virtual bool Process() { + return input_mgr->SendEvent(name, num_vals, val); + } + +private: + const string name; + const int num_vals; + const Value* const *val; +}; + +class SendEntryMessage : public threading::OutputMessage { +public: + SendEntryMessage(ReaderFrontend* reader, const int id, const Value* const *val) + : threading::OutputMessage("SendEntry", reader), + id(id), val(val) {} + + virtual bool Process() { + input_mgr->SendEntry(Object(), id, val); + return true; + } + +private: + const int id; + const Value* const *val; +}; + +class EndCurrentSendMessage : public threading::OutputMessage { +public: + EndCurrentSendMessage(ReaderFrontend* reader, int id) + : threading::OutputMessage("SendEntry", reader), + id(id) {} + + virtual bool Process() { + input_mgr->EndCurrentSend(Object(), id); + return true; + } + +private: + int id; +}; ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { @@ -56,37 +135,44 @@ ReaderBackend::~ReaderBackend() void ReaderBackend::Error(const string &msg) { - SendOut(new ErrorMessage(frontend, msg); + SendOut(new ErrorMessage(frontend, msg)); } +/* +void ReaderBackend::Error(const char *msg) +{ + SendOut(new ErrorMessage(frontend, string(msg))); +} */ + + void ReaderBackend::Put(int id, const Value* const *val) { - SendOut(new PutMessage(frontend, id, val); + SendOut(new PutMessage(frontend, id, val)); } void ReaderBackend::Delete(int id, const Value* const *val) { - SendOut(new DeleteMessage(frontend, id, val); + SendOut(new DeleteMessage(frontend, id, val)); } void ReaderBackend::Clear(int id) { - SendOut(new ClearMessage(frontend, id); + SendOut(new ClearMessage(frontend, id)); } -bool ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) +void ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) { - SendOut(new SendEventMessage(frontend, name, num_vals, vals); + SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } void ReaderBackend::EndCurrentSend(int id) { - SendOut(new EndCurrentSendMessage(frontent, id); + SendOut(new EndCurrentSendMessage(frontend, id)); } void ReaderBackend::SendEntry(int id, const Value* const *vals) { - SendOut(new SendEntryMessage(frontend, id, vals); + SendOut(new SendEntryMessage(frontend, id, vals)); } bool ReaderBackend::Init(string arg_source) diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 1fe44a09b2..a37daaf4b6 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -42,7 +42,7 @@ protected: // Reports an error to the user. void Error(const string &msg); - void Error(const char *msg); + //void Error(const char *msg); // The following methods return the information as passed to Init(). const string Source() const { return source; } @@ -50,7 +50,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); + void SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table void Put(int id, const threading::Value* const *val); diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 44638d90b3..a7f9a4d2f6 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -1,28 +1,117 @@ // See the file "COPYING" in the main distribution directory for copyright. -#ifndef INPUT_READERFRONTEND_H -#define INPUT_READERFRONTEND_H - #include "Manager.h" - +#include "ReaderFrontend.h" +#include "ReaderBackend.h" #include "threading/MsgThread.h" -namespace logging { - -class ReaderBackend; - -class ReaderFrontend { - - ReaderFrontend(bro_int_t type); - - virtual ~ReaderFrontend(); +namespace input { -protected: - friend class Manager; +class InitMessage : public threading::InputMessage +{ +public: + InitMessage(ReaderBackend* backend, const string source) + : threading::InputMessage("Init", backend), + source(source) { } + + virtual bool Process() { return Object()->Init(source); } + +private: + const string source; }; +class UpdateMessage : public threading::InputMessage +{ +public: + UpdateMessage(ReaderBackend* backend) + : threading::InputMessage("Update", backend) + { } + + virtual bool Process() { return Object()->Update(); } +}; + +class FinishMessage : public threading::InputMessage +{ +public: + FinishMessage(ReaderBackend* backend) + : threading::InputMessage("Finish", backend) + { } + + virtual bool Process() { Object()->Finish(); return true; } +}; + +class AddFilterMessage : public threading::InputMessage +{ +public: + AddFilterMessage(ReaderBackend* backend, const int id, const int num_fields, const threading::Field* const* fields) + : threading::InputMessage("AddFilter", backend), + id(id), num_fields(num_fields), fields(fields) { } + + virtual bool Process() { return Object()->AddFilter(id, num_fields, fields); } + +private: + const int id; + const int num_fields; + const threading::Field* const* fields; +}; + +ReaderFrontend::ReaderFrontend(bro_int_t type) { + disabled = initialized = false; + ty_name = ""; + backend = input_mgr->CreateBackend(this, type); + + assert(backend); + backend->Start(); +} + +ReaderFrontend::~ReaderFrontend() { +} + +void ReaderFrontend::Init(string arg_source) { + if ( disabled ) + return; + + if ( initialized ) + reporter->InternalError("writer initialize twice"); + + source = arg_source; + initialized = true; + + backend->SendIn(new InitMessage(backend, arg_source)); +} + +void ReaderFrontend::Update() { + if ( disabled ) + return; + + backend->SendIn(new UpdateMessage(backend)); +} + +void ReaderFrontend::Finish() { + if ( disabled ) + return; + + backend->SendIn(new FinishMessage(backend)); +} + +void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const threading::Field* const* fields) { + if ( disabled ) + return; + + backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); +} + +string ReaderFrontend::Name() const + { + if ( source.size() ) + return ty_name; + + return ty_name + "/" + source; + } + + + } -#endif /* INPUT_READERFRONTEND_H */ diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 984ba30794..876082d9a6 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -3,13 +3,12 @@ #ifndef INPUT_READERFRONTEND_H #define INPUT_READERFRONTEND_H -#include "Manager.h" +#include "../threading/MsgThread.h" +#include "../threading/SerializationTypes.h" -#include "threading/MsgThread.h" +namespace input { -namespace input { - -class ReaderBackend; +class Manager; class ReaderFrontend { public: @@ -21,7 +20,7 @@ public: void Update(); - void AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); void Finish(); @@ -32,17 +31,19 @@ public: * This method is safe to call from any thread. */ string Name() const; - protected: friend class Manager; - const string Source() const { return source; } + const string Source() const { return source; }; string ty_name; // Name of the backend type. Set by the manager. private: + ReaderBackend* backend; // The backend we have instanatiated. string source; + bool disabled; // True if disabled. + bool initialized; // True if initialized. }; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 257cb4cf71..e798f69a36 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -1,11 +1,17 @@ // See the file "COPYING" in the main distribution directory for copyright. -#include "InputReaderAscii.h" -#include "DebugLogger.h" +#include "Ascii.h" #include "NetVar.h" +#include #include +#include "../../threading/SerializationTypes.h" + +using namespace input::reader; +using threading::Value; +using threading::Field; + FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) { @@ -31,7 +37,7 @@ FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } -InputReaderAscii::InputReaderAscii() +Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; @@ -53,13 +59,13 @@ InputReaderAscii::InputReaderAscii() } -InputReaderAscii::~InputReaderAscii() +Ascii::~Ascii() { DoFinish(); } -void InputReaderAscii::DoFinish() +void Ascii::DoFinish() { filters.empty(); if ( file != 0 ) { @@ -69,7 +75,7 @@ void InputReaderAscii::DoFinish() } } -bool InputReaderAscii::DoInit(string path) +bool Ascii::DoInit(string path) { fname = path; @@ -82,7 +88,7 @@ bool InputReaderAscii::DoInit(string path) return true; } -bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) { +bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { if ( HasFilter(id) ) { return false; // no, we don't want to add this a second time } @@ -96,7 +102,7 @@ bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* return true; } -bool InputReaderAscii::DoRemoveFilter ( int id ) { +bool Ascii::DoRemoveFilter ( int id ) { if (!HasFilter(id) ) { return false; } @@ -107,7 +113,7 @@ bool InputReaderAscii::DoRemoveFilter ( int id ) { } -bool InputReaderAscii::HasFilter(int id) { +bool Ascii::HasFilter(int id) { map::iterator it = filters.find(id); if ( it == filters.end() ) { return false; @@ -116,7 +122,7 @@ bool InputReaderAscii::HasFilter(int id) { } -bool InputReaderAscii::ReadHeader() { +bool Ascii::ReadHeader() { // try to read the header line... string line; if ( !GetLine(line) ) { @@ -142,7 +148,7 @@ bool InputReaderAscii::ReadHeader() { for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const LogField* field = (*it).second.fields[i]; + const Field* field = (*it).second.fields[i]; map::iterator fit = fields.find(field->name); if ( fit == fields.end() ) { @@ -169,7 +175,7 @@ bool InputReaderAscii::ReadHeader() { return true; } -bool InputReaderAscii::GetLine(string& str) { +bool Ascii::GetLine(string& str) { while ( getline(*file, str) ) { if ( str[0] != '#' ) { return true; @@ -184,7 +190,7 @@ bool InputReaderAscii::GetLine(string& str) { return false; } -TransportProto InputReaderAscii::StringToProto(const string &proto) { +TransportProto Ascii::StringToProto(const string &proto) { if ( proto == "unknown" ) { return TRANSPORT_UNKNOWN; } else if ( proto == "tcp" ) { @@ -202,12 +208,12 @@ TransportProto InputReaderAscii::StringToProto(const string &proto) { return TRANSPORT_UNKNOWN; } -LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { +Value* Ascii::EntryToVal(string s, FieldMapping field) { - LogVal* val = new LogVal(field.type, true); + Value* val = new Value(field.type, true); if ( s.compare(unset_field) == 0 ) { // field is not set... - return new LogVal(field.type, false); + return new Value(field.type, false); } switch ( field.type ) { @@ -306,7 +312,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { if ( s.compare(empty_field) == 0 ) length = 0; - LogVal** lvals = new LogVal* [length]; + Value** lvals = new Value* [length]; if ( field.type == TYPE_TABLE ) { val->val.set_val.vals = lvals; @@ -333,7 +339,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - LogVal* newval = EntryToVal(element, field.subType()); + Value* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -365,7 +371,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { } // read the entire file and send appropriate thingies back to InputMgr -bool InputReaderAscii::DoUpdate() { +bool Ascii::DoUpdate() { // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { @@ -405,7 +411,7 @@ bool InputReaderAscii::DoUpdate() { for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - LogVal** fields = new LogVal*[(*it).second.num_fields]; + Value** fields = new Value*[(*it).second.num_fields]; int fpos = 0; for ( vector::iterator fit = (*it).second.columnMap.begin(); @@ -417,7 +423,7 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = EntryToVal(stringfields[(*fit).position], *fit); + Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { return false; } From 88233efb2c8b6700dd2ec0a1f6addc65a282b622 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 13 Feb 2012 22:29:55 -0800 Subject: [PATCH 058/123] It works. Even including all unit tests. But: there are still a few places where I am sure that there are race conditions & memory leaks & I do not really like the current interface & I have to add a few more messages between the front and backend. But - it works :) --- scripts/base/frameworks/input/main.bro | 2 + src/input/Manager.cc | 78 ++++++++++++++----- src/input/Manager.h | 14 ++-- src/input/ReaderBackend.cc | 28 +++---- src/input/ReaderBackend.h | 8 +- src/input/ReaderFrontend.cc | 1 + src/input/readers/Ascii.cc | 4 +- src/main.cc | 1 + src/threading/Manager.cc | 2 +- src/threading/MsgThread.cc | 2 +- .../scripts/base/frameworks/input/basic.bro | 5 +- .../frameworks/input/onecolumn-norecord.bro | 4 + .../frameworks/input/onecolumn-record.bro | 4 + .../scripts/base/frameworks/input/port.bro | 7 ++ .../base/frameworks/input/predicate.bro | 3 + .../base/frameworks/input/twofilters.bro | 13 +++- 16 files changed, 127 insertions(+), 49 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index c76eba80b9..cac1aca54a 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -114,6 +114,8 @@ export { ## filter: the `TableFilter` record describing the filter. global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; + global update_finished: event(id: Input::ID); + } @load base/input.bif diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 189a034b0f..79d42fe71f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -215,7 +215,7 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - ReaderFrontend* reader_obj = new ReaderFrontend(id->AsEnum()); + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); assert(reader_obj); // get the source... @@ -680,7 +680,7 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(const ReaderFrontend* reader, const int id, const Value* const *vals) { +void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -692,18 +692,25 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, const Value* return; } + int readFields; if ( i->filters[id]->filter_type == TABLE_FILTER ) { - SendEntryTable(reader, id, vals); + readFields = SendEntryTable(reader, id, vals); } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(reader, type, id, vals); } else { assert(false); } + for ( int i = 0; i < readFields; i++ ) { + delete vals[i]; + } + delete [] vals; + + } -void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { +int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -733,7 +740,7 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); - return; + return filter->num_val_fields + filter->num_idx_fields; } else { assert( filter->num_val_fields > 0 ); // updated @@ -794,11 +801,11 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... delete(filter->currDict->RemoveEntry(idxhash)); - return; + return filter->num_val_fields + filter->num_idx_fields; } else { // keep old one filter->currDict->Insert(idxhash, h); - return; + return filter->num_val_fields + filter->num_idx_fields; } } @@ -809,7 +816,7 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); - return; + return filter->num_val_fields + filter->num_idx_fields; } filter->tab->Assign(idxval, k, valval); @@ -842,6 +849,9 @@ void Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const V } } } + + + return filter->num_val_fields + filter->num_idx_fields; } @@ -926,9 +936,21 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); + + // Send event that the current update is indeed finished. + + + EventHandler* handler = event_registry->Lookup("Input::update_finished"); + if ( handler == 0 ) { + reporter->InternalError("Input::update_finished not found!"); + } + + + Ref(i->id); + SendEvent(handler, 1, i->id); } -void Manager::Put(const ReaderFrontend* reader, int id, const Value* const *vals) { +void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -951,7 +973,7 @@ void Manager::Put(const ReaderFrontend* reader, int id, const Value* const *vals } -void Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { +int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); bool updated = false; @@ -985,11 +1007,13 @@ void Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, } } - SendEvent(filter->event, out_vals); + SendEvent(filter->event, out_vals); + + return filter->fields->NumFields(); } -void Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { +int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { ReaderInfo *i = FindReader(reader); assert(i); @@ -1011,6 +1035,8 @@ void Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const } filter->tab->Assign(idxval, valval); + + return filter->num_idx_fields + filter->num_val_fields; } void Manager::Clear(const ReaderFrontend* reader, int id) { @@ -1028,7 +1054,7 @@ void Manager::Clear(const ReaderFrontend* reader, int id) { filter->tab->RemoveAll(); } -bool Manager::Delete(const ReaderFrontend* reader, int id, const Value* const *vals) { +bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -1037,18 +1063,29 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, const Value* const *v assert(i->HasFilter(id)); + bool success = false; + int readVals = 0; + if ( i->filters[id]->filter_type == TABLE_FILTER ) { TableFilter* filter = (TableFilter*) i->filters[id]; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); - return( filter->tab->Delete(idxval) != 0 ); + readVals = filter->num_idx_fields + filter->num_val_fields; + success = ( filter->tab->Delete(idxval) != 0 ); } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); - return true; + readVals = SendEventFilterEvent(reader, type, id, vals); + success = true; } else { assert(false); return false; } + + for ( int i = 0; i < readVals; i++ ) { + delete vals[i]; + } + delete [] vals; + + return success; } void Manager::Error(ReaderFrontend* reader, const char* msg) @@ -1056,7 +1093,7 @@ void Manager::Error(ReaderFrontend* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -bool Manager::SendEvent(const string& name, const int num_vals, const Value* const *vals) +bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { @@ -1078,6 +1115,11 @@ bool Manager::SendEvent(const string& name, const int num_vals, const Value* con mgr.Dispatch(new Event(handler, vl)); + for ( int i = 0; i < num_vals; i++ ) { + delete vals[i]; + } + delete [] vals; + return true; } diff --git a/src/input/Manager.h b/src/input/Manager.h index a0b98294ca..45c07895f2 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -44,15 +44,15 @@ protected: void Error(ReaderFrontend* reader, const char* msg); // for readers to write to input stream in direct mode (reporting new/deleted values directly) - void Put(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); void Clear(const ReaderFrontend* reader, int id); - bool Delete(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) - void SendEntry(const ReaderFrontend* reader, const int id, const threading::Value* const *vals); + void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); void EndCurrentSend(const ReaderFrontend* reader, const int id); - bool SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); + bool SendEvent(const string& name, const int num_vals, threading::Value* *vals); ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); @@ -60,9 +60,9 @@ protected: private: struct ReaderInfo; - void SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); - void PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); - void SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); bool IsCompatibleType(BroType* t, bool atomic_only=false); diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 72c8f95d8e..f9992f5f0e 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -26,7 +26,7 @@ private: class PutMessage : public threading::OutputMessage { public: - PutMessage(ReaderFrontend* reader, int id, const Value* const *val) + PutMessage(ReaderFrontend* reader, int id, Value* *val) : threading::OutputMessage("Put", reader), id(id), val(val) {} @@ -37,12 +37,12 @@ public: private: int id; - const Value* const *val; + Value* *val; }; class DeleteMessage : public threading::OutputMessage { public: - DeleteMessage(ReaderFrontend* reader, int id, const Value* const *val) + DeleteMessage(ReaderFrontend* reader, int id, Value* *val) : threading::OutputMessage("Delete", reader), id(id), val(val) {} @@ -52,7 +52,7 @@ public: private: int id; - const Value* const *val; + Value* *val; }; class ClearMessage : public threading::OutputMessage { @@ -72,7 +72,7 @@ private: class SendEventMessage : public threading::OutputMessage { public: - SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, const Value* const *val) + SendEventMessage(ReaderFrontend* reader, const string& name, const int num_vals, Value* *val) : threading::OutputMessage("SendEvent", reader), name(name), num_vals(num_vals), val(val) {} @@ -83,14 +83,14 @@ public: private: const string name; const int num_vals; - const Value* const *val; + Value* *val; }; class SendEntryMessage : public threading::OutputMessage { public: - SendEntryMessage(ReaderFrontend* reader, const int id, const Value* const *val) + SendEntryMessage(ReaderFrontend* reader, const int id, Value* *val) : threading::OutputMessage("SendEntry", reader), - id(id), val(val) {} + id(id), val(val) { } virtual bool Process() { input_mgr->SendEntry(Object(), id, val); @@ -99,13 +99,13 @@ public: private: const int id; - const Value* const *val; + Value* *val; }; class EndCurrentSendMessage : public threading::OutputMessage { public: EndCurrentSendMessage(ReaderFrontend* reader, int id) - : threading::OutputMessage("SendEntry", reader), + : threading::OutputMessage("EndCurrentSend", reader), id(id) {} virtual bool Process() { @@ -145,12 +145,12 @@ void ReaderBackend::Error(const char *msg) } */ -void ReaderBackend::Put(int id, const Value* const *val) +void ReaderBackend::Put(int id, Value* *val) { SendOut(new PutMessage(frontend, id, val)); } -void ReaderBackend::Delete(int id, const Value* const *val) +void ReaderBackend::Delete(int id, Value* *val) { SendOut(new DeleteMessage(frontend, id, val)); } @@ -160,7 +160,7 @@ void ReaderBackend::Clear(int id) SendOut(new ClearMessage(frontend, id)); } -void ReaderBackend::SendEvent(const string& name, const int num_vals, const Value* const *vals) +void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) { SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } @@ -170,7 +170,7 @@ void ReaderBackend::EndCurrentSend(int id) SendOut(new EndCurrentSendMessage(frontend, id)); } -void ReaderBackend::SendEntry(int id, const Value* const *vals) +void ReaderBackend::SendEntry(int id, Value* *vals) { SendOut(new SendEntryMessage(frontend, id, vals)); } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index a37daaf4b6..c12d187545 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -50,15 +50,15 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, const threading::Value* const *vals); + void SendEvent(const string& name, const int num_vals, threading::Value* *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table - void Put(int id, const threading::Value* const *val); - void Delete(int id, const threading::Value* const *val); + void Put(int id, threading::Value* *val); + void Delete(int id, threading::Value* *val); void Clear(int id); // Table-functions (tracking mode): Only changed lines are propagated. - void SendEntry(int id, const threading::Value* const *vals); + void SendEntry(int id, threading::Value* *vals); void EndCurrentSend(int id); diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index a7f9a4d2f6..0dac33d5e8 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -56,6 +56,7 @@ private: const threading::Field* const* fields; }; + ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; ty_name = ""; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e798f69a36..095d74bf11 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -445,10 +445,12 @@ bool Ascii::DoUpdate() { SendEntry((*it).first, fields); - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + /* Do not do this, ownership changes to other thread + * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { delete fields[i]; } delete [] fields; + */ } } diff --git a/src/main.cc b/src/main.cc index 9df06aa0a0..8205b6de0b 100644 --- a/src/main.cc +++ b/src/main.cc @@ -295,6 +295,7 @@ void terminate_bro() log_mgr->Terminate(); thread_mgr->Terminate(); + mgr.Drain(); delete timer_mgr; delete dns_mgr; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d008d2e5e8..7b571e753c 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -111,7 +111,7 @@ void Manager::Process() else { - string s = msg->Name() + " failed, terminating thread"; + string s = msg->Name() + " failed, terminating thread " + t->Name() + " (in ThreadManager)"; reporter->Error("%s", s.c_str()); t->Stop(); } diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index b7782b9a05..5f77a1c9f8 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -267,7 +267,7 @@ void MsgThread::Run() if ( ! result ) { - string s = msg->Name() + " failed, terminating thread"; + string s = msg->Name() + " failed, terminating thread (MsgThread)"; Error(s.c_str()); Stop(); break; diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index d1b6659eb6..3b75220625 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -48,7 +48,10 @@ event bro_init() Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); - print servers; Input::remove_tablefilter(A::INPUT, "ssh"); Input::remove_stream(A::INPUT); } + +event Input::update_finished(id: Input::ID) { + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 88838cc8d6..712a877960 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -34,5 +34,9 @@ event bro_init() Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { print servers; } + diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index fc4d862cd3..7b62ddcddd 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -34,5 +34,9 @@ event bro_init() Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { print servers; } + diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index c14892ae36..65d73c54f7 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -39,3 +39,10 @@ event bro_init() Input::remove_tablefilter(A::INPUT, "input"); Input::remove_stream(A::INPUT); } + +event Input::update_finished(id: Input::ID) { + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; +} + diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 5e6bae7b62..bc1ab89bb2 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -42,6 +42,9 @@ event bro_init() $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 5af664e0e9..d5bff0c5bb 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -35,6 +35,8 @@ type Val: record { global destination1: table[int] of Val = table(); global destination2: table[int] of Val = table(); +global done: bool = F; + event bro_init() { # first read in the old stuff into the table... @@ -45,6 +47,15 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); Input::force_update(A::INPUT); +} + +event Input::update_finished(id: Input::ID) { + if ( done == T ) { + return; + } + + done = T; + if ( 1 in destination1 ) { print "VALID"; } @@ -90,6 +101,4 @@ event bro_init() if ( 7 in destination2 ) { print "VALID"; } - - } From a850cc59922118a8dd62e5c9c9e9c6abe22bd942 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 15 Feb 2012 15:14:04 -0800 Subject: [PATCH 059/123] make filter removal and stream closure asynchronous. --- src/input/Manager.cc | 64 ++++++++++++++++++--- src/input/Manager.h | 5 +- src/input/ReaderBackend.cc | 110 +++++++++++++++++------------------- src/input/ReaderBackend.h | 13 ++--- src/input/ReaderFrontend.cc | 29 ++++++++-- src/input/ReaderFrontend.h | 57 +++++++++++++++++++ 6 files changed, 200 insertions(+), 78 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 79d42fe71f..6655ae5e82 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -534,7 +534,6 @@ bool Manager::RemoveStream(const EnumVal* id) { if ( (*s)->id == id ) { i = (*s); - readers.erase(s); // remove from vector break; } } @@ -545,11 +544,29 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); - delete(i); - return true; } +bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { + ReaderInfo *i = 0; + + + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->reader && (*s)->reader == reader ) + { + i = *s; + delete(i); + readers.erase(s); + return true; + } + } + + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; + +} + bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -615,20 +632,51 @@ bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); - if ( it == i->filters.end() ) { + bool found = false; + int filterId; + + for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { + if ( (*it).second->name == name ) { + found = true; + filterId = (*it).first; + + if ( (*it).second->filter_type != TABLE_FILTER ) { + reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); + return false; + } + + break; + } + } + + if ( !found ) { + reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); return false; } - if ( i->filters[id->InternalInt()]->filter_type != TABLE_FILTER ) { - // wrong type; + i->reader->RemoveFilter(filterId); + + return true; +} + +bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + map::iterator it = i->filters.find(filterId); + if ( it == i->filters.end() ) { + reporter->Error("Got RemoveFilterContinuation where filter nonexistant for %d", filterId); return false; } delete (*it).second; i->filters.erase(it); + return true; -} +} bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); diff --git a/src/input/Manager.h b/src/input/Manager.h index 45c07895f2..9e35dd2199 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -32,13 +32,14 @@ public: protected: friend class ReaderFrontend; - friend class ErrorMessage; friend class PutMessage; friend class DeleteMessage; friend class ClearMessage; friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; + friend class FilterRemovedMessage; + friend class ReaderFinishedMessage; // Reports an error for the given reader. void Error(ReaderFrontend* reader, const char* msg); @@ -56,6 +57,8 @@ protected: ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); + bool RemoveStreamContinuation(const ReaderFrontend* reader); private: struct ReaderInfo; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index f9992f5f0e..5cb4fe34f2 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -9,21 +9,6 @@ using threading::Field; namespace input { -class ErrorMessage : public threading::OutputMessage { -public: - ErrorMessage(ReaderFrontend* reader, string message) - : threading::OutputMessage("Error", reader), - message(message) {} - - virtual bool Process() { - input_mgr->Error(Object(), message.c_str()); - return true; - } - -private: - string message; -}; - class PutMessage : public threading::OutputMessage { public: PutMessage(ReaderFrontend* reader, int id, Value* *val) @@ -104,7 +89,7 @@ private: class EndCurrentSendMessage : public threading::OutputMessage { public: - EndCurrentSendMessage(ReaderFrontend* reader, int id) + EndCurrentSendMessage(ReaderFrontend* reader, const int id) : threading::OutputMessage("EndCurrentSend", reader), id(id) {} @@ -114,9 +99,46 @@ public: } private: - int id; + const int id; }; +class FilterRemovedMessage : public threading::OutputMessage { +public: + FilterRemovedMessage(ReaderFrontend* reader, const int id) + : threading::OutputMessage("FilterRemoved", reader), + id(id) {} + + virtual bool Process() { + return input_mgr->RemoveFilterContinuation(Object(), id); + } + +private: + const int id; +}; + +class ReaderFinishedMessage : public threading::OutputMessage { +public: + ReaderFinishedMessage(ReaderFrontend* reader) + : threading::OutputMessage("ReaderFinished", reader) {} + + virtual bool Process() { + return input_mgr->RemoveStreamContinuation(Object()); + } + +private: +}; + + +class DisableMessage : public threading::OutputMessage +{ +public: + DisableMessage(ReaderFrontend* writer) + : threading::OutputMessage("Disable", writer) {} + + virtual bool Process() { Object()->SetDisable(); return true; } +}; + + ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() { buf = 0; @@ -133,18 +155,6 @@ ReaderBackend::~ReaderBackend() } -void ReaderBackend::Error(const string &msg) -{ - SendOut(new ErrorMessage(frontend, msg)); -} - -/* -void ReaderBackend::Error(const char *msg) -{ - SendOut(new ErrorMessage(frontend, string(msg))); -} */ - - void ReaderBackend::Put(int id, Value* *val) { SendOut(new PutMessage(frontend, id, val)); @@ -181,6 +191,11 @@ bool ReaderBackend::Init(string arg_source) // disable if DoInit returns error. disabled = !DoInit(arg_source); + + if ( disabled ) { + DisableFrontend(); + } + return !disabled; } @@ -192,13 +207,17 @@ bool ReaderBackend::AddFilter(int id, int arg_num_fields, bool ReaderBackend::RemoveFilter(int id) { - return DoRemoveFilter(id); + bool success = DoRemoveFilter(id); + SendOut(new FilterRemovedMessage(frontend, id)); + return success; // yes, I know, noone reads this. } void ReaderBackend::Finish() { DoFinish(); disabled = true; + DisableFrontend(); + SendOut(new ReaderFinishedMessage(frontend)); } bool ReaderBackend::Update() @@ -206,32 +225,9 @@ bool ReaderBackend::Update() return DoUpdate(); } - -// stolen from logwriter -const char* ReaderBackend::Fmt(const char* format, ...) - { - if ( ! buf ) - buf = (char*) malloc(buf_len); - - va_list al; - va_start(al, format); - int n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - - if ( (unsigned int) n >= buf_len ) - { // Not enough room, grow the buffer. - buf_len = n + 32; - buf = (char*) realloc(buf, buf_len); - - // Is it portable to restart? - va_start(al, format); - n = safe_vsnprintf(buf, buf_len, format, al); - va_end(al); - } - - return buf; - } - - +void ReaderBackend::DisableFrontend() +{ + SendOut(new DisableMessage(frontend)); +} } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index c12d187545..de4a056c22 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -26,6 +26,12 @@ public: void Finish(); bool Update(); + + /** + * Disables the frontend that has instantiated this backend. Once + * disabled,the frontend will not send any further message over. + */ + void DisableFrontend(); protected: // Methods that have to be overwritten by the individual readers @@ -40,16 +46,9 @@ protected: // update file contents to logmgr virtual bool DoUpdate() = 0; - // Reports an error to the user. - void Error(const string &msg); - //void Error(const char *msg); - // The following methods return the information as passed to Init(). const string Source() const { return source; } - // A thread-safe version of fmt(). (stolen from logwriter) - const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, threading::Value* *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 0dac33d5e8..0fdf90d9ad 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -5,8 +5,9 @@ #include "ReaderBackend.h" #include "threading/MsgThread.h" -namespace input { +// FIXME: cleanup of disabled inputreaders is missing. we need this, because stuff can e.g. fail in init and might never be removed afterwards. +namespace input { class InitMessage : public threading::InputMessage { @@ -56,6 +57,19 @@ private: const threading::Field* const* fields; }; +class RemoveFilterMessage : public threading::InputMessage +{ +public: + RemoveFilterMessage(ReaderBackend* backend, const int id) + : threading::InputMessage("RemoveFilter", backend), + id(id) { } + + virtual bool Process() { return Object()->RemoveFilter(id); } + +private: + const int id; +}; + ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; @@ -103,15 +117,20 @@ void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const thr backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); } +void ReaderFrontend::RemoveFilter(const int id) { + if ( disabled ) + return; + + backend->SendIn(new RemoveFilterMessage(backend, id)); +} + string ReaderFrontend::Name() const - { +{ if ( source.size() ) return ty_name; return ty_name + "/" + source; - } - - +} } diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 876082d9a6..97433c8af6 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -10,20 +10,77 @@ namespace input { class Manager; +/** + * Bridge class between the input::Manager and backend input threads. The + * Manager instantiates one \a ReaderFrontend for each open input stream. + * Each frontend in turns instantiates a ReaderBackend-derived class + * internally that's specific to the particular input format. That backend + * spawns a new thread, and it receives messages from the frontend that + * correspond to method called by the manager. + */ class ReaderFrontend { public: + /** + * Constructor. + * + * type: The backend writer type, with the value corresponding to the + * script-level \c Input::Reader enum (e.g., \a READER_ASCII). The + * frontend will internally instantiate a ReaderBackend of the + * corresponding type. + * + * Frontends must only be instantiated by the main thread. + */ ReaderFrontend(bro_int_t type); + /** + * Destructor. + * + * Frontends must only be destroyed by the main thread. + */ virtual ~ReaderFrontend(); + /** + * Initializes the reader. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. If the backend method fails, it + * sends a message back that will asynchronously call Disable(). + * + * See ReaderBackend::Init() for arguments. + * This method must only be called from the main thread. + */ void Init(string arg_source); void Update(); + /* * The method takes + * ownership of \a fields. */ + void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); + void RemoveFilter ( const int id ); + void Finish(); + /** + * Disables the reader frontend. From now on, all method calls that + * would normally send message over to the backend, turn into no-ops. + * Note though that it does not stop the backend itself, use Finsh() + * to do that as well (this method is primarily for use as callback + * when the backend wants to disable the frontend). + * + * Disabled frontend will eventually be discarded by the + * input::Manager. + * + * This method must only be called from the main thread. + */ + void SetDisable() { disabled = true; } + + /** + * Returns true if the reader frontend has been disabled with SetDisable(). + */ + bool Disabled() { return disabled; } + /** * Returns a descriptive name for the reader, including the type of * the backend and the source used. From 84883348ecc11643b6e96cecdbcdefce2bef45ba Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 11:27:10 -0800 Subject: [PATCH 060/123] interface documentation. to a big part stolen from the logging framework --- scripts/base/frameworks/input/main.bro | 8 +- src/input/Manager.cc | 33 +++-- src/input/Manager.h | 122 ++++++++++++++++- src/input/ReaderBackend.h | 182 +++++++++++++++++++++++-- src/input/ReaderFrontend.h | 31 ++++- src/types.bif | 6 + 6 files changed, 350 insertions(+), 32 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index cac1aca54a..7e581070e6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,11 +4,14 @@ module Input; export { + redef enum Input::ID += { TABLE_READ }; - + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + const default_mode = MANUAL &redef; + ## Stream decription type used for the `create_stream` method type StreamDescription: record { ## String that allows the reader to find the source. @@ -17,6 +20,9 @@ export { ## Reader to use for this steam reader: Reader &default=default_reader; + + ## Read mode to use for this stream + mode: Mode &default=default_mode; }; ## TableFilter description type used for the `add_tablefilter` method. diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6655ae5e82..d4e5cdaee9 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -685,18 +685,28 @@ bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); - if ( it == i->filters.end() ) { + bool found = false; + int filterId; + for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { + if ( (*it).second->name == name ) { + found = true; + filterId = (*it).first; + + if ( (*it).second->filter_type != EVENT_FILTER ) { + reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); + return false; + } + + break; + } + } + + if ( !found ) { + reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); return false; } - if ( i->filters[id->InternalInt()]->filter_type != EVENT_FILTER ) { - // wrong type; - return false; - } - - delete (*it).second; - i->filters.erase(it); + i->reader->RemoveFilter(filterId); return true; } @@ -1136,11 +1146,6 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { return success; } -void Manager::Error(ReaderFrontend* reader, const char* msg) -{ - reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); -} - bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); diff --git a/src/input/Manager.h b/src/input/Manager.h index 9e35dd2199..be84ee416d 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -1,4 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. +// +// Class for managing input streams and filters #ifndef INPUT_MANAGER_H #define INPUT_MANAGER_H @@ -16,18 +18,100 @@ namespace input { class ReaderFrontend; class ReaderBackend; +/** + * Singleton class for managing input streams. + */ class Manager { public: + /** + * Constructor. + */ Manager(); + + /** + * Destructor. + */ + ~Manager(); + /** + * Creates a new input stream. + * + * @param id The enum value corresponding the input stream. + * + * @param description A record of script type \c Input:StreamDescription. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ ReaderFrontend* CreateStream(EnumVal* id, RecordVal* description); + + /** + * Force update on a input stream. + * Forces a re-read of the whole input source. + * Usually used, when an input stream is opened in managed mode. + * Otherwise, this can be used to trigger a input source check before a heartbeat message arrives. + * May be ignored by the reader. + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool ForceUpdate(const EnumVal* id); + + /** + * Deletes an existing input stream + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool RemoveStream(const EnumVal* id); + /** + * Add a filter to an input source, which will write the data from the data source into + * a Bro table. + * + * @param id The enum value corresponding the input stream. + * + * @param description A record of script type \c Input:TableFilter. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool AddTableFilter(EnumVal *id, RecordVal* filter); + + /** + * Removes a tablefilter from the log stream + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool RemoveTableFilter(EnumVal* id, const string &name); + /** + * Add a filter to an input source, which sends events for read input data. + * + * @param id The enum value corresponding the input stream. + * + * @param description A record of script type \c Input:EventFilter. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool AddEventFilter(EnumVal *id, RecordVal* filter); + + /** + * Removes a eventfilter from the log stream + * + * @param id The enum value corresponding the input stream. + * + * This method corresponds directly to the internal BiF defined in + * input.bif, which just forwards here. + */ bool RemoveEventFilter(EnumVal* id, const string &name); protected: @@ -41,46 +125,76 @@ protected: friend class FilterRemovedMessage; friend class ReaderFinishedMessage; - // Reports an error for the given reader. - void Error(ReaderFrontend* reader, const char* msg); - - // for readers to write to input stream in direct mode (reporting new/deleted values directly) + // For readers to write to input stream in direct mode (reporting new/deleted values directly) + // Functions take ownership of threading::Value fields void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); void Clear(const ReaderFrontend* reader, int id); bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) + // Functions take ownership of threading::Value fields void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); void EndCurrentSend(const ReaderFrontend* reader, const int id); + // Allows readers to directly send Bro events. + // The num_vals and vals must be the same the named event expects. + // Takes ownership of threading::Value fields bool SendEvent(const string& name, const int num_vals, threading::Value* *vals); + // Instantiates a new ReaderBackend of the given type (note that + // doing so creates a new thread!). ReaderBackend* CreateBackend(ReaderFrontend* frontend, bro_int_t type); + // Functions are called from the ReaderBackend to notify the manager, that a filter has been removed + // or a stream has been closed. + // Used to prevent race conditions where data for a specific filter is still in the queue when the + // RemoveFilter directive is executed by the main thread. + // This makes sure all data that has ben queued for a filter is still received. bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); bool RemoveStreamContinuation(const ReaderFrontend* reader); private: struct ReaderInfo; + // SendEntry implementation for Tablefilter int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + + // Put implementation for Tablefilter int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + + // SendEntry and Put implementation for Eventfilter int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types + // from the log framework bool IsCompatibleType(BroType* t, bool atomic_only=false); + // Check if a record is made up of compatible types and return a list of all fields that are in the record in order. + // Recursively unrolls records bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + // Send events void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); + // get a hashkey for a set of threading::Values HashKey* HashValues(const int num_elements, const threading::Value* const *vals); + + // Get the memory used by a specific value int GetValueLength(const threading::Value* val); + // Copies the raw data in a specific threading::Value to position sta int CopyValue(char *data, const int startpos, const threading::Value* val); + // Convert Threading::Value to an internal Bro Type (works also with Records) Val* ValueToVal(const threading::Value* val, BroType* request_type); + + // Convert Threading::Value to an internal Bro List type Val* ValueToIndexVal(int num_fields, const RecordType* type, const threading::Value* const *vals); + + // Converts a threading::value to a record type. mostly used by ValueToVal RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + + // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); ReaderInfo* FindReader(const ReaderFrontend* reader); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index de4a056c22..c6fbaac715 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -11,20 +11,90 @@ namespace input { class ReaderFrontend; +/** + * Base class for reader implementation. When the input:Manager creates a + * new input stream, it instantiates a ReaderFrontend. That then in turn + * creates a ReaderBackend of the right type. The frontend then forwards + * message over the backend as its methods are called. + * + * All of this methods must be called only from the corresponding child + * thread (the constructor is the one exception.) + */ class ReaderBackend : public threading::MsgThread { public: + /** + * Constructor. + * + * @param frontend The frontend reader that created this backend. The + * *only* purpose of this value is to be passed back via messages as + * a argument to callbacks. One must not otherwise access the + * frontend, it's running in a different thread. + * + * @param frontend pointer to the reader frontend + */ ReaderBackend(ReaderFrontend* frontend); + /** + * Destructor. + */ virtual ~ReaderBackend(); - + + /** + * One-time initialization of the reader to define the input source. + * + * @param arg_source A string left to the interpretation of the reader + * implementation; it corresponds to the value configured on the + * script-level for the input stream. + * + * @param num_fields The number of log fields for the stream. + * + * @param fields An array of size \a num_fields with the log fields. + * The methods takes ownership of the array. + * + * @return False if an error occured. + */ bool Init(string arg_source); + /** + * Add an input filter to the input stream + * + * @param id identifier of the input stream + * + * @param arg_num_fields number of fields contained in \a fields + * + * @param fields the types and names of the fields to be retrieved from the input source + * + * @return False if an error occured. + */ bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + + /** + * Remove an input filter to the input stream + * + * @param id identifier of the input stream + * + * @return False if an error occured. + */ bool RemoveFilter ( int id ); + /** + * Finishes reading from this input stream in a regular fashion. Must not be + * called if an error has been indicated earlier. After calling this, + * no further reading from the stream can be performed + * + * @return False if an error occured. + */ void Finish(); + /** + * Force trigger an update of the input stream. + * The action that will be taken depends on the current read mode and the individual input backend + * + * An backend can choose to ignore this. + * + * @return False if an error occured. + */ bool Update(); /** @@ -34,30 +104,126 @@ public: void DisableFrontend(); protected: - // Methods that have to be overwritten by the individual readers + // Methods that have to be overwritten by the individual readers + + /** + * Reader-specific intialization method. + * + * A reader implementation must override this method. If it returns + * false, it will be assumed that a fatal error has occured that + * prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an + * implementation should also call Error() to indicate what happened. + */ virtual bool DoInit(string arg_sources) = 0; + /** + * Reader-specific method to add a filter. + * + * A reader implementation must override this method. + */ virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; + /** + * Reader-specific method to remove a filter. + * + * A reader implementation must override this method. + */ virtual bool DoRemoveFilter( int id ) = 0; + /** + * Reader-specific method implementing input finalization at + * termination. + * + * A reader implementation must override this method but it can just + * ignore calls if an input source must not be closed. + * + * After the method is called, the writer will be deleted. If an error occurs + * during shutdown, an implementation should also call Error() to indicate what + * happened. + */ virtual void DoFinish() = 0; - // update file contents to logmgr + /** + * Reader-specific method implementing the forced update trigger + * + * A reader implementation must override this method but it can just ignore + * calls, if a forced update does not fit the input source or the current input + * reading mode + */ virtual bool DoUpdate() = 0; - // The following methods return the information as passed to Init(). + /** + * Returns the input source as passed into the constructor. + */ const string Source() const { return source; } + /** + * Method allowing a reader to send a specified bro event. + * Vals must match the values expected by the bro event. + * + * @param name name of the bro event to send + * + * @param num_vals number of entries in \a vals + * + * @param vals the values to be given to the event + */ void SendEvent(const string& name, const int num_vals, threading::Value* *vals); - // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + // Content-sending-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + /** + * Method allowing a reader to send a list of values read for a specific filter back to the manager. + * + * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised + * + * @param id the input filter id for which the values are sent + * + * @param val list of threading::Values expected by the filter + */ void Put(int id, threading::Value* *val); + + /** + * Method allowing a reader to delete a specific value from a bro table. + * + * If the receiving filter is an event, only a removed event is raised + * + * @param id the input filter id for which the values are sent + * + * @param val list of threading::Values expected by the filter + */ void Delete(int id, threading::Value* *val); + + /** + * Method allowing a reader to clear a value from a bro table. + * + * If the receiving filter is an event, this is ignored. + * + * @param id the input filter id for which the values are sent + */ void Clear(int id); - // Table-functions (tracking mode): Only changed lines are propagated. + // Content-sending-functions (tracking mode): Only changed lines are propagated. + + + /** + * Method allowing a reader to send a list of values read for a specific filter back to the manager. + * + * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised. + * + * @param id the input filter id for which the values are sent + * + * @param val list of threading::Values expected by the filter + */ void SendEntry(int id, threading::Value* *vals); + + /** + * Method telling the manager, that the current list of entries sent by SendEntry is finished. + * + * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer + * present in the input source + * + * @param id the input filter id for which the values are sent + */ void EndCurrentSend(int id); @@ -68,11 +234,7 @@ private: string source; - // When an error occurs, this method is called to set a flag marking the - // writer as disabled. - bool disabled; - bool Disabled() { return disabled; } // For implementing Fmt(). char* buf; diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 97433c8af6..c29071612d 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -51,15 +51,37 @@ public: */ void Init(string arg_source); + /** + * Force an update of the current input source. Actual action depends on + * the opening mode and on the input source. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. + * This method must only be called from the main thread. + */ void Update(); - /* * The method takes - * ownership of \a fields. */ - + /** + * Add a filter to the current input source. + * + * See ReaderBackend::AddFilter for arguments. + * + * The method takes ownership of \a fields + */ void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); + /** + * Removes a filter to the current input source. + */ void RemoveFilter ( const int id ); + /** + * Finalizes writing to this tream. + * + * This method generates a message to the backend reader and triggers + * the corresponding message there. + * This method must only be called from the main thread. + */ void Finish(); /** @@ -92,6 +114,9 @@ public: protected: friend class Manager; + /** + * Returns the source as passed into the constructor + */ const string Source() const { return source; }; string ty_name; // Name of the backend type. Set by the manager. diff --git a/src/types.bif b/src/types.bif index 9256fe3bd0..1529319197 100644 --- a/src/types.bif +++ b/src/types.bif @@ -185,4 +185,10 @@ enum ID %{ Unknown, %} +enum Mode %{ + MANUAL, + REREAD, + STREAM, +%} + module GLOBAL; From 91943c26559590c858f8c7e30db6ffc41a68ec59 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 15:03:20 -0800 Subject: [PATCH 061/123] * rework script interface, add autostart stream flag that starts up a stream automatically when first filter has been added ( probably the most common use case ) * change internal reader interface again * remove some quite embarassing bugs that must have been in the interface for rather long * add different read methods to script & internal interface (like normal, streaming, etc). Not implemented in ascii reader yet. --- scripts/base/frameworks/input/main.bro | 3 + src/input/Manager.cc | 24 ++++---- src/input/ReaderBackend.cc | 32 +++++++++- src/input/ReaderBackend.h | 46 +++++++++++++-- src/input/ReaderFrontend.cc | 31 ++++++++-- src/input/ReaderFrontend.h | 13 ++++- src/input/readers/Ascii.cc | 58 +++++++++++++++++-- src/input/readers/Ascii.h | 10 +++- src/threading/Manager.cc | 2 +- src/types.bif | 6 +- .../scripts/base/frameworks/input/basic.bro | 1 - .../scripts/base/frameworks/input/event.bro | 1 - .../frameworks/input/onecolumn-norecord.bro | 1 - .../frameworks/input/onecolumn-record.bro | 1 - .../scripts/base/frameworks/input/port.bro | 1 - .../base/frameworks/input/predicate.bro | 1 - .../base/frameworks/input/tableevent.bro | 1 - .../base/frameworks/input/twofilters.bro | 2 +- 18 files changed, 191 insertions(+), 43 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 7e581070e6..445f947106 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -23,6 +23,9 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; + + ## Automatically start the input stream after the first filter has been added + autostart: bool &default=T; }; ## TableFilter description type used for the `add_tablefilter` method. diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d4e5cdaee9..4438a07c6c 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -214,6 +214,10 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); + Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); + bool do_autostart = ( autostart->InternalInt() == 1 ); + Unref(autostart); // Ref'd by LookupWithDefault ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); assert(reader_obj); @@ -229,16 +233,7 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) readers.push_back(info); - reader_obj->Init(source); - /* if ( success == false ) { - assert( RemoveStream(id) ); - return 0; - } */ - reader_obj->Update(); - /* if ( success == false ) { - assert ( RemoveStream(id) ); - return 0; - } */ + reader_obj->Init(source, mode->InternalInt(), do_autostart); return reader_obj; @@ -785,7 +780,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = 0; if ( filter->num_val_fields > 0 ) - HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + valhash = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -794,6 +789,13 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before + + valhash->Hash(); + + h->valhash->Hash(); + + + if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { // ok, exact duplicate filter->lastDict->Remove(idxhash); diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 5cb4fe34f2..cfc74d33a8 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -185,24 +185,42 @@ void ReaderBackend::SendEntry(int id, Value* *vals) SendOut(new SendEntryMessage(frontend, id, vals)); } -bool ReaderBackend::Init(string arg_source) +bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) { source = arg_source; + autostart = arg_autostart; + SetName("InputReader/"+source); // disable if DoInit returns error. - disabled = !DoInit(arg_source); + disabled = !DoInit(arg_source, mode); if ( disabled ) { + Error("Init failed"); DisableFrontend(); } return !disabled; } +bool ReaderBackend::StartReading() { + int success = DoStartReading(); + + if ( success == false ) { + DisableFrontend(); + } + + return success; +} + bool ReaderBackend::AddFilter(int id, int arg_num_fields, const Field* const * arg_fields) { - return DoAddFilter(id, arg_num_fields, arg_fields); + bool success = DoAddFilter(id, arg_num_fields, arg_fields); + if ( success && autostart) { + autostart = false; + return StartReading(); + } + return success; } bool ReaderBackend::RemoveFilter(int id) @@ -230,4 +248,12 @@ void ReaderBackend::DisableFrontend() SendOut(new DisableMessage(frontend)); } +bool ReaderBackend::DoHeartbeat(double network_time, double current_time) +{ + MsgThread::DoHeartbeat(network_time, current_time); + + return true; +} + + } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index c6fbaac715..e34db3e559 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -51,9 +51,24 @@ public: * @param fields An array of size \a num_fields with the log fields. * The methods takes ownership of the array. * + * @param mode the opening mode for the input source + * + * @param autostart automatically start the input source after the first filter has been added + * * @return False if an error occured. */ - bool Init(string arg_source); + bool Init(string arg_source, int mode, bool autostart); + + /** + * One-time start method of the reader. + * + * This method is called from the scripting layer, after all filters have been added. + * No data should be read before this method is called. + * + * If autostart in Init is set to true, this method is called automatically by the backend after + * the first filter has been added. + */ + bool StartReading(); /** * Add an input filter to the input stream @@ -107,7 +122,8 @@ protected: // Methods that have to be overwritten by the individual readers /** - * Reader-specific intialization method. + * Reader-specific intialization method. Note that data may only be read from the input source + * after the Start function has been called. * * A reader implementation must override this method. If it returns * false, it will be assumed that a fatal error has occured that @@ -115,7 +131,19 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoInit(string arg_sources) = 0; + virtual bool DoInit(string arg_sources, int mode) = 0; + + /** + * Reader-specific start method. After this function has been called, data may be read from + * the input source and be sent to the specified filters + * + * A reader implementation must override this method. + * If it returns false, it will be assumed that a fatal error has occured + * that prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an implementation + * should also call Error to indicate what happened. + */ + virtual bool DoStartReading() = 0; /** * Reader-specific method to add a filter. @@ -225,7 +253,14 @@ protected: * @param id the input filter id for which the values are sent */ void EndCurrentSend(int id); - + + /** + * Triggered by regular heartbeat messages from the main thread. + * + * This method can be overridden but once must call + * ReaderBackend::DoHeartbeat(). + */ + virtual bool DoHeartbeat(double network_time, double current_time); private: // Frontend that instantiated us. This object must not be access from @@ -238,7 +273,8 @@ private: // For implementing Fmt(). char* buf; - unsigned int buf_len; + unsigned int buf_len; + bool autostart; }; } diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 0fdf90d9ad..f7fc23bf72 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,14 +12,16 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source) + InitMessage(ReaderBackend* backend, const string source, const int mode, const bool autostart) : threading::InputMessage("Init", backend), - source(source) { } + source(source), mode(mode), autostart(autostart) { } - virtual bool Process() { return Object()->Init(source); } + virtual bool Process() { return Object()->Init(source, mode, autostart); } private: const string source; + const int mode; + const bool autostart; }; class UpdateMessage : public threading::InputMessage @@ -42,6 +44,16 @@ public: virtual bool Process() { Object()->Finish(); return true; } }; +class StartReadingMessage : public threading::InputMessage +{ +public: + StartReadingMessage(ReaderBackend* backend) + : threading::InputMessage("StartReading", backend) + { } + + virtual bool Process() { Object()->StartReading(); return true; } +}; + class AddFilterMessage : public threading::InputMessage { public: @@ -83,17 +95,17 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) { ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source) { +void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { if ( disabled ) return; if ( initialized ) - reporter->InternalError("writer initialize twice"); + reporter->InternalError("reader initialize twice"); source = arg_source; initialized = true; - backend->SendIn(new InitMessage(backend, arg_source)); + backend->SendIn(new InitMessage(backend, arg_source, mode, autostart)); } void ReaderFrontend::Update() { @@ -132,6 +144,13 @@ string ReaderFrontend::Name() const return ty_name + "/" + source; } +void ReaderFrontend::StartReading() { + if ( disabled ) + return; + + backend->SendIn(new StartReadingMessage(backend)); +} + } diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index c29071612d..d67ca299c0 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -49,7 +49,18 @@ public: * See ReaderBackend::Init() for arguments. * This method must only be called from the main thread. */ - void Init(string arg_source); + void Init(string arg_source, int mode, bool autostart); + + /** + * Start the reader. + * + * This methods starts the reader, after all necessary filters have been added. + * It is not necessary to call this function, if autostart has been set. + * If autostart has been set, the reader will be initialized automatically after the first filter has been added + * + * This method must only be called from the main thread. + */ + void StartReading(); /** * Force an update of the current input source. Actual action depends on diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 095d74bf11..cd1723e5e4 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -8,10 +8,14 @@ #include "../../threading/SerializationTypes.h" +#define MANUAL 0 +#define REREAD 1 + using namespace input::reader; using threading::Value; using threading::Field; + FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) { @@ -75,16 +79,41 @@ void Ascii::DoFinish() } } -bool Ascii::DoInit(string path) +bool Ascii::DoInit(string path, int arg_mode) { + started = false; fname = path; + mode = arg_mode; file = new ifstream(path.c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); + Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } + if ( ( mode != MANUAL ) && (mode != REREAD) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + + return true; +} + +bool Ascii::DoStartReading() { + if ( started == true ) { + Error("Started twice"); + return false; + } + + started = true; + switch ( mode ) { + case MANUAL: + DoUpdate(); + break; + default: + assert(false); + } + return true; } @@ -132,7 +161,7 @@ bool Ascii::ReadHeader() { map fields; - // construcr list of field names. + // construct list of field names. istringstream splitstream(line); int pos=0; while ( splitstream ) { @@ -146,6 +175,7 @@ bool Ascii::ReadHeader() { for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + (*it).second.columnMap.clear(); for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { const Field* field = (*it).second.fields[i]; @@ -372,7 +402,6 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { file->close(); @@ -418,6 +447,7 @@ bool Ascii::DoUpdate() { fit != (*it).second.columnMap.end(); fit++ ){ + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; @@ -455,6 +485,7 @@ bool Ascii::DoUpdate() { } + //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. @@ -463,3 +494,22 @@ bool Ascii::DoUpdate() { } return true; } + +bool Ascii::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( mode ) { + case MANUAL: + // yay, we do nothing :) + break; + case REREAD: + + + default: + assert(false); + } + + return true; +} + diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index a3bf5c21a6..766716e29d 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -39,7 +39,7 @@ public: protected: - virtual bool DoInit(string path); + virtual bool DoInit(string path, int mode); virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); @@ -48,9 +48,13 @@ protected: virtual void DoFinish(); virtual bool DoUpdate(); + + virtual bool DoStartReading(); private: + virtual bool DoHeartbeat(double network_time, double current_time); + struct Filter { unsigned int num_fields; @@ -84,6 +88,10 @@ private: string unset_field; + int mode; + + bool started; + }; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 7b571e753c..472d10139a 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -106,7 +106,7 @@ void Manager::Process() Message* msg = t->RetrieveOut(); - if ( msg->Process() && network_time ) + if ( msg->Process() ) //&& network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init. did_process = true; else diff --git a/src/types.bif b/src/types.bif index 1529319197..e2a47a7ece 100644 --- a/src/types.bif +++ b/src/types.bif @@ -186,9 +186,9 @@ enum ID %{ %} enum Mode %{ - MANUAL, - REREAD, - STREAM, + MANUAL = 0, + REREAD = 1, + STREAM = 2, %} module GLOBAL; diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 3b75220625..156898edca 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -47,7 +47,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::INPUT); Input::remove_tablefilter(A::INPUT, "ssh"); Input::remove_stream(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index a07f0934a0..41eba1613c 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -38,5 +38,4 @@ event bro_init() { Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); - Input::force_update(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 712a877960..bcbba05a3e 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -33,7 +33,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); - Input::force_update(A::INPUT); } event Input::update_finished(id: Input::ID) { diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 7b62ddcddd..1c532ba6a9 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -33,7 +33,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::INPUT); } event Input::update_finished(id: Input::ID) { diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 65d73c54f7..801d6bac3f 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -32,7 +32,6 @@ event bro_init() # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::INPUT); print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index bc1ab89bb2..009911e6a8 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -41,7 +41,6 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::force_update(A::INPUT); } event Input::update_finished(id: Input::ID) { diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 36e8171689..0c86ac94b8 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -44,5 +44,4 @@ event bro_init() { Input::create_stream(A::LOG, [$source="input.log"]); Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); - Input::force_update(A::LOG); } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index d5bff0c5bb..260f73e58f 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -40,7 +40,7 @@ global done: bool = F; event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); + Input::create_stream(A::INPUT, [$source="input.log", $autostart=F]); Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); From d21a450f36ba621f5802f9c1c7b9f28ce2ec264d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 16 Feb 2012 15:40:07 -0800 Subject: [PATCH 062/123] add streaming reads & automatic re-reading of files to ascii reader. completely untested, but compiles & old tests still work --- src/input/Manager.cc | 7 ---- src/input/readers/Ascii.cc | 70 ++++++++++++++++++++++++++++++-------- src/input/readers/Ascii.h | 1 + 3 files changed, 57 insertions(+), 21 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 4438a07c6c..ea4c5643fa 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -789,13 +789,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - - valhash->Hash(); - - h->valhash->Hash(); - - - if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { // ok, exact duplicate filter->lastDict->Remove(idxhash); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index cd1723e5e4..5a3569a95f 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -10,6 +10,11 @@ #define MANUAL 0 #define REREAD 1 +#define STREAM 2 + +#include +#include +#include using namespace input::reader; using threading::Value; @@ -84,6 +89,7 @@ bool Ascii::DoInit(string path, int arg_mode) started = false; fname = path; mode = arg_mode; + mtime = 0; file = new ifstream(path.c_str()); if ( !file->is_open() ) { @@ -91,7 +97,7 @@ bool Ascii::DoInit(string path, int arg_mode) return false; } - if ( ( mode != MANUAL ) && (mode != REREAD) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; } @@ -402,23 +408,58 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) - if ( file && file->is_open() ) { - file->close(); - } - file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); - return false; + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } + + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } + + mtime = sb.st_mtime; + // file changed. reread. + + // fallthrough + case MANUAL: + case STREAM: + + // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) + if ( file && file->is_open() ) { + if ( mode == STREAM ) { + file->clear(); // remove end of file evil bits + break; + } + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + + + if ( ReadHeader() == false ) { + return false; + } + + break; + default: + assert(false); + } + + // // file->seekg(0, ios::beg); // do not forget clear. - if ( ReadHeader() == false ) { - return false; - } string line; while ( GetLine(line ) ) { @@ -504,8 +545,9 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) // yay, we do nothing :) break; case REREAD: - - + case STREAM: + DoUpdate(); + break; default: assert(false); } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 766716e29d..017e5630d4 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -91,6 +91,7 @@ private: int mode; bool started; + time_t mtime; }; From 4126b458ca002093bf964243bb5cdd8a45931544 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 13:18:15 -0800 Subject: [PATCH 063/123] Automatic file re-refresh and streaming works. * simple testcase for file refresh (check for changes) and streaming reads * add events for simple put and delete operations * fix bugs in table filter events (type for first element was wrong) * and I think a couple of other small bugs --- .../scripts.base.frameworks.input.reread/out | 67 ++++++++++ .../scripts.base.frameworks.input.stream/out | 115 ++++++++++++++++++ .../scripts/base/frameworks/input/reread.bro | 92 ++++++++++++++ .../scripts/base/frameworks/input/stream.bro | 89 ++++++++++++++ 4 files changed, 363 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.reread/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.stream/out create mode 100644 testing/btest/scripts/base/frameworks/input/reread.bro create mode 100644 testing/btest/scripts/base/frameworks/input/stream.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out new file mode 100644 index 0000000000..4234a5056d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -0,0 +1,67 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.stream/out b/testing/btest/Baseline/scripts.base.frameworks.input.stream/out new file mode 100644 index 0000000000..39b06c9092 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.stream/out @@ -0,0 +1,115 @@ +============EVENT============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_CHANGED +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro new file mode 100644 index 0000000000..5058f4a068 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -0,0 +1,92 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); +} + +event Input::update_finished(id: Input::ID) { + print outfile, servers; + + try = try + 1; + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); + } +} diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro new file mode 100644 index 0000000000..db368074aa --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -0,0 +1,89 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, tpe; + print outfile, left; + print outfile, right; + print outfile, "============SERVERS============"; + print outfile, servers; + + try = try + 1; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::STREAM]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); +} + From 4f57817b1a7af9b4084a22d00e195c46afa3c3af Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 13:20:29 -0800 Subject: [PATCH 064/123] ...forgotten in last commit. --- src/input/Manager.cc | 163 +++++++++++++++++++++++++++++++++---- src/input/readers/Ascii.cc | 56 +++++++++---- src/input/readers/Ascii.h | 5 +- 3 files changed, 191 insertions(+), 33 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index ea4c5643fa..66dadfdb2d 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -818,13 +818,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } - Val* oldval = 0; - if ( updated == true ) { - assert(filter->num_val_fields > 0); - // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); - } - // call filter first to determine if we really add / change the entry if ( filter->pred ) { @@ -865,6 +858,13 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } + Val* oldval = 0; + if ( updated == true ) { + assert(filter->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval); + } + //i->tab->Assign(idxval, valval); HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { @@ -884,21 +884,22 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va if ( filter->event ) { EnumVal* ev; - Ref(idxval); + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); if ( updated ) { // in case of update send back the old value. assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(filter->event, 3, ev, idxval, oldval); + SendEvent(filter->event, 3, ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, idxval); + SendEvent(filter->event, 3, ev, predidx); } else { - SendEvent(filter->event, 3, ev, idxval, valval); + SendEvent(filter->event, 3, ev, predidx, valval); } } } @@ -973,10 +974,11 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { } if ( filter->event ) { - Ref(idx); + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(filter->event, 3, ev, idx, val); + SendEvent(filter->event, 3, ev, predidx, val); } filter->tab->Delete(ih->idxkey); @@ -991,8 +993,6 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->currDict = new PDict(InputHash); // Send event that the current update is indeed finished. - - EventHandler* handler = event_registry->Lookup("Input::update_finished"); if ( handler == 0 ) { reporter->InternalError("Input::update_finished not found!"); @@ -1077,6 +1077,7 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; + int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { @@ -1087,7 +1088,91 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * valval = ValueToRecordVal(vals, filter->rtype, &position); } - filter->tab->Assign(idxval, valval); + // if we have a subscribed event, we need to figure out, if this is an update or not + // same for predicates + if ( filter->pred || filter->event ) { + bool updated = false; + Val* oldval = 0; + + if ( filter->num_val_fields > 0 ) { + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval, false); + } + + if ( oldval != 0 ) { + // it is an update + updated = true; + Ref(oldval); // have to do that, otherwise it may disappear in assign + } + + + // predicate if we want the update or not + if ( filter->pred ) { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + Ref(valval); + + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. + vl.append(ev); + vl.append(predidx); + if ( filter->num_val_fields > 0 ) + vl.append(valval); + + Val* v = filter->pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // do nothing + Unref(idxval); + Unref(valval); + Unref(oldval); + return filter->num_val_fields + filter->num_idx_fields; + } + + } + + + filter->tab->Assign(idxval, valval); + + if ( filter->event ) { + EnumVal* ev; + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + + if ( updated ) { // in case of update send back the old value. + assert ( filter->num_val_fields > 0 ); + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + SendEvent(filter->event, 3, ev, predidx, oldval); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + Ref(valval); + if ( filter->num_val_fields == 0 ) { + SendEvent(filter->event, 3, ev, predidx); + } else { + SendEvent(filter->event, 3, ev, predidx, valval); + } + } + + } + + + + + + } else { + // no predicates or other stuff + + filter->tab->Assign(idxval, valval); + } return filter->num_idx_fields + filter->num_val_fields; } @@ -1122,8 +1207,52 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { if ( i->filters[id]->filter_type == TABLE_FILTER ) { TableFilter* filter = (TableFilter*) i->filters[id]; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; - success = ( filter->tab->Delete(idxval) != 0 ); + bool filterresult = true; + + if ( filter->pred || filter->event ) { + Val *val = filter->tab->Lookup(idxval); + + if ( filter->pred ) { + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + int startpos = 0; + Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + + val_list vl(3); + vl.append(ev); + vl.append(predidx); + vl.append(val); + Val* v = filter->pred->Call(&vl); + filterresult = v->AsBool(); + Unref(v); + + if ( filterresult == false ) { + // keep it. + Unref(idxval); + success = true; + } + + } + + // only if filter = true -> no filtering + if ( filterresult && filter->event ) { + Ref(idxval); + assert(val != 0); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(filter->event, 3, ev, idxval, val); + } + } + + // only if filter = true -> no filtering + if ( filterresult ) { + success = ( filter->tab->Delete(idxval) != 0 ); + if ( !success ) { + reporter->Error("Internal error while deleting values from input table"); + } + } } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); readVals = SendEventFilterEvent(reader, type, id, vals); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 5a3569a95f..b0b046b75b 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -91,16 +91,22 @@ bool Ascii::DoInit(string path, int arg_mode) mode = arg_mode; mtime = 0; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + file = new ifstream(path.c_str()); if ( !file->is_open() ) { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } - - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + + if ( ReadHeader(false) == false ) { + Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); + file->close(); return false; - } + } return true; } @@ -114,6 +120,8 @@ bool Ascii::DoStartReading() { started = true; switch ( mode ) { case MANUAL: + case REREAD: + case STREAM: DoUpdate(); break; default: @@ -157,16 +165,25 @@ bool Ascii::HasFilter(int id) { } -bool Ascii::ReadHeader() { +bool Ascii::ReadHeader(bool useCached) { // try to read the header line... string line; - if ( !GetLine(line) ) { - Error("could not read first line"); - return false; - } - map fields; + if ( !useCached ) { + if ( !GetLine(line) ) { + Error("could not read first line"); + return false; + } + + + + headerline = line; + + } else { + line = headerline; + } + // construct list of field names. istringstream splitstream(line); int pos=0; @@ -179,7 +196,7 @@ bool Ascii::ReadHeader() { pos++; } - + //printf("Updating fields from description %s\n", line.c_str()); for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { (*it).second.columnMap.clear(); @@ -433,6 +450,7 @@ bool Ascii::DoUpdate() { if ( file && file->is_open() ) { if ( mode == STREAM ) { file->clear(); // remove end of file evil bits + ReadHeader(true); // in case filters changed break; } file->close(); @@ -444,7 +462,7 @@ bool Ascii::DoUpdate() { } - if ( ReadHeader() == false ) { + if ( ReadHeader(false) == false ) { return false; } @@ -512,9 +530,14 @@ bool Ascii::DoUpdate() { fpos++; } + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( (unsigned int) fpos == (*it).second.num_fields ); - SendEntry((*it).first, fields); + if ( mode == STREAM ) { + Put((*it).first, fields); + } else { + SendEntry((*it).first, fields); + } /* Do not do this, ownership changes to other thread * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { @@ -530,9 +553,12 @@ bool Ascii::DoUpdate() { //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - EndCurrentSend((*it).first); + if ( mode != STREAM ) { + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + EndCurrentSend((*it).first); + } } + return true; } diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 017e5630d4..d2376e4fe1 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -69,7 +69,7 @@ private: TransportProto StringToProto(const string &proto); - bool ReadHeader(); + bool ReadHeader(bool useCached); threading::Value* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); @@ -87,6 +87,9 @@ private: string empty_field; string unset_field; + + // keep a copy of the headerline to determine field locations when filters change + string headerline; int mode; From fe5b376d2858d599c2a14ddc8003c59c217550c5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 13:23:25 -0800 Subject: [PATCH 065/123] ...and update for table event testcase after fix. --- .../scripts.base.frameworks.input.tableevent/out | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index e32a2aea00..54048a86b8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,21 +1,21 @@ Input::EVENT_NEW -1 +[i=1] T Input::EVENT_NEW -2 +[i=2] T Input::EVENT_NEW -3 +[i=3] F Input::EVENT_NEW -4 +[i=4] F Input::EVENT_NEW -5 +[i=5] F Input::EVENT_NEW -6 +[i=6] F Input::EVENT_NEW -7 +[i=7] T From edd30da082d288d295939429689c0a74a0787340 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 15:30:21 -0800 Subject: [PATCH 066/123] better testcase & fix a few bugs (that took way too long to find). --- src/input/Manager.cc | 34 ++++++++----- src/input/readers/Ascii.cc | 21 +++++--- .../scripts.base.frameworks.input.reread/out | 48 +++++++++++++++++++ .../scripts/base/frameworks/input/reread.bro | 11 ++++- 4 files changed, 95 insertions(+), 19 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 66dadfdb2d..243567e0e6 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -22,7 +22,7 @@ using threading::Value; using threading::Field; struct InputHash { - HashKey* valhash; + hash_t valhash; HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. }; @@ -776,11 +776,15 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + //reporter->Error("Result: %d\n", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = 0; - if ( filter->num_val_fields > 0 ) - valhash = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + + hash_t valhash = 0; + if ( filter->num_val_fields > 0 ) { + HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); + valhash = valhashkey->Hash(); + delete(valhashkey); + } //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -789,7 +793,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { + if ( filter->num_val_fields == 0 || h->valhash == valhash ) { // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); @@ -862,7 +866,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va if ( updated == true ) { assert(filter->num_val_fields > 0); // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); + oldval = filter->tab->Lookup(idxval, false); } //i->tab->Assign(idxval, valval); @@ -872,6 +876,8 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va return filter->num_val_fields + filter->num_idx_fields; } + if ( filter->event && updated ) + Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); @@ -891,7 +897,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - Ref(oldval); SendEvent(filter->event, 3, ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); @@ -1468,7 +1473,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { int length = 0; memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); length += sizeof(val->val.port_val.port); - memcpy(data+startpos, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); + memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); length += sizeof(val->val.port_val.proto); return length; break; @@ -1500,7 +1505,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { int length = 0; memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); length += sizeof(val->val.subnet_val.width); - memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + memcpy(data+startpos+length, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); length += sizeof(val->val.subnet_val.net); return length; break; @@ -1508,7 +1513,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_TABLE: { int length = 0; - for ( int i = 0; i < val->val.set_val.size; i++ ) { + int j = val->val.set_val.size; + for ( int i = 0; i < j; i++ ) { length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); } return length; @@ -1531,6 +1537,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { } reporter->InternalError("internal error"); + assert(false); return 0; } @@ -1550,13 +1557,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } + memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; position += CopyValue(data, position, val); } + hash_t key = HashKey::HashBytes(data, length); + assert(position == length); - return new HashKey(data, length); + return new HashKey(data, length, key, true); } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index b0b046b75b..d4b3d91e00 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -268,7 +268,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { if ( s.compare(unset_field) == 0 ) { // field is not set... return new Value(field.type, false); } - + switch ( field.type ) { case TYPE_ENUM: case TYPE_STRING: @@ -302,6 +302,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { break; case TYPE_PORT: + val->val.port_val.port = 0; val->val.port_val.port = atoi(s.c_str()); val->val.port_val.proto = TRANSPORT_UNKNOWN; break; @@ -312,19 +313,27 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // NOTE: dotted_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... #ifdef BROv6 if ( s.find(':') != s.npos ) { - uint32* addr = dotted_to_addr6(s.c_str()); + uint32* addr = new uint32[4]; + if ( inet_pton(AF_INET6, s.c_str(), addr) <= 0 ) { + Error(Fmt("Bad IPv6 address: %s", s.c_str())); + val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = val->val.subnet_val.net[3] = 0; + } copy_addr(val->val.subnet_val.net, addr); delete addr; } else { val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = 0; - val->val.subnet_val.net[3] = dotted_to_addr(s.c_str()); + if ( inet_aton(s.c_str(), &(val->val.subnet_val.net[3])) <= 0 ) { + Error(Fmt("Bad addres: %s", s.c_str())); + val->val.subnet_val.net[3] = 0; + } } #else - val->val.subnet_val.net = dotted_to_addr(s.c_str()); + if ( inet_aton(s.c_str(), (in_addr*) &(val->val.subnet_val.net)) <= 0 ) { + Error(Fmt("Bad addres: %s", s.c_str())); + val->val.subnet_val.net = 0; + } #endif break; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 4234a5056d..9516cb2a92 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1,3 +1,19 @@ +============EVENT============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ { [-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -12,6 +28,22 @@ BB }, vc=[10, 20, 30], ve=[]] } +============EVENT============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ { [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -38,6 +70,22 @@ BB }, vc=[10, 20, 30], ve=[]] } +============EVENT============ +Input::EVENT_CHANGED +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ { [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 5058f4a068..58df37af84 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -70,16 +70,25 @@ global outfile: file; global try: count; +event line(tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, tpe; + print outfile, left; + print outfile, right; +} + event bro_init() { outfile = open ("../out"); try = 0; # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); } + event Input::update_finished(id: Input::ID) { + print outfile, "==========SERVERS============"; print outfile, servers; try = try + 1; From d5b413c4e719eed31a998c7ddc4d5361c8fbb670 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 20 Feb 2012 17:13:41 -0800 Subject: [PATCH 067/123] reduce number of needed hash operations --- src/input/Manager.cc | 26 ++++++++------------------ src/input/Manager.h | 2 -- src/input/readers/Ascii.cc | 1 - 3 files changed, 8 insertions(+), 21 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 243567e0e6..a7afdc3a78 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -782,8 +782,8 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); - valhash = valhashkey->Hash(); - delete(valhashkey); + valhash = valhashkey->Hash(); + delete(valhashkey); } //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); @@ -873,19 +873,17 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); - return filter->num_val_fields + filter->num_idx_fields; + assert(false); } + InputHash* ih = new InputHash(); + ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); + ih->valhash = valhash; + if ( filter->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); - InputHash* ih = new InputHash(); - k = filter->tab->ComputeHash(idxval); - ih->idxkey = k; - ih->valhash = valhash; - //i->tab->Delete(k); - filter->currDict->Insert(idxhash, ih); if ( filter->event ) { @@ -1557,7 +1555,7 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } - memset(data, 0, length); + //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; position += CopyValue(data, position, val); @@ -1695,11 +1693,3 @@ Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) return 0; } - -string Manager::Hash(const string &input) { - unsigned char digest[16]; - hash_md5(input.length(), (const unsigned char*) input.c_str(), digest); - string out((const char*) digest, 16); - return out; -} - diff --git a/src/input/Manager.h b/src/input/Manager.h index be84ee416d..b4fc6cff7f 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -202,8 +202,6 @@ private: vector readers; - string Hash(const string &input); - class Filter; class TableFilter; class EventFilter; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index d4b3d91e00..e128cd1164 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -302,7 +302,6 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { break; case TYPE_PORT: - val->val.port_val.port = 0; val->val.port_val.port = atoi(s.c_str()); val->val.port_val.proto = TRANSPORT_UNKNOWN; break; From 531189b5fdd39fb9f927b8af075957ad2892b9d4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 22 Feb 2012 08:56:45 -0800 Subject: [PATCH 068/123] try to make ascii reader a little bit more robust to failure - mainly ignore messages after a reader has disabled itself --- src/input/ReaderBackend.cc | 20 +++++++++++++++++++- src/input/ReaderBackend.h | 7 ++++++- src/input/readers/Ascii.cc | 8 ++++++-- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index cfc74d33a8..8ddb6a2f42 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -203,6 +203,9 @@ bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) } bool ReaderBackend::StartReading() { + if ( disabled ) + return false; + int success = DoStartReading(); if ( success == false ) { @@ -215,6 +218,9 @@ bool ReaderBackend::StartReading() { bool ReaderBackend::AddFilter(int id, int arg_num_fields, const Field* const * arg_fields) { + if ( disabled ) + return false; + bool success = DoAddFilter(id, arg_num_fields, arg_fields); if ( success && autostart) { autostart = false; @@ -225,6 +231,9 @@ bool ReaderBackend::AddFilter(int id, int arg_num_fields, bool ReaderBackend::RemoveFilter(int id) { + if ( disabled ) + return false; + bool success = DoRemoveFilter(id); SendOut(new FilterRemovedMessage(frontend, id)); return success; // yes, I know, noone reads this. @@ -240,11 +249,20 @@ void ReaderBackend::Finish() bool ReaderBackend::Update() { - return DoUpdate(); + if ( disabled ) + return false; + + bool success = DoUpdate(); + if ( !success ) { + DisableFrontend(); + } + + return success; } void ReaderBackend::DisableFrontend() { + disabled = true; // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now SendOut(new DisableMessage(frontend)); } diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index e34db3e559..68fd5f3a37 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -177,7 +177,12 @@ protected: * * A reader implementation must override this method but it can just ignore * calls, if a forced update does not fit the input source or the current input - * reading mode + * reading mode. + * + * If it returns false, it will be assumed that a fatal error has occured + * that prevents the reader from further operation; it will then be + * disabled and eventually deleted. When returning false, an implementation + * should also call Error to indicate what happened. */ virtual bool DoUpdate() = 0; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index e128cd1164..73b8500d5e 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -458,7 +458,10 @@ bool Ascii::DoUpdate() { if ( file && file->is_open() ) { if ( mode == STREAM ) { file->clear(); // remove end of file evil bits - ReadHeader(true); // in case filters changed + if ( !ReadHeader(true) ) // in case filters changed + { + return false; // header reading failed + } break; } file->close(); @@ -522,6 +525,7 @@ bool Ascii::DoUpdate() { Value* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { + Error("Could not convert String value to Val"); return false; } @@ -580,7 +584,7 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - DoUpdate(); + Update(); // call update and not DoUpdate, because update actually checks disabled. break; default: assert(false); From 7e5f7338269d22332950b2ed9dd446c673fa21c3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 22 Feb 2012 09:44:45 -0800 Subject: [PATCH 069/123] raw input reader for seth, which can simply read a file into string-events given a line separator. --- scripts/base/frameworks/input/__load__.bro | 1 + scripts/base/frameworks/input/readers/raw.bro | 9 + src/CMakeLists.txt | 1 + src/input.bif | 2 + src/input/Manager.cc | 2 + src/input/readers/Ascii.cc | 6 +- src/input/readers/Raw.cc | 230 ++++++++++++++++++ src/input/readers/Raw.h | 70 ++++++ src/types.bif | 1 + .../scripts.base.frameworks.input.raw/out | 8 + .../scripts/base/frameworks/input/raw.bro | 35 +++ 11 files changed, 363 insertions(+), 2 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/raw.bro create mode 100644 src/input/readers/Raw.cc create mode 100644 src/input/readers/Raw.h create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.raw/out create mode 100644 testing/btest/scripts/base/frameworks/input/raw.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index a3315186d5..b41fe5e95f 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1,3 +1,4 @@ @load ./main @load ./readers/ascii +@load ./readers/raw diff --git a/scripts/base/frameworks/input/readers/raw.bro b/scripts/base/frameworks/input/readers/raw.bro new file mode 100644 index 0000000000..45deed3eda --- /dev/null +++ b/scripts/base/frameworks/input/readers/raw.bro @@ -0,0 +1,9 @@ +##! Interface for the raw input reader. + +module InputRaw; + +export { + ## Separator between input records. + ## Please note that the separator has to be exactly one character long + const record_separator = "\n" &redef; +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6a84053bce..dd294ace7c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -424,6 +424,7 @@ set(bro_SRCS input/ReaderBackend.cc input/ReaderFrontend.cc input/readers/Ascii.cc + input/readers/Raw.cc ${dns_SRCS} diff --git a/src/input.bif b/src/input.bif index 2e9324ec56..5418b7bbd4 100644 --- a/src/input.bif +++ b/src/input.bif @@ -62,3 +62,5 @@ const set_separator: string; const empty_field: string; const unset_field: string; +module InputRaw; +const record_separator: string; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a7afdc3a78..d3009aa619 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -6,6 +6,7 @@ #include "ReaderFrontend.h" #include "ReaderBackend.h" #include "readers/Ascii.h" +#include "readers/Raw.h" #include "Event.h" #include "EventHandler.h" @@ -143,6 +144,7 @@ struct ReaderDefinition { ReaderDefinition input_readers[] = { { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, + { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, // End marker { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 73b8500d5e..733cca6352 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -133,6 +133,7 @@ bool Ascii::DoStartReading() { bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { if ( HasFilter(id) ) { + Error("Filter was added twice, ignoring."); return false; // no, we don't want to add this a second time } @@ -147,6 +148,7 @@ bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields bool Ascii::DoRemoveFilter ( int id ) { if (!HasFilter(id) ) { + Error("Filter removal of nonexisting filter requested."); return false; } @@ -263,11 +265,11 @@ TransportProto Ascii::StringToProto(const string &proto) { Value* Ascii::EntryToVal(string s, FieldMapping field) { - Value* val = new Value(field.type, true); - if ( s.compare(unset_field) == 0 ) { // field is not set... return new Value(field.type, false); } + + Value* val = new Value(field.type, true); switch ( field.type ) { case TYPE_ENUM: diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc new file mode 100644 index 0000000000..c435624865 --- /dev/null +++ b/src/input/readers/Raw.cc @@ -0,0 +1,230 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Raw.h" +#include "NetVar.h" + +#include +#include + +#include "../../threading/SerializationTypes.h" + +#define MANUAL 0 +#define REREAD 1 +#define STREAM 2 + +#include +#include +#include + +using namespace input::reader; +using threading::Value; +using threading::Field; + +Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) +{ + file = 0; + + //keyMap = new map(); + + separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len()); + if ( separator.size() != 1 ) { + Error("separator length has to be 1. Separator will be truncated."); + } + +} + +Raw::~Raw() +{ + DoFinish(); +} + +void Raw::DoFinish() +{ + filters.empty(); + if ( file != 0 ) { + file->close(); + delete(file); + file = 0; + } +} + +bool Raw::DoInit(string path, int arg_mode) +{ + started = false; + fname = path; + mode = arg_mode; + mtime = 0; + + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + + return true; +} + +bool Raw::DoStartReading() { + if ( started == true ) { + Error("Started twice"); + return false; + } + + started = true; + switch ( mode ) { + case MANUAL: + case REREAD: + case STREAM: + DoUpdate(); + break; + default: + assert(false); + } + + return true; +} + +bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { + + if ( arg_num_fields != 1 ) { + Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); + return false; + } + + if ( fields[0]->type != TYPE_STRING ) { + Error("Filter for raw reader contains a field that is not of type string."); + return false; + } + + if ( HasFilter(id) ) { + Error("Filter was added twice, ignoring"); + return false; // no, we don't want to add this a second time + } + + Filter f; + f.num_fields = arg_num_fields; + f.fields = fields; + + filters[id] = f; + + return true; +} + +bool Raw::DoRemoveFilter ( int id ) { + if (!HasFilter(id) ) { + Error("Filter removal of nonexisting filter requested."); + return false; + } + + assert ( filters.erase(id) == 1 ); + + return true; +} + + +bool Raw::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } + return true; +} + +bool Raw::GetLine(string& str) { + while ( getline(*file, str, separator[0]) ) { + return true; + } + + return false; +} + + +// read the entire file and send appropriate thingies back to InputMgr +bool Raw::DoUpdate() { + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } + + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } + + mtime = sb.st_mtime; + // file changed. reread. + + // fallthrough + case MANUAL: + case STREAM: + + if ( file && file->is_open() ) { + if ( mode == STREAM ) { + file->clear(); // remove end of file evil bits + break; + } + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + + break; + default: + assert(false); + + } + + string line; + while ( GetLine(line) ) { + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + + assert ((*it).second.num_fields == 1); + + Value** fields = new Value*[1]; + + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val = new string(line); + fields[0] = val; + + Put((*it).first, fields); + + } + + } + + return true; +} + + +bool Raw::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( mode ) { + case MANUAL: + // yay, we do nothing :) + break; + case REREAD: + case STREAM: + Update(); // call update and not DoUpdate, because update actually checks disabled. + break; + default: + assert(false); + } + + return true; +} + diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h new file mode 100644 index 0000000000..e046cb2ff7 --- /dev/null +++ b/src/input/readers/Raw.h @@ -0,0 +1,70 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_RAW_H +#define INPUT_READERS_RAW_H + +#include +#include + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +class Raw : public ReaderBackend { +public: + Raw(ReaderFrontend* frontend); + ~Raw(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Raw(frontend); } + +protected: + + virtual bool DoInit(string path, int mode); + + virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + + virtual bool DoRemoveFilter ( int id ); + + virtual void DoFinish(); + + virtual bool DoUpdate(); + + virtual bool DoStartReading(); + +private: + + virtual bool DoHeartbeat(double network_time, double current_time); + + struct Filter { + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping + }; + + bool HasFilter(int id); + + bool GetLine(string& str); + + ifstream* file; + string fname; + + map filters; + + // Options set from the script-level. + string separator; + + // keep a copy of the headerline to determine field locations when filters change + string headerline; + + int mode; + + bool started; + time_t mtime; + +}; + + +} +} + +#endif /* INPUT_READERS_RAW_H */ diff --git a/src/types.bif b/src/types.bif index e2a47a7ece..a9c6ecb3a8 100644 --- a/src/types.bif +++ b/src/types.bif @@ -173,6 +173,7 @@ module Input; enum Reader %{ READER_DEFAULT, READER_ASCII, + READER_RAW, %} enum Event %{ diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out new file mode 100644 index 0000000000..2059013c5d --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -0,0 +1,8 @@ +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro new file mode 100644 index 0000000000..5f196648b6 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -0,0 +1,35 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Val: record { + s: string; +}; + +event line(tpe: Input::Event, s: string) { + print s; +} + +event bro_init() +{ + Input::create_stream(A::INPUT, [$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM]); + Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); +} From 93fac7a4be74d441001ec3ddb70e13655bee636c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 22 Feb 2012 10:46:35 -0800 Subject: [PATCH 070/123] fix one of the bugs seth found in the input framework. (bug in PutTable when the table contained only one element and that element should not be wrapped into a record) --- src/input/Manager.cc | 6 +- .../out | 7 + .../scripts.base.frameworks.input.reread/out | 267 ++++++++++++++++++ .../frameworks/input/predicate-stream.bro | 83 ++++++ .../scripts/base/frameworks/input/reread.bro | 32 ++- 5 files changed, 388 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicate-stream.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d3009aa619..9a350bef20 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1083,12 +1083,11 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { valval = 0; - } else if ( filter->num_val_fields == 1 && !filter->want_record ) { - valval = ValueToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); + } else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) { + valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); } else { valval = ValueToRecordVal(vals, filter->rtype, &position); } @@ -1130,6 +1129,7 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * if ( filter->num_val_fields > 0 ) vl.append(valval); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate-stream/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 9516cb2a92..b844990978 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -110,6 +110,273 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] } done diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro new file mode 100644 index 0000000000..f08aaef998 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -0,0 +1,83 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out +# +# only difference from predicate.bro is, that this one uses a stream source. +# the reason is, that the code-paths are quite different, because then the ascii reader uses the put and not the sendevent interface + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); +global ct: int; + +event line(tpe: Input::Event, left: Idx, right: bool) { + ct = ct + 1; + if ( ct < 3 ) { + return; + } + if ( ct > 3 ) { + print "Too many events"; + return; + } + + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} + +event bro_init() +{ + ct = 0; + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="input.log", $mode=Input::STREAM]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); +} + diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 58df37af84..8e573494fd 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,11 +1,15 @@ # # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro %INPUT -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log -# @TEST-EXEC: sleep 3 +# @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input3.log input.log -# @TEST-EXEC: btest-bg-wait -k 5 +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 # @TEST-EXEC: btest-diff out @TEST-START-FILE input1.log @@ -31,6 +35,26 @@ T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} @TEST-END-FILE +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -45 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -46 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -47 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE @load frameworks/communication/listen @@ -92,7 +116,7 @@ event Input::update_finished(id: Input::ID) { print outfile, servers; try = try + 1; - if ( try == 3 ) { + if ( try == 5 ) { print outfile, "done"; close(outfile); Input::remove_tablefilter(A::INPUT, "ssh"); From d81607c3e93452595e3453cbfe9c378fb71a40fb Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 23 Feb 2012 14:36:04 -0800 Subject: [PATCH 071/123] fix empty field bug in threaded version --- src/input/Manager.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 9a350bef20..9dba56b174 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1547,7 +1547,8 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; - length += GetValueLength(val); + if ( val->present ) + length += GetValueLength(val); } //reporter->Error("Length: %d", length); @@ -1560,7 +1561,8 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; - position += CopyValue(data, position, val); + if ( val->present ) + position += CopyValue(data, position, val); } hash_t key = HashKey::HashBytes(data, length); From d553a3c6f69c1110b37196632e47cb6ffd10e406 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 23 Feb 2012 15:30:39 -0800 Subject: [PATCH 072/123] fix strange bug when using predicates and events at the same time on a tablefilter. Testcase is now more involved. --- src/input/Manager.cc | 52 +++-- .../scripts.base.frameworks.input.reread/out | 212 +++++++++++++++++- .../scripts/base/frameworks/input/reread.bro | 10 +- 3 files changed, 249 insertions(+), 25 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 9dba56b174..b709d26ea8 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -364,20 +364,21 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } - Val* name = fval->Lookup(rtype->FieldOffset("name")); - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); - RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { - val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); } - TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); - Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val ? event_val->AsFunc() : 0; + Unref(event_val); if ( event ) { FuncType* etype = event->FType()->AsFuncType(); @@ -450,14 +451,17 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; - filter->tab = dst->Ref()->AsTableVal(); - filter->rtype = val ? val->Ref()->AsRecordType() : 0; - filter->itype = idx->Ref()->AsRecordType(); + filter->tab = dst->AsTableVal(); + filter->rtype = val ? val->AsRecordType() : 0; + filter->itype = idx->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); filter->lastDict = new PDict(InputHash); filter->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + Unref(name); + Unref(pred); if ( valfields > 1 ) { assert(filter->want_record); @@ -861,7 +865,7 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } } - } + } Val* oldval = 0; @@ -948,16 +952,16 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { val = filter->tab->Lookup(idx); assert(val != 0); } + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + if ( filter->pred ) { - - bool doBreak = false; // ask predicate, if we want to expire this element... - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - //Ref(idx); - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); + Ref(ev); + Ref(predidx); Ref(val); val_list vl(3); @@ -971,21 +975,23 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... + Unref(predidx); + Unref(ev); filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); continue; - } - - - } + } + } if ( filter->event ) { - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); + Ref(predidx); Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(ev); SendEvent(filter->event, 3, ev, predidx, val); } + Unref(predidx); + Unref(ev); + filter->tab->Delete(ih->idxkey); filter->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index b844990978..f244f11a73 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -1,3 +1,18 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW [i=-42] @@ -28,6 +43,21 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW [i=-43] @@ -70,6 +100,21 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_CHANGED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_CHANGED [i=-43] @@ -112,6 +157,21 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW [i=-44] @@ -126,6 +186,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW @@ -142,7 +217,7 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -============EVENT============ +============PREDICATE============ Input::EVENT_NEW [i=-46] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ @@ -159,6 +234,21 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW [i=-47] [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -171,6 +261,36 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Input::EVENT_NEW +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_NEW @@ -274,6 +394,96 @@ BB }, vc=[10, 20, 30], ve=[]] } +============PREDICATE============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] ============EVENT============ Input::EVENT_REMOVED [i=-43] diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 8e573494fd..742d68605b 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -107,7 +107,15 @@ event bro_init() try = 0; # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); } From faf5c95752c2cb32f269b624262676902a22426f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 11 Mar 2012 19:41:41 -0700 Subject: [PATCH 073/123] a couple of small fixes ( default values, all null lines) --- src/input/Manager.cc | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a647b3c945..27580e0e82 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -104,7 +104,6 @@ struct Manager::ReaderInfo { EnumVal* type; ReaderFrontend* reader; - //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions bool HasFilter(int id); @@ -160,7 +159,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - reporter->Error("unknown reader when creating reader"); + reporter->Error("The reader that was requested was not found and could not be initialized."); return 0; } @@ -181,7 +180,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } else { // ohok. init failed, kill factory for all eternity ir->factory = 0; - DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); return 0; } @@ -225,8 +224,11 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) assert(reader_obj); // get the source... - const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); + assert ( sourceval != 0 ); + const BroString* bsource = sourceval->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); + Unref(sourceval); ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; @@ -255,13 +257,14 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { return false; } - Val* name = fval->Lookup(rtype->FieldOffset("name")); - RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); - Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); Func* event = event_val->AsFunc(); + Unref(event_val); { FuncType* etype = event->FType()->AsFuncType(); @@ -330,8 +333,10 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { logf[i] = fieldsV[i]; } + Unref(fields); // ref'd by lookupwithdefault EventFilter* filter = new EventFilter(); filter->name = name->AsString()->CheckString(); + Unref(name); // ref'd by lookupwithdefault filter->id = id->Ref()->AsEnumVal(); filter->num_fields = fieldsV.size(); filter->fields = fields->Ref()->AsRecordType(); @@ -369,8 +374,9 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; - if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) { val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + Unref(val); // The lookupwithdefault in the if-clause ref'ed val. } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); @@ -780,10 +786,12 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va assert(i->filters[id]->filter_type == TABLE_FILTER); TableFilter* filter = (TableFilter*) i->filters[id]; - //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - //reporter->Error("Result: %d\n", (uint64_t) idxhash->Hash()); - //reporter->Error("Hashing %d val fields", i->num_val_fields); + + if ( idxhash == 0 ) { + reporter->Error("Could not hash line. Ignoring"); + return filter->num_val_fields + filter->num_idx_fields; + } hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { @@ -792,10 +800,6 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va delete(valhashkey); } - //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); - - //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before @@ -1609,14 +1613,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { length += GetValueLength(val); } - //reporter->Error("Length: %d", length); + if ( length == 0 ) { + reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); + return NULL; + } int position = 0; char *data = (char*) malloc(length); if ( data == 0 ) { reporter->InternalError("Could not malloc?"); } - //memset(data, 0, length); for ( int i = 0; i < num_elements; i++ ) { const Value* val = vals[i]; if ( val->present ) From 92555badd4d6bca6a3f9ed96c3dab1087aac9940 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 11 Mar 2012 20:43:26 -0700 Subject: [PATCH 074/123] cleanup, more sanity tests, a little bit more documentation --- doc/input.rst | 27 +++-- scripts/base/frameworks/input/main.bro | 3 - src/input/Manager.cc | 159 ++++++++++++++----------- src/input/Manager.h | 3 + 4 files changed, 110 insertions(+), 82 deletions(-) diff --git a/doc/input.rst b/doc/input.rst index 78e96fe06e..e201af9fed 100644 --- a/doc/input.rst +++ b/doc/input.rst @@ -34,9 +34,11 @@ very similar to the abstracts used in the logging framework: Readers A reader defines the input format for the specific input stream. - At the moment, Bro comes with only one type of reader, which can - read the tab seperated ASCII logfiles that were generated by the + At the moment, Bro comes with two types of reader. The default reader is READER_ASCII, + which can read the tab seperated ASCII logfiles that were generated by the logging framework. + READER_RAW can files containing records separated by a character(like e.g. newline) and send + one event per line. Basics @@ -68,7 +70,21 @@ The fields that can be set when creating a stream are: ``reader`` The reader used for this stream. Default is ``READER_ASCII``. - + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. + + ``autostart`` + If set to yes, the first update operation is triggered automatically after the first filter has been added to the stream. + This has to be set to false if several filters are added to the input source. + In this case Input::force_update has to be called manually once after all filters have been added. Filters ======= @@ -101,9 +117,6 @@ could be defined as follows: ... Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); - - # read the file after all filters have been set - Input::force_update(Foo::INPUT); } The fields that can be set for an event filter are: @@ -156,7 +169,7 @@ an approach similar to this: Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); - # read the file after all filters have been set + # read the file after all filters have been set (only needed if autostart is set to false) Input::force_update(Foo::INPUT); } diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 445f947106..c6995121bd 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -182,9 +182,6 @@ function read_table(description: Input::StreamDescription, filter: Input::TableF if ( ok ) { ok = add_tablefilter(id, filter); } - if ( ok ) { - ok = force_update(id); - } if ( ok ) { ok = remove_stream(id); } else { diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 27580e0e82..db98cb7a33 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -327,7 +327,6 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { return false; } - Field** logf = new Field*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { logf[i] = fieldsV[i]; @@ -380,6 +379,30 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); + // check if index fields match tabla description + { + int num = idx->NumFields(); + const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); + + loop_over_list(*tl, j) + { + if ( j >= num ) { + reporter->Error("Table type has more indexes than index definition"); + return false; + } + + if ( !same_type(idx->FieldType(j), (*tl)[j]) ) { + reporter->Error("Table type does not match index type"); + return false; + } + } + + if ( num != j ) { + reporter->Error("Table has less elements than index definition"); + return false; + } + } + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); Val* event_val = fval->LookupWithDefault(rtype->FieldOffset("ev")); @@ -571,7 +594,6 @@ bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { reporter->Error("Stream not found in RemoveStreamContinuation"); return false; - } bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { @@ -738,7 +760,6 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu idxval = l; } - //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; @@ -771,8 +792,6 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals delete vals[i]; } delete [] vals; - - } int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { @@ -846,17 +865,15 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } + + bool result; + if ( filter->num_val_fields > 0 ) { // we have values + result = CallPred(filter->pred, 3, ev, predidx, valval); + } else { + // no values + result = CallPred(filter->pred, 2, ev, predidx); + } - val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. - vl.append(ev); - vl.append(predidx); - if ( filter->num_val_fields > 0 ) - vl.append(valval); - - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... @@ -968,14 +985,8 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { Ref(predidx); Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(predidx); - vl.append(val); - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - + bool result = CallPred(filter->pred, 3, ev, predidx, val); + if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... @@ -1038,7 +1049,6 @@ void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { } else { assert(false); } - } int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { @@ -1132,18 +1142,15 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } + + bool result; + if ( filter->num_val_fields > 0 ) { // we have values + result = CallPred(filter->pred, 3, ev, predidx, valval); + } else { + // no values + result = CallPred(filter->pred, 2, ev, predidx); + } - val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. - vl.append(ev); - vl.append(predidx); - if ( filter->num_val_fields > 0 ) - vl.append(valval); - - - Val* v = filter->pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - if ( result == false ) { // do nothing Unref(idxval); @@ -1154,7 +1161,6 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } - filter->tab->Assign(idxval, valval); if ( filter->event ) { @@ -1176,13 +1182,9 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * SendEvent(filter->event, 3, ev, predidx, valval); } } - } - - - } else { // no predicates or other stuff @@ -1192,6 +1194,7 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * return filter->num_idx_fields + filter->num_val_fields; } +// Todo:: perhaps throw some kind of clear-event? void Manager::Clear(const ReaderFrontend* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -1207,6 +1210,7 @@ void Manager::Clear(const ReaderFrontend* reader, int id) { filter->tab->RemoveAll(); } +// put interface: delete old entry from table. bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -1235,13 +1239,7 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - val_list vl(3); - vl.append(ev); - vl.append(predidx); - vl.append(val); - Val* v = filter->pred->Call(&vl); - filterresult = v->AsBool(); - Unref(v); + filterresult = CallPred(filter->pred, 3, ev, predidx, val); if ( filterresult == false ) { // keep it. @@ -1285,6 +1283,26 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { return success; } +bool Manager::CallPred(Func* pred_func, const int numvals, ...) +{ + bool result; + val_list vl(numvals); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl.append( va_arg(lP, Val*) ); + } + va_end(lP); + + Val* v = pred_func->Call(&vl); + result = v->AsBool(); + Unref(v); + + return(result); +} + bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -1341,8 +1359,15 @@ void Manager::SendEvent(EventHandlerPtr ev, list events) mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } - +// Convert a bro list value to a bro record value. I / we could think about moving this functionality to val.cc RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + assert(position != 0 ); // we need the pointer to point to data; + + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("ListValToRecordVal called on non-record-value."); + return 0; + } + RecordVal* rec = new RecordVal(request_type->AsRecordType()); int maxpos = list->Length(); @@ -1364,20 +1389,14 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, return rec; } - - +// Convert a threading value to a record value RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { - if ( position == 0 ) { - reporter->InternalError("Need position"); - return 0; - } + assert(position != 0); // we need the pointer to point to data. - /* if ( request_type->Tag() != TYPE_RECORD ) { - reporter->InternalError("I only work with records"); + reporter->InternalError("ValueToRecordVal called on non-record-value."); return 0; - } */ - + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); for ( int i = 0; i < request_type->NumFields(); i++ ) { @@ -1394,11 +1413,12 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *reque } return rec; - } - +// Count the length of the values +// used to create a correct length buffer for hashing later int Manager::GetValueLength(const Value* val) { + assert( val->present ); // presence has to be checked elsewhere int length = 0; switch (val->type) { @@ -1485,19 +1505,20 @@ int Manager::GetValueLength(const Value* val) { } +// Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. +// Used for hashing the values for lookup in the bro table int Manager::CopyValue(char *data, const int startpos, const Value* val) { + assert( val->present ); // presence has to be checked elsewhere + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+startpos) = val->val.int_val; return sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: - //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; @@ -1516,7 +1537,6 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - //*(data+startpos) = val->val.double_val; memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); return sizeof(val->val.double_val); break; @@ -1598,12 +1618,11 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { return 0; } - reporter->InternalError("internal error"); assert(false); return 0; - } +// Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { int length = 0; @@ -1633,10 +1652,9 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { assert(position == length); return new HashKey(data, length, key, true); - - } +// convert threading value to Bro value Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1647,7 +1665,6 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { if ( !val->present ) { return 0; // unset field } - switch ( val->type ) { case TYPE_BOOL: @@ -1760,8 +1777,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { reporter->InternalError("unsupported type for input_read"); } - - reporter->InternalError("Impossible error"); + assert(false); return NULL; } @@ -1778,7 +1794,6 @@ Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) return 0; } - Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) diff --git a/src/input/Manager.h b/src/input/Manager.h index b4fc6cff7f..96ea0e43db 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -177,6 +177,9 @@ private: void SendEvent(EventHandlerPtr ev, const int numvals, ...); void SendEvent(EventHandlerPtr ev, list events); + // Call predicate function and return result + bool CallPred(Func* pred_func, const int numvals, ...); + // get a hashkey for a set of threading::Values HashKey* HashValues(const int num_elements, const threading::Value* const *vals); From b4e6971aab46054e68887836e1a1be40cfd4b9c5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 14 Mar 2012 14:45:53 -0700 Subject: [PATCH 075/123] Add regular debugging output for interesting operations (stream/filter operations) to input framework (this was way overdue) --- scripts/base/frameworks/input/main.bro | 25 +++++++- src/DebugLogger.cc | 3 +- src/DebugLogger.h | 1 + src/input/Manager.cc | 88 +++++++++++++++++++++++++- 4 files changed, 114 insertions(+), 3 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index c6995121bd..1df8563d94 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,7 +5,7 @@ module Input; export { - redef enum Input::ID += { TABLE_READ }; + redef enum Input::ID += { TABLE_READ, EVENT_READ }; ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -123,6 +123,8 @@ export { ## filter: the `TableFilter` record describing the filter. global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; + global read_event: function(description: Input::StreamDescription, filter: Input::EventFilter) : bool; + global update_finished: event(id: Input::ID); } @@ -182,6 +184,27 @@ function read_table(description: Input::StreamDescription, filter: Input::TableF if ( ok ) { ok = add_tablefilter(id, filter); } + if ( ok ) { + ok = remove_tablefilter(id, filter$name); + } + if ( ok ) { + ok = remove_stream(id); + } else { + remove_stream(id); + } + + return ok; +} + +function read_event(description: Input::StreamDescription, filter: Input::EventFilter) : bool { + local ok: bool = T; + # since we create and delete it ourselves this should be ok... at least for singlethreaded operation + local id: Input::ID = Input::EVENT_READ; + + ok = create_stream(id, description); + if ( ok ) { + ok = add_eventfilter(id, filter); + } if ( ok ) { ok = remove_stream(id); } else { diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index c41a0552c6..3394486ff2 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -15,7 +15,8 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, - { "logging", 0, false }, { "threading", 0, false } + { "logging", 0, false }, {"input", 0, false }, + { "threading", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 71e21bfa26..ca422072c5 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -24,6 +24,7 @@ enum DebugStream { DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli DBG_LOGGING, // Logging streams + DBG_INPUT, // Input streams DBG_THREADING, // Threading system NUM_DBGS // Has to be last diff --git a/src/input/Manager.cc b/src/input/Manager.cc index db98cb7a33..aa50453bdf 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -205,12 +205,24 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) // create a new input reader object to be used at whomevers leisure lateron. ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) { + { + ReaderInfo *i = FindReader(id); + if ( i != 0 ) { + ODesc desc; + id->Describe(&desc); + reporter->Error("Trying create already existing input stream %s", desc.Description()); + return 0; + } + } + ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) { - reporter->Error("Streamdescription argument not of right type"); + ODesc desc; + id->Describe(&desc); + reporter->Error("Streamdescription argument not of right type for new input stream %s", desc.Description()); return 0; } @@ -239,6 +251,13 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) reader_obj->Init(source, mode->InternalInt(), do_autostart); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", + desc.Description()); +#endif + return reader_obj; } @@ -503,6 +522,13 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { i->filters[filterid] = filter; i->reader->AddFilter( filterid, fieldsV.size(), fields ); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully created new table filter %s for stream", + filter->name.c_str(), desc.Description()); +#endif + return true; } @@ -574,6 +600,13 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s", + desc.Description()); +#endif + return true; } @@ -586,6 +619,12 @@ bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { if ( (*s)->reader && (*s)->reader == reader ) { i = *s; +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", + desc.Description()); +#endif delete(i); readers.erase(s); return true; @@ -651,6 +690,13 @@ bool Manager::ForceUpdate(const EnumVal* id) i->reader->Update(); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Forcing update of stream %s", + desc.Description()); +#endif + return true; // update is async :( } @@ -685,6 +731,13 @@ bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { i->reader->RemoveFilter(filterId); +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Queued removal of tablefilter %s for stream %s", + name.c_str(), desc.Description()); +#endif + return true; } @@ -701,6 +754,13 @@ bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int f return false; } +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Executed removal of (table|event)-filter %s for stream %s", + (*it).second->name.c_str(), desc.Description()); +#endif + delete (*it).second; i->filters.erase(it); @@ -736,6 +796,13 @@ bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { } i->reader->RemoveFilter(filterId); + +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Queued removal of eventfilter %s for stream %s", + name.c_str(), desc.Description()); +#endif return true; } @@ -948,6 +1015,13 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { assert(i->HasFilter(id)); +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Got EndCurrentSend for filter %d and stream %s", + id, desc.Description()); +#endif + if ( i->filters[id]->filter_type == EVENT_FILTER ) { // nothing to do.. return; @@ -1018,6 +1092,11 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for filter %d and stream %s, queueing update_finished event", + id, desc.Description()); +#endif + // Send event that the current update is indeed finished. EventHandler* handler = event_registry->Lookup("Input::update_finished"); if ( handler == 0 ) { @@ -1202,6 +1281,13 @@ void Manager::Clear(const ReaderFrontend* reader, int id) { return; } +#ifdef DEBUG + ODesc desc; + i->id->Describe(&desc); + DBG_LOG(DBG_INPUT, "Got Clear for filter %d and stream %s", + id, desc.Description()); +#endif + assert(i->HasFilter(id)); assert(i->filters[id]->filter_type == TABLE_FILTER); From 57ffe1be777f395de2cb94c95c02b9f234109744 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 15 Mar 2012 18:41:51 -0700 Subject: [PATCH 076/123] completely change interface again. compiles, not really tested. basic test works 70% of the time, coredumps in the other 30 - but was not easy to debug on a first glance (most interestingly the crash happens in the logging framework - I wonder how that works). Other tests are not adjusted to the new interface yet. --- scripts/base/frameworks/input/main.bro | 190 ++----- src/input.bif | 39 +- src/input/Manager.cc | 511 ++++++------------ src/input/Manager.h | 96 +--- src/input/ReaderBackend.cc | 115 +--- src/input/ReaderBackend.h | 80 +-- src/input/ReaderFrontend.cc | 83 +-- src/input/ReaderFrontend.h | 27 +- src/input/readers/Ascii.cc | 182 +++---- src/input/readers/Ascii.h | 24 +- src/input/readers/Raw.cc | 89 +-- src/input/readers/Raw.h | 23 +- src/types.bif | 4 - .../scripts/base/frameworks/input/basic.bro | 12 +- 14 files changed, 403 insertions(+), 1072 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 1df8563d94..4f7f9983d1 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,15 +5,15 @@ module Input; export { - redef enum Input::ID += { TABLE_READ, EVENT_READ }; - ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; const default_mode = MANUAL &redef; - ## Stream decription type used for the `create_stream` method - type StreamDescription: record { + ## TableFilter description type used for the `table` method. + type TableDescription: record { + ## Common definitions for tables and events + ## String that allows the reader to find the source. ## For `READER_ASCII`, this is the filename. source: string; @@ -26,13 +26,12 @@ export { ## Automatically start the input stream after the first filter has been added autostart: bool &default=T; - }; - ## TableFilter description type used for the `add_tablefilter` method. - type TableFilter: record { ## Descriptive name. Used to remove a filter at a later time name: string; + ## Special definitions for tables + ## Table which will contain the data read by the input framework destination: any; ## Record that defines the values used as the index of the table @@ -55,11 +54,28 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; - ## EventFilter description type used for the `add_eventfilter` method. - type EventFilter: record { - ## Descriptive name. Used to remove a filter at a later time - name: string; + ## EventFilter description type used for the `event` method. + type EventDescription: record { + ## Common definitions for tables and events + + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. + source: string; + + ## Reader to use for this steam + reader: Reader &default=default_reader; + ## Read mode to use for this stream + mode: Mode &default=default_mode; + + ## Automatically start the input stream after the first filter has been added + autostart: bool &default=T; + + ## Descriptive name. Used to remove a filter at a later time + name: string; + + ## Special definitions for events + ## Record describing the fields to be retrieved from the source input. fields: any; ## If want_record if false (default), the event receives each value in fields as a seperate argument. @@ -72,61 +88,29 @@ export { }; - #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - - ## Create a new input stream from a given source. Returns true on success. + ## Create a new table input from a given source. Returns true on success. ## - ## id: `Input::ID` enum value identifying this stream - ## description: `StreamDescription` record describing the source. - global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; - - ## Remove a current input stream. Returns true on success. + ## description: `TableDescription` record describing the source. + global add_table: function(description: Input::TableDescription) : bool; + + ## Create a new event input from a given source. Returns true on success. ## - ## id: `Input::ID` enum value identifying the stream to be removed - global remove_stream: function(id: Input::ID) : bool; + ## description: `TableDescription` record describing the source. + global add_event: function(description: Input::EventDescription) : bool; + + ## Remove a input stream. Returns true on success and false if the named stream was not found. + ## + ## id: string value identifying the stream to be removed + global remove: function(id: string) : bool; ## Forces the current input to be checked for changes. + ## Returns true on success and false if the named stream was not found ## - ## id: `Input::ID` enum value identifying the stream - global force_update: function(id: Input::ID) : bool; - - ## Adds a table filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## filter: the `TableFilter` record describing the filter. - global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; - - ## Removes a named table filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## name: the name of the filter to be removed. - global remove_tablefilter: function(id: Input::ID, name: string) : bool; - - ## Adds an event filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## filter: the `EventFilter` record describing the filter. - global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; - - ## Removes a named event filter to a specific input stream. Returns true on success. - ## - ## id: `Input::ID` enum value identifying the stream - ## name: the name of the filter to be removed. - global remove_eventfilter: function(id: Input::ID, name: string) : bool; - #global get_filter: function(id: ID, name: string) : Filter; - - ## Convenience function for reading a specific input source exactly once using - ## exactly one tablefilter - ## - ## id: `Input::ID` enum value identifying the stream - ## description: `StreamDescription` record describing the source. - ## filter: the `TableFilter` record describing the filter. - global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; - - global read_event: function(description: Input::StreamDescription, filter: Input::EventFilter) : bool; - - global update_finished: event(id: Input::ID); + ## id: string value identifying the stream + global force_update: function(id: string) : bool; + ## Event that is called, when the update of a specific source is finished + global update_finished: event(id: string); } @load base/input.bif @@ -134,90 +118,26 @@ export { module Input; -#global filters: table[ID, string] of Filter; +#global streams: table[string] of Filter; +# ^ change to set containing the names -function create_stream(id: Input::ID, description: Input::StreamDescription) : bool +function add_table(description: Input::TableDescription) : bool { - return __create_stream(id, description); + return __create_table_stream(description); } -function remove_stream(id: Input::ID) : bool +function add_event(description: Input::EventDescription) : bool + { + return __create_event_stream(description); + } + +function remove(id: string) : bool { return __remove_stream(id); } -function force_update(id: Input::ID) : bool +function force_update(id: string) : bool { return __force_update(id); } -function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool - { -# filters[id, filter$name] = filter; - return __add_tablefilter(id, filter); - } - -function remove_tablefilter(id: Input::ID, name: string) : bool - { -# delete filters[id, name]; - return __remove_tablefilter(id, name); - } - -function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool - { -# filters[id, filter$name] = filter; - return __add_eventfilter(id, filter); - } - -function remove_eventfilter(id: Input::ID, name: string) : bool - { -# delete filters[id, name]; - return __remove_eventfilter(id, name); - } - -function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { - local ok: bool = T; - # since we create and delete it ourselves this should be ok... at least for singlethreaded operation - local id: Input::ID = Input::TABLE_READ; - - ok = create_stream(id, description); - if ( ok ) { - ok = add_tablefilter(id, filter); - } - if ( ok ) { - ok = remove_tablefilter(id, filter$name); - } - if ( ok ) { - ok = remove_stream(id); - } else { - remove_stream(id); - } - - return ok; -} - -function read_event(description: Input::StreamDescription, filter: Input::EventFilter) : bool { - local ok: bool = T; - # since we create and delete it ourselves this should be ok... at least for singlethreaded operation - local id: Input::ID = Input::EVENT_READ; - - ok = create_stream(id, description); - if ( ok ) { - ok = add_eventfilter(id, filter); - } - if ( ok ) { - ok = remove_stream(id); - } else { - remove_stream(id); - } - - return ok; -} - -#function get_filter(id: ID, name: string) : Filter -# { -# if ( [id, name] in filters ) -# return filters[id, name]; -# -# return no_filter; -# } diff --git a/src/input.bif b/src/input.bif index 5418b7bbd4..1157b7b62b 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,52 +7,33 @@ module Input; #include "NetVar.h" %%} -type StreamDescription: record; -type TableFilter: record; -type EventFilter: record; +type TableDescription: record; +type EventDescription: record; -function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool +function Input::__create_table_stream%(description: Input::TableDescription%) : bool %{ - input::ReaderFrontend *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); - return new Val( the_reader != 0, TYPE_BOOL ); - %} - -function Input::__remove_stream%(id: Input::ID%) : bool - %{ - bool res = input_mgr->RemoveStream(id->AsEnumVal()); + bool res = input_mgr->CreateTableStream(description->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Input::ID%) : bool +function Input::__create_event_stream%(description: Input::EventDescription%) : bool %{ - bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + bool res = input_mgr->CreateEventStream(description->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool +function Input::__remove_stream%(id: string%) : bool %{ - bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->RemoveStream(id->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool +function Input::__force_update%(id: string%) : bool %{ - bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL); - %} - -function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool - %{ - bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->ForceUpdate(id->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL); - %} - # Options for Ascii Reader module InputAscii; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index aa50453bdf..44b5bf44db 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -31,14 +31,25 @@ declare(PDict, InputHash); class Manager::Filter { public: - EnumVal* id; string name; + string source; + + int mode; FilterType filter_type; // to distinguish between event and table filters + EnumVal* type; + ReaderFrontend* reader; + virtual ~Filter(); }; +Manager::Filter::~Filter() { + Unref(type); + + delete(reader); +} + class Manager::TableFilter: public Manager::Filter { public: @@ -85,10 +96,6 @@ Manager::EventFilter::EventFilter() { filter_type = EVENT_FILTER; } -Manager::Filter::~Filter() { - Unref(id); -} - Manager::TableFilter::~TableFilter() { Unref(tab); Unref(itype); @@ -99,41 +106,6 @@ Manager::TableFilter::~TableFilter() { delete lastDict; } -struct Manager::ReaderInfo { - EnumVal* id; - EnumVal* type; - ReaderFrontend* reader; - - map filters; // filters that can prevent our actions - - bool HasFilter(int id); - - ~ReaderInfo(); - }; - -Manager::ReaderInfo::~ReaderInfo() { - map::iterator it = filters.begin(); - - while ( it != filters.end() ) { - delete (*it).second; - ++it; - } - - Unref(type); - Unref(id); - - delete(reader); -} - -bool Manager::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} - - struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -203,37 +175,34 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } // create a new input reader object to be used at whomevers leisure lateron. -ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) +bool Manager::CreateStream(Filter* info, RecordVal* description) { - { - ReaderInfo *i = FindReader(id); - if ( i != 0 ) { - ODesc desc; - id->Describe(&desc); - reporter->Error("Trying create already existing input stream %s", desc.Description()); - return 0; - } - } - ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) { - ODesc desc; - id->Describe(&desc); - reporter->Error("Streamdescription argument not of right type for new input stream %s", desc.Description()); - return 0; + reporter->Error("Streamdescription argument not of right type for new input stream"); + return false; + } + + Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); + string name = name_val->AsString()->CheckString(); + Unref(name_val); + + { + Filter *i = FindFilter(name); + if ( i != 0 ) { + reporter->Error("Trying create already existing input stream %s", name.c_str()); + return false; + } } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); - bool do_autostart = ( autostart->InternalInt() == 1 ); - Unref(autostart); // Ref'd by LookupWithDefault - - ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); - assert(reader_obj); + + ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); + assert(reader_obj); // get the source... Val* sourceval = description->LookupWithDefault(rtype->FieldOffset("source")); @@ -241,42 +210,45 @@ ReaderFrontend* Manager::CreateStream(EnumVal* id, RecordVal* description) const BroString* bsource = sourceval->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); Unref(sourceval); + + EnumVal* mode = description->LookupWithDefault(rtype->FieldOffset("mode"))->AsEnumVal(); + info->mode = mode->InternalInt(); + Unref(mode); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->id = id->Ref()->AsEnumVal(); + info->name = name; + info->source = source; - readers.push_back(info); - - reader_obj->Init(source, mode->InternalInt(), do_autostart); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", - desc.Description()); + name.c_str()); #endif - return reader_obj; + return true; } -bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Stream not found"); - return false; - } +bool Manager::CreateEventStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) { reporter->Error("filter argument not of right type"); return false; } + + EventFilter* filter = new EventFilter(); + { + bool res = CreateStream(filter, fval); + if ( res == false ) { + delete filter; + return false; + } + } + - Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -352,42 +324,36 @@ bool Manager::AddEventFilter(EnumVal *id, RecordVal* fval) { } Unref(fields); // ref'd by lookupwithdefault - EventFilter* filter = new EventFilter(); - filter->name = name->AsString()->CheckString(); - Unref(name); // ref'd by lookupwithdefault - filter->id = id->Ref()->AsEnumVal(); filter->num_fields = fieldsV.size(); filter->fields = fields->Ref()->AsRecordType(); filter->event = event_registry->Lookup(event->GetID()->Name()); filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - int filterid = 0; - if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> - } - i->filters[filterid] = filter; - i->reader->AddFilter( filterid, fieldsV.size(), logf ); + assert(filter->reader); + filter->reader->Init(filter->source, filter->mode, filter->num_fields, logf ); + readers[filter->reader] = filter; return true; } -bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Stream not found"); - return false; - } - +bool Manager::CreateTableStream(RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) { reporter->Error("filter argument not of right type"); return false; } + TableFilter* filter = new TableFilter(); + { + bool res = CreateStream(filter, fval); + if ( res == false ) { + delete filter; + return false; + } + } - Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); @@ -493,9 +459,6 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - TableFilter* filter = new TableFilter(); - filter->name = name->AsString()->CheckString(); - filter->id = id->Ref()->AsEnumVal(); filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; @@ -508,25 +471,22 @@ bool Manager::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault - Unref(name); Unref(pred); if ( valfields > 1 ) { assert(filter->want_record); } + + + assert(filter->reader); + filter->reader->Init(filter->source, filter->mode, fieldsV.size(), fields ); + + readers[filter->reader] = filter; - int filterid = 0; - if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> - } - i->filters[filterid] = filter; - i->reader->AddFilter( filterid, fieldsV.size(), fields ); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Successfully created new table filter %s for stream", - filter->name.c_str(), desc.Description()); + DBG_LOG(DBG_INPUT, "Successfully created table stream %s", + filter->name.c_str()); #endif return true; @@ -583,16 +543,8 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) } -bool Manager::RemoveStream(const EnumVal* id) { - ReaderInfo *i = 0; - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->id == id ) - { - i = (*s); - break; - } - } +bool Manager::RemoveStream(const string &name) { + Filter *i = FindFilter(name); if ( i == 0 ) { return false; // not found @@ -601,38 +553,29 @@ bool Manager::RemoveStream(const EnumVal* id) { i->reader->Finish(); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s", - desc.Description()); + name.c_str()); #endif return true; } -bool Manager::RemoveStreamContinuation(const ReaderFrontend* reader) { - ReaderInfo *i = 0; +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); - - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->reader && (*s)->reader == reader ) - { - i = *s; -#ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", - desc.Description()); -#endif - delete(i); - readers.erase(s); - return true; - } + if ( i == 0 ) { + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; } - - reporter->Error("Stream not found in RemoveStreamContinuation"); - return false; + + +#ifdef DEBUG + DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", + i->name.c_str()); +#endif + readers.erase(reader); + delete(i); + return true; } bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { @@ -680,132 +623,24 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co return true; } -bool Manager::ForceUpdate(const EnumVal* id) +bool Manager::ForceUpdate(const string &name) { - ReaderInfo *i = FindReader(id); + Filter *i = FindFilter(name); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->Error("Stream %s not found", name.c_str()); return false; } i->reader->Update(); #ifdef DEBUG - ODesc desc; - id->Describe(&desc); DBG_LOG(DBG_INPUT, "Forcing update of stream %s", - desc.Description()); + name.c_str()); #endif return true; // update is async :( } -bool Manager::RemoveTableFilter(EnumVal* id, const string &name) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - bool found = false; - int filterId; - - for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { - if ( (*it).second->name == name ) { - found = true; - filterId = (*it).first; - - if ( (*it).second->filter_type != TABLE_FILTER ) { - reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); - return false; - } - - break; - } - } - - if ( !found ) { - reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); - return false; - } - - i->reader->RemoveFilter(filterId); - -#ifdef DEBUG - ODesc desc; - id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Queued removal of tablefilter %s for stream %s", - name.c_str(), desc.Description()); -#endif - - return true; -} - -bool Manager::RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId) { - ReaderInfo *i = FindReader(reader); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - map::iterator it = i->filters.find(filterId); - if ( it == i->filters.end() ) { - reporter->Error("Got RemoveFilterContinuation where filter nonexistant for %d", filterId); - return false; - } - -#ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Executed removal of (table|event)-filter %s for stream %s", - (*it).second->name.c_str(), desc.Description()); -#endif - - delete (*it).second; - i->filters.erase(it); - - return true; -} - -bool Manager::RemoveEventFilter(EnumVal* id, const string &name) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - bool found = false; - int filterId; - for ( map::iterator it = i->filters.begin(); it != i->filters.end(); ++it ) { - if ( (*it).second->name == name ) { - found = true; - filterId = (*it).first; - - if ( (*it).second->filter_type != EVENT_FILTER ) { - reporter->Error("Trying to remove filter %s of wrong type", name.c_str()); - return false; - } - - break; - } - } - - if ( !found ) { - reporter->Error("Trying to remove nonexisting filter %s", name.c_str()); - return false; - } - - i->reader->RemoveFilter(filterId); - -#ifdef DEBUG - ODesc desc; - id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Queued removal of eventfilter %s for stream %s", - name.c_str(), desc.Description()); -#endif - return true; -} - Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; @@ -833,24 +668,19 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu } -void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); - return; - } - - if ( !i->HasFilter(id) ) { - reporter->InternalError("Unknown filter"); + reporter->InternalError("Unknown reader in SendEntry"); return; } int readFields; - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - readFields = SendEntryTable(reader, id, vals); - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) { + readFields = SendEntryTable(i, vals); + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(i, type, vals); } else { assert(false); } @@ -861,16 +691,13 @@ void Manager::SendEntry(const ReaderFrontend* reader, const int id, Value* *vals delete [] vals; } -int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool updated = false; assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; HashKey* idxhash = HashValues(filter->num_idx_fields, vals); @@ -1006,29 +833,25 @@ int Manager::SendEntryTable(const ReaderFrontend* reader, const int id, const Va } -void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { - ReaderInfo *i = FindReader(reader); +void Manager::EndCurrentSend(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in EndCurrentSend"); return; } - assert(i->HasFilter(id)); - #ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Got EndCurrentSend for filter %d and stream %s", - id, desc.Description()); + DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", + i->name.c_str()); #endif - if ( i->filters[id]->filter_type == EVENT_FILTER ) { + if ( i->filter_type == EVENT_FILTER ) { // nothing to do.. return; } - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = filter->lastDict->InitForIteration(); @@ -1093,8 +916,8 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { filter->currDict = new PDict(InputHash); #ifdef DEBUG - DBG_LOG(DBG_INPUT, "EndCurrentSend complete for filter %d and stream %s, queueing update_finished event", - id, desc.Description()); + DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", + i->name.c_str()); #endif // Send event that the current update is indeed finished. @@ -1104,42 +927,40 @@ void Manager::EndCurrentSend(const ReaderFrontend* reader, int id) { } - Ref(i->id); - SendEvent(handler, 1, i->id); + SendEvent(handler, 1, new BroString(i->name)); } -void Manager::Put(const ReaderFrontend* reader, int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +void Manager::Put(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Put"); return; } - if ( !i->HasFilter(id) ) { - reporter->InternalError("Unknown filter"); - return; - } - - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - PutTable(reader, id, vals); - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + int readFields; + if ( i->filter_type == TABLE_FILTER ) { + readFields = PutTable(i, vals); + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - SendEventFilterEvent(reader, type, id, vals); + readFields = SendEventFilterEvent(i, type, vals); } else { assert(false); } + + for ( int i = 0; i < readFields; i++ ) { + delete vals[i]; + } + delete [] vals; + } -int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { bool updated = false; assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == EVENT_FILTER); - EventFilter* filter = (EventFilter*) i->filters[id]; + assert(i->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i; Val *val; list out_vals; @@ -1170,14 +991,11 @@ int Manager::SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, i } -int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const *vals) { - ReaderInfo *i = FindReader(reader); - +int Manager::PutTable(Filter* i, const Value* const *vals) { assert(i); - assert(i->HasFilter(id)); - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; @@ -1274,43 +1092,37 @@ int Manager::PutTable(const ReaderFrontend* reader, int id, const Value* const * } // Todo:: perhaps throw some kind of clear-event? -void Manager::Clear(const ReaderFrontend* reader, int id) { - ReaderInfo *i = FindReader(reader); +void Manager::Clear(ReaderFrontend* reader) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Clear"); return; } #ifdef DEBUG - ODesc desc; - i->id->Describe(&desc); - DBG_LOG(DBG_INPUT, "Got Clear for filter %d and stream %s", - id, desc.Description()); + DBG_LOG(DBG_INPUT, "Got Clear for stream %s", + i->name.c_str()); #endif - assert(i->HasFilter(id)); - - assert(i->filters[id]->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i->filters[id]; + assert(i->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i; filter->tab->RemoveAll(); } // put interface: delete old entry from table. -bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { - ReaderInfo *i = FindReader(reader); +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { + Filter *i = FindFilter(reader); if ( i == 0 ) { - reporter->InternalError("Unknown reader"); + reporter->InternalError("Unknown reader in Delete"); return false; } - assert(i->HasFilter(id)); - bool success = false; int readVals = 0; - if ( i->filters[id]->filter_type == TABLE_FILTER ) { - TableFilter* filter = (TableFilter*) i->filters[id]; + if ( i->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; @@ -1352,9 +1164,9 @@ bool Manager::Delete(const ReaderFrontend* reader, int id, Value* *vals) { reporter->Error("Internal error while deleting values from input table"); } } - } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - readVals = SendEventFilterEvent(reader, type, id, vals); + readVals = SendEventFilterEvent(i, type, vals); success = true; } else { assert(false); @@ -1867,29 +1679,24 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { return NULL; } -Manager::ReaderInfo* Manager::FindReader(const ReaderFrontend* reader) +Manager::Filter* Manager::FindFilter(const string &name) { - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader && (*s)->reader == reader ) + if ( (*s).second->name == name ) { - return *s; + return (*s).second; } } return 0; } -Manager::ReaderInfo* Manager::FindReader(const EnumVal* id) - { - for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) - { - if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) - { - return *s; - } - } - - return 0; +Manager::Filter* Manager::FindFilter(ReaderFrontend* reader) +{ + map::iterator s = readers.find(reader); + if ( s != readers.end() ) { + return s->second; } - + return 0; +} diff --git a/src/input/Manager.h b/src/input/Manager.h index 96ea0e43db..71169c4bc2 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -11,7 +11,7 @@ #include "../EventHandler.h" #include "../RemoteSerializer.h" -#include +#include namespace input { @@ -35,6 +35,9 @@ public: /** * Creates a new input stream. + * Add a filter to an input source, which will write the data from the data source into + * a Bro table. + * Add a filter to an input source, which sends events for read input data. * * @param id The enum value corresponding the input stream. * @@ -43,7 +46,9 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - ReaderFrontend* CreateStream(EnumVal* id, RecordVal* description); + bool CreateTableStream(RecordVal* description); + bool CreateEventStream(RecordVal* description); + /** * Force update on a input stream. @@ -57,7 +62,7 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool ForceUpdate(const EnumVal* id); + bool ForceUpdate(const string &id); /** * Deletes an existing input stream @@ -67,53 +72,8 @@ public: * This method corresponds directly to the internal BiF defined in * input.bif, which just forwards here. */ - bool RemoveStream(const EnumVal* id); + bool RemoveStream(const string &id); - /** - * Add a filter to an input source, which will write the data from the data source into - * a Bro table. - * - * @param id The enum value corresponding the input stream. - * - * @param description A record of script type \c Input:TableFilter. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool AddTableFilter(EnumVal *id, RecordVal* filter); - - /** - * Removes a tablefilter from the log stream - * - * @param id The enum value corresponding the input stream. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool RemoveTableFilter(EnumVal* id, const string &name); - - /** - * Add a filter to an input source, which sends events for read input data. - * - * @param id The enum value corresponding the input stream. - * - * @param description A record of script type \c Input:EventFilter. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool AddEventFilter(EnumVal *id, RecordVal* filter); - - /** - * Removes a eventfilter from the log stream - * - * @param id The enum value corresponding the input stream. - * - * This method corresponds directly to the internal BiF defined in - * input.bif, which just forwards here. - */ - bool RemoveEventFilter(EnumVal* id, const string &name); - protected: friend class ReaderFrontend; friend class PutMessage; @@ -122,19 +82,18 @@ protected: friend class SendEventMessage; friend class SendEntryMessage; friend class EndCurrentSendMessage; - friend class FilterRemovedMessage; friend class ReaderFinishedMessage; // For readers to write to input stream in direct mode (reporting new/deleted values directly) // Functions take ownership of threading::Value fields - void Put(const ReaderFrontend* reader, int id, threading::Value* *vals); - void Clear(const ReaderFrontend* reader, int id); - bool Delete(const ReaderFrontend* reader, int id, threading::Value* *vals); + void Put(ReaderFrontend* reader, threading::Value* *vals); + void Clear(ReaderFrontend* reader); + bool Delete(ReaderFrontend* reader, threading::Value* *vals); // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) // Functions take ownership of threading::Value fields - void SendEntry(const ReaderFrontend* reader, const int id, threading::Value* *vals); - void EndCurrentSend(const ReaderFrontend* reader, const int id); + void SendEntry(ReaderFrontend* reader, threading::Value* *vals); + void EndCurrentSend(ReaderFrontend* reader); // Allows readers to directly send Bro events. // The num_vals and vals must be the same the named event expects. @@ -150,20 +109,23 @@ protected: // Used to prevent race conditions where data for a specific filter is still in the queue when the // RemoveFilter directive is executed by the main thread. // This makes sure all data that has ben queued for a filter is still received. - bool RemoveFilterContinuation(const ReaderFrontend* reader, const int filterId); - bool RemoveStreamContinuation(const ReaderFrontend* reader); + bool RemoveStreamContinuation(ReaderFrontend* reader); private: - struct ReaderInfo; + class Filter; + class TableFilter; + class EventFilter; + + bool CreateStream(Filter*, RecordVal* description); // SendEntry implementation for Tablefilter - int SendEntryTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int SendEntryTable(Filter* i, const threading::Value* const *vals); // Put implementation for Tablefilter - int PutTable(const ReaderFrontend* reader, int id, const threading::Value* const *vals); + int PutTable(Filter* i, const threading::Value* const *vals); // SendEntry and Put implementation for Eventfilter - int SendEventFilterEvent(const ReaderFrontend* reader, EnumVal* type, int id, const threading::Value* const *vals); + int SendEventFilterEvent(Filter* i, EnumVal* type, const threading::Value* const *vals); // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types // from the log framework @@ -200,16 +162,12 @@ private: // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); - ReaderInfo* FindReader(const ReaderFrontend* reader); - ReaderInfo* FindReader(const EnumVal* id); - - vector readers; - - class Filter; - class TableFilter; - class EventFilter; + Filter* FindFilter(const string &name); + Filter* FindFilter(ReaderFrontend* reader); enum FilterType { TABLE_FILTER, EVENT_FILTER }; + + map readers; }; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 5af02b1acc..b33e19d297 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -11,48 +11,44 @@ namespace input { class PutMessage : public threading::OutputMessage { public: - PutMessage(ReaderFrontend* reader, int id, Value* *val) + PutMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("Put", reader), - id(id), val(val) {} + val(val) {} virtual bool Process() { - input_mgr->Put(Object(), id, val); + input_mgr->Put(Object(), val); return true; } private: - int id; Value* *val; }; class DeleteMessage : public threading::OutputMessage { public: - DeleteMessage(ReaderFrontend* reader, int id, Value* *val) + DeleteMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("Delete", reader), - id(id), val(val) {} + val(val) {} virtual bool Process() { - return input_mgr->Delete(Object(), id, val); + return input_mgr->Delete(Object(), val); } private: - int id; Value* *val; }; class ClearMessage : public threading::OutputMessage { public: - ClearMessage(ReaderFrontend* reader, int id) - : threading::OutputMessage("Clear", reader), - id(id) {} + ClearMessage(ReaderFrontend* reader) + : threading::OutputMessage("Clear", reader) {} virtual bool Process() { - input_mgr->Clear(Object(), id); + input_mgr->Clear(Object()); return true; } private: - int id; }; class SendEventMessage : public threading::OutputMessage { @@ -73,47 +69,30 @@ private: class SendEntryMessage : public threading::OutputMessage { public: - SendEntryMessage(ReaderFrontend* reader, const int id, Value* *val) + SendEntryMessage(ReaderFrontend* reader, Value* *val) : threading::OutputMessage("SendEntry", reader), - id(id), val(val) { } + val(val) { } virtual bool Process() { - input_mgr->SendEntry(Object(), id, val); + input_mgr->SendEntry(Object(), val); return true; } private: - const int id; Value* *val; }; class EndCurrentSendMessage : public threading::OutputMessage { public: - EndCurrentSendMessage(ReaderFrontend* reader, const int id) - : threading::OutputMessage("EndCurrentSend", reader), - id(id) {} + EndCurrentSendMessage(ReaderFrontend* reader) + : threading::OutputMessage("EndCurrentSend", reader) {} virtual bool Process() { - input_mgr->EndCurrentSend(Object(), id); + input_mgr->EndCurrentSend(Object()); return true; } private: - const int id; -}; - -class FilterRemovedMessage : public threading::OutputMessage { -public: - FilterRemovedMessage(ReaderFrontend* reader, const int id) - : threading::OutputMessage("FilterRemoved", reader), - id(id) {} - - virtual bool Process() { - return input_mgr->RemoveFilterContinuation(Object(), id); - } - -private: - const int id; }; class ReaderFinishedMessage : public threading::OutputMessage { @@ -155,19 +134,19 @@ ReaderBackend::~ReaderBackend() } -void ReaderBackend::Put(int id, Value* *val) +void ReaderBackend::Put(Value* *val) { - SendOut(new PutMessage(frontend, id, val)); + SendOut(new PutMessage(frontend, val)); } -void ReaderBackend::Delete(int id, Value* *val) +void ReaderBackend::Delete(Value* *val) { - SendOut(new DeleteMessage(frontend, id, val)); + SendOut(new DeleteMessage(frontend, val)); } -void ReaderBackend::Clear(int id) +void ReaderBackend::Clear() { - SendOut(new ClearMessage(frontend, id)); + SendOut(new ClearMessage(frontend)); } void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) @@ -175,70 +154,32 @@ void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *va SendOut(new SendEventMessage(frontend, name, num_vals, vals)); } -void ReaderBackend::EndCurrentSend(int id) +void ReaderBackend::EndCurrentSend() { - SendOut(new EndCurrentSendMessage(frontend, id)); + SendOut(new EndCurrentSendMessage(frontend)); } -void ReaderBackend::SendEntry(int id, Value* *vals) +void ReaderBackend::SendEntry(Value* *vals) { - SendOut(new SendEntryMessage(frontend, id, vals)); + SendOut(new SendEntryMessage(frontend, vals)); } -bool ReaderBackend::Init(string arg_source, int mode, bool arg_autostart) +bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* arg_fields) { source = arg_source; - autostart = arg_autostart; SetName("InputReader/"+source); // disable if DoInit returns error. - disabled = !DoInit(arg_source, mode); + int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); - if ( disabled ) { + if ( !success ) { Error("Init failed"); DisableFrontend(); } - return !disabled; -} - -bool ReaderBackend::StartReading() { - if ( disabled ) - return false; - - int success = DoStartReading(); - - if ( success == false ) { - DisableFrontend(); - } - return success; } -bool ReaderBackend::AddFilter(int id, int arg_num_fields, - const Field* const * arg_fields) -{ - if ( disabled ) - return false; - - bool success = DoAddFilter(id, arg_num_fields, arg_fields); - if ( success && autostart) { - autostart = false; - return StartReading(); - } - return success; -} - -bool ReaderBackend::RemoveFilter(int id) -{ - if ( disabled ) - return false; - - bool success = DoRemoveFilter(id); - SendOut(new FilterRemovedMessage(frontend, id)); - return success; // yes, I know, noone reads this. -} - void ReaderBackend::Finish() { DoFinish(); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index e167f8ff47..28fd99f2b9 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -53,46 +53,14 @@ public: * * @param mode the opening mode for the input source * - * @param autostart automatically start the input source after the first filter has been added - * - * @return False if an error occured. - */ - bool Init(string arg_source, int mode, bool autostart); - - /** - * One-time start method of the reader. - * - * This method is called from the scripting layer, after all filters have been added. - * No data should be read before this method is called. - * - * If autostart in Init is set to true, this method is called automatically by the backend after - * the first filter has been added. - */ - bool StartReading(); - - /** - * Add an input filter to the input stream - * - * @param id identifier of the input stream - * * @param arg_num_fields number of fields contained in \a fields * * @param fields the types and names of the fields to be retrieved from the input source * * @return False if an error occured. */ - bool AddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); + bool Init(string arg_source, int mode, int arg_num_fields, const threading::Field* const* fields); - - /** - * Remove an input filter to the input stream - * - * @param id identifier of the input stream - * - * @return False if an error occured. - */ - bool RemoveFilter ( int id ); - /** * Finishes reading from this input stream in a regular fashion. Must not be * called if an error has been indicated earlier. After calling this, @@ -131,33 +99,7 @@ protected: * disabled and eventually deleted. When returning false, an * implementation should also call Error() to indicate what happened. */ - virtual bool DoInit(string arg_sources, int mode) = 0; - - /** - * Reader-specific start method. After this function has been called, data may be read from - * the input source and be sent to the specified filters - * - * A reader implementation must override this method. - * If it returns false, it will be assumed that a fatal error has occured - * that prevents the reader from further operation; it will then be - * disabled and eventually deleted. When returning false, an implementation - * should also call Error to indicate what happened. - */ - virtual bool DoStartReading() = 0; - - /** - * Reader-specific method to add a filter. - * - * A reader implementation must override this method. - */ - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ) = 0; - - /** - * Reader-specific method to remove a filter. - * - * A reader implementation must override this method. - */ - virtual bool DoRemoveFilter( int id ) = 0; + virtual bool DoInit(string arg_sources, int mode, int arg_num_fields, const threading::Field* const* fields) = 0; /** * Reader-specific method implementing input finalization at @@ -209,31 +151,26 @@ protected: * * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void Put(int id, threading::Value* *val); + void Put(threading::Value* *val); /** * Method allowing a reader to delete a specific value from a bro table. * * If the receiving filter is an event, only a removed event is raised * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void Delete(int id, threading::Value* *val); + void Delete(threading::Value* *val); /** * Method allowing a reader to clear a value from a bro table. * * If the receiving filter is an event, this is ignored. * - * @param id the input filter id for which the values are sent */ - void Clear(int id); + void Clear(); // Content-sending-functions (tracking mode): Only changed lines are propagated. @@ -243,11 +180,9 @@ protected: * * If the filter points to a table, the values are inserted into the table; if it points to an event, the event is raised. * - * @param id the input filter id for which the values are sent - * * @param val list of threading::Values expected by the filter */ - void SendEntry(int id, threading::Value* *vals); + void SendEntry(threading::Value* *vals); /** * Method telling the manager, that the current list of entries sent by SendEntry is finished. @@ -255,9 +190,8 @@ protected: * For table filters, all entries that were not updated since the last EndCurrentSend will be deleted, because they are no longer * present in the input source * - * @param id the input filter id for which the values are sent */ - void EndCurrentSend(int id); + void EndCurrentSend(); /** * Triggered by regular heartbeat messages from the main thread. diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index f7fc23bf72..9711997821 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -12,16 +12,17 @@ namespace input { class InitMessage : public threading::InputMessage { public: - InitMessage(ReaderBackend* backend, const string source, const int mode, const bool autostart) + InitMessage(ReaderBackend* backend, const string source, const int mode, const int num_fields, const threading::Field* const* fields) : threading::InputMessage("Init", backend), - source(source), mode(mode), autostart(autostart) { } + source(source), mode(mode), num_fields(num_fields), fields(fields) { } - virtual bool Process() { return Object()->Init(source, mode, autostart); } + virtual bool Process() { return Object()->Init(source, mode, num_fields, fields); } private: const string source; const int mode; - const bool autostart; + const int num_fields; + const threading::Field* const* fields; }; class UpdateMessage : public threading::InputMessage @@ -44,44 +45,6 @@ public: virtual bool Process() { Object()->Finish(); return true; } }; -class StartReadingMessage : public threading::InputMessage -{ -public: - StartReadingMessage(ReaderBackend* backend) - : threading::InputMessage("StartReading", backend) - { } - - virtual bool Process() { Object()->StartReading(); return true; } -}; - -class AddFilterMessage : public threading::InputMessage -{ -public: - AddFilterMessage(ReaderBackend* backend, const int id, const int num_fields, const threading::Field* const* fields) - : threading::InputMessage("AddFilter", backend), - id(id), num_fields(num_fields), fields(fields) { } - - virtual bool Process() { return Object()->AddFilter(id, num_fields, fields); } - -private: - const int id; - const int num_fields; - const threading::Field* const* fields; -}; - -class RemoveFilterMessage : public threading::InputMessage -{ -public: - RemoveFilterMessage(ReaderBackend* backend, const int id) - : threading::InputMessage("RemoveFilter", backend), - id(id) { } - - virtual bool Process() { return Object()->RemoveFilter(id); } - -private: - const int id; -}; - ReaderFrontend::ReaderFrontend(bro_int_t type) { disabled = initialized = false; @@ -95,7 +58,7 @@ ReaderFrontend::ReaderFrontend(bro_int_t type) { ReaderFrontend::~ReaderFrontend() { } -void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { +void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, const threading::Field* const* fields) { if ( disabled ) return; @@ -105,37 +68,33 @@ void ReaderFrontend::Init(string arg_source, int mode, bool autostart) { source = arg_source; initialized = true; - backend->SendIn(new InitMessage(backend, arg_source, mode, autostart)); + backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); } void ReaderFrontend::Update() { if ( disabled ) return; + if ( !initialized ) { + reporter->Error("Tried to call update on uninitialized reader"); + return; + } + backend->SendIn(new UpdateMessage(backend)); } void ReaderFrontend::Finish() { if ( disabled ) return; + + if ( !initialized ) { + reporter->Error("Tried to call finish on uninitialized reader"); + return; + } backend->SendIn(new FinishMessage(backend)); } -void ReaderFrontend::AddFilter(const int id, const int arg_num_fields, const threading::Field* const* fields) { - if ( disabled ) - return; - - backend->SendIn(new AddFilterMessage(backend, id, arg_num_fields, fields)); -} - -void ReaderFrontend::RemoveFilter(const int id) { - if ( disabled ) - return; - - backend->SendIn(new RemoveFilterMessage(backend, id)); -} - string ReaderFrontend::Name() const { if ( source.size() ) @@ -144,13 +103,5 @@ string ReaderFrontend::Name() const return ty_name + "/" + source; } -void ReaderFrontend::StartReading() { - if ( disabled ) - return; - - backend->SendIn(new StartReadingMessage(backend)); } -} - - diff --git a/src/input/ReaderFrontend.h b/src/input/ReaderFrontend.h index 0df4c00e8c..1c3306e0c1 100644 --- a/src/input/ReaderFrontend.h +++ b/src/input/ReaderFrontend.h @@ -49,18 +49,7 @@ public: * See ReaderBackend::Init() for arguments. * This method must only be called from the main thread. */ - void Init(string arg_source, int mode, bool autostart); - - /** - * Start the reader. - * - * This methods starts the reader, after all necessary filters have been added. - * It is not necessary to call this function, if autostart has been set. - * If autostart has been set, the reader will be initialized automatically after the first filter has been added - * - * This method must only be called from the main thread. - */ - void StartReading(); + void Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* fields); /** * Force an update of the current input source. Actual action depends on @@ -72,20 +61,6 @@ public: */ void Update(); - /** - * Add a filter to the current input source. - * - * See ReaderBackend::AddFilter for arguments. - * - * The method takes ownership of \a fields - */ - void AddFilter( const int id, const int arg_num_fields, const threading::Field* const* fields ); - - /** - * Removes a filter to the current input source. - */ - void RemoveFilter ( const int id ); - /** * Finalizes writing to this tream. * diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index ab26442524..bb59b3fc1d 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -76,7 +76,6 @@ Ascii::~Ascii() void Ascii::DoFinish() { - filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -84,9 +83,8 @@ void Ascii::DoFinish() } } -bool Ascii::DoInit(string path, int arg_mode) +bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { - started = false; fname = path; mode = arg_mode; mtime = 0; @@ -107,17 +105,10 @@ bool Ascii::DoInit(string path, int arg_mode) file->close(); return false; } + + num_fields = arg_num_fields; + fields = arg_fields; - return true; -} - -bool Ascii::DoStartReading() { - if ( started == true ) { - Error("Started twice"); - return false; - } - - started = true; switch ( mode ) { case MANUAL: case REREAD: @@ -131,46 +122,11 @@ bool Ascii::DoStartReading() { return true; } -bool Ascii::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { - if ( HasFilter(id) ) { - Error("Filter was added twice, ignoring."); - return false; // no, we don't want to add this a second time - } - - Filter f; - f.num_fields = arg_num_fields; - f.fields = fields; - - filters[id] = f; - - return true; -} - -bool Ascii::DoRemoveFilter ( int id ) { - if (!HasFilter(id) ) { - Error("Filter removal of nonexisting filter requested."); - return false; - } - - assert ( filters.erase(id) == 1 ); - - return true; -} - - -bool Ascii::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} - bool Ascii::ReadHeader(bool useCached) { // try to read the header line... string line; - map fields; + map ifields; if ( !useCached ) { if ( !GetLine(line) ) { @@ -194,37 +150,35 @@ bool Ascii::ReadHeader(bool useCached) { if ( !getline(splitstream, s, separator[0])) break; - fields[s] = pos; + ifields[s] = pos; pos++; } //printf("Updating fields from description %s\n", line.c_str()); - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - (*it).second.columnMap.clear(); - - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const Field* field = (*it).second.fields[i]; - - map::iterator fit = fields.find(field->name); - if ( fit == fields.end() ) { - Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); - return false; - } - - - FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); - if ( field->secondary_name != "" ) { - map::iterator fit2 = fields.find(field->secondary_name); - if ( fit2 == fields.end() ) { - Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); - return false; - } - f.secondary_position = fields[field->secondary_name]; - } - (*it).second.columnMap.push_back(f); + columnMap.clear(); + + for ( unsigned int i = 0; i < num_fields; i++ ) { + const Field* field = fields[i]; + + map::iterator fit = ifields.find(field->name); + if ( fit == ifields.end() ) { + Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + return false; } + + FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = ifields.find(field->secondary_name); + if ( fit2 == ifields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; + } + f.secondary_position = ifields[field->secondary_name]; + } + columnMap.push_back(f); } + // well, that seems to have worked... return true; @@ -461,57 +415,55 @@ bool Ascii::DoUpdate() { pos--; // for easy comparisons of max element. - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - Value** fields = new Value*[(*it).second.num_fields]; + Value** fields = new Value*[num_fields]; - int fpos = 0; - for ( vector::iterator fit = (*it).second.columnMap.begin(); - fit != (*it).second.columnMap.end(); - fit++ ){ + int fpos = 0; + for ( vector::iterator fit = columnMap.begin(); + fit != columnMap.end(); + fit++ ){ - if ( (*fit).position > pos || (*fit).secondary_position > pos ) { - Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); - return false; - } - - Value* val = EntryToVal(stringfields[(*fit).position], *fit); - if ( val == 0 ) { - Error("Could not convert String value to Val"); - return false; - } - - if ( (*fit).secondary_position != -1 ) { - // we have a port definition :) - assert(val->type == TYPE_PORT ); - // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - - val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); - } - - fields[fpos] = val; - - fpos++; + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); + return false; } - //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); - assert ( (unsigned int) fpos == (*it).second.num_fields ); + Value* val = EntryToVal(stringfields[(*fit).position], *fit); + if ( val == 0 ) { + Error("Could not convert String value to Val"); + return false; + } + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - if ( mode == STREAM ) { - Put((*it).first, fields); - } else { - SendEntry((*it).first, fields); + val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); } - /* Do not do this, ownership changes to other thread - * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - delete fields[i]; - } - delete [] fields; - */ + fields[fpos] = val; + + fpos++; } + //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); + assert ( (unsigned int) fpos == num_fields ); + + if ( mode == STREAM ) { + Put(fields); + } else { + SendEntry(fields); + } + + /* Do not do this, ownership changes to other thread + * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; + */ + } @@ -519,9 +471,7 @@ bool Ascii::DoUpdate() { //file->seekg(0, ios::beg); // and seek to start. if ( mode != STREAM ) { - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - EndCurrentSend((*it).first); - } + EndCurrentSend(); } return true; diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 3bb0e91853..40f92be717 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -39,33 +39,22 @@ public: protected: - virtual bool DoInit(string path, int mode); - - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); - - virtual bool DoRemoveFilter ( int id ); + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoFinish(); virtual bool DoUpdate(); - virtual bool DoStartReading(); - private: virtual bool DoHeartbeat(double network_time, double current_time); - struct Filter { - unsigned int num_fields; + unsigned int num_fields; - const threading::Field* const * fields; // raw mapping + const threading::Field* const * fields; // raw mapping - // map columns in the file to columns to send back to the manager - vector columnMap; - - }; - - bool HasFilter(int id); + // map columns in the file to columns to send back to the manager + vector columnMap; bool ReadHeader(bool useCached); threading::Value* EntryToVal(string s, FieldMapping type); @@ -75,8 +64,6 @@ private: ifstream* file; string fname; - map filters; - // Options set from the script-level. string separator; @@ -91,7 +78,6 @@ private: int mode; - bool started; time_t mtime; }; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f2892e7af5..27415b525f 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -40,7 +40,6 @@ Raw::~Raw() void Raw::DoFinish() { - filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -48,9 +47,8 @@ void Raw::DoFinish() } } -bool Raw::DoInit(string path, int arg_mode) +bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { - started = false; fname = path; mode = arg_mode; mtime = 0; @@ -66,16 +64,19 @@ bool Raw::DoInit(string path, int arg_mode) return false; } - return true; -} - -bool Raw::DoStartReading() { - if ( started == true ) { - Error("Started twice"); + if ( arg_num_fields != 1 ) { + Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); return false; - } + } + + if ( fields[0]->type != TYPE_STRING ) { + Error("Filter for raw reader contains a field that is not of type string."); + return false; + } + + num_fields = arg_num_fields; + fields = arg_fields; - started = true; switch ( mode ) { case MANUAL: case REREAD: @@ -89,51 +90,6 @@ bool Raw::DoStartReading() { return true; } -bool Raw::DoAddFilter( int id, int arg_num_fields, const Field* const* fields ) { - - if ( arg_num_fields != 1 ) { - Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); - return false; - } - - if ( fields[0]->type != TYPE_STRING ) { - Error("Filter for raw reader contains a field that is not of type string."); - return false; - } - - if ( HasFilter(id) ) { - Error("Filter was added twice, ignoring"); - return false; // no, we don't want to add this a second time - } - - Filter f; - f.num_fields = arg_num_fields; - f.fields = fields; - - filters[id] = f; - - return true; -} - -bool Raw::DoRemoveFilter ( int id ) { - if (!HasFilter(id) ) { - Error("Filter removal of nonexisting filter requested."); - return false; - } - - assert ( filters.erase(id) == 1 ); - - return true; -} - - -bool Raw::HasFilter(int id) { - map::iterator it = filters.find(id); - if ( it == filters.end() ) { - return false; - } - return true; -} bool Raw::GetLine(string& str) { while ( getline(*file, str, separator[0]) ) { @@ -188,21 +144,16 @@ bool Raw::DoUpdate() { string line; while ( GetLine(line) ) { - for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + assert (num_fields == 1); + + Value** fields = new Value*[1]; - assert ((*it).second.num_fields == 1); + // filter has exactly one text field. convert to it. + Value* val = new Value(TYPE_STRING, true); + val->val.string_val = new string(line); + fields[0] = val; - Value** fields = new Value*[1]; - - // filter has exactly one text field. convert to it. - Value* val = new Value(TYPE_STRING, true); - val->val.string_val = new string(line); - fields[0] = val; - - Put((*it).first, fields); - - } - + Put(fields); } return true; diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index e046cb2ff7..ace4e0ee88 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -19,37 +19,21 @@ public: protected: - virtual bool DoInit(string path, int mode); - - virtual bool DoAddFilter( int id, int arg_num_fields, const threading::Field* const* fields ); - - virtual bool DoRemoveFilter ( int id ); + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); virtual void DoFinish(); virtual bool DoUpdate(); - virtual bool DoStartReading(); - private: virtual bool DoHeartbeat(double network_time, double current_time); - struct Filter { - unsigned int num_fields; - - const threading::Field* const * fields; // raw mapping - }; - - bool HasFilter(int id); - bool GetLine(string& str); ifstream* file; string fname; - map filters; - // Options set from the script-level. string separator; @@ -58,8 +42,11 @@ private: int mode; - bool started; time_t mtime; + + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping }; diff --git a/src/types.bif b/src/types.bif index a9c6ecb3a8..26850bfa93 100644 --- a/src/types.bif +++ b/src/types.bif @@ -182,10 +182,6 @@ enum Event %{ EVENT_REMOVED, %} -enum ID %{ - Unknown, -%} - enum Mode %{ MANUAL = 0, REREAD = 1, diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 156898edca..827b1ce283 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -45,12 +41,10 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(id: string) { print servers; } From 367c4b4a7e7d8522d16a64f1d113a56104cab316 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 07:53:29 -0700 Subject: [PATCH 077/123] make raw reading work. apparently there was a crash in the reader plugin, but main bro did not notice but waited for eternity for it do to something. --- src/input/Manager.cc | 18 ++++++++---------- src/input/readers/Raw.cc | 9 +++++++-- .../scripts/base/frameworks/input/raw.bro | 9 ++------- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 44b5bf44db..af82b676c6 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -220,11 +220,8 @@ bool Manager::CreateStream(Filter* info, RecordVal* description) info->name = name; info->source = source; - -#ifdef DEBUG - DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", - name.c_str()); -#endif + DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", + name.c_str()); return true; @@ -334,6 +331,10 @@ bool Manager::CreateEventStream(RecordVal* fval) { filter->reader->Init(filter->source, filter->mode, filter->num_fields, logf ); readers[filter->reader] = filter; + + DBG_LOG(DBG_INPUT, "Successfully created event stream %s", + filter->name.c_str()); + return true; } @@ -482,12 +483,9 @@ bool Manager::CreateTableStream(RecordVal* fval) { filter->reader->Init(filter->source, filter->mode, fieldsV.size(), fields ); readers[filter->reader] = filter; - -#ifdef DEBUG - DBG_LOG(DBG_INPUT, "Successfully created table stream %s", - filter->name.c_str()); -#endif + DBG_LOG(DBG_INPUT, "Successfully created table stream %s", + filter->name.c_str()); return true; } diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 27415b525f..a83314c491 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -63,6 +63,9 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con Error(Fmt("Init: cannot open %s", fname.c_str())); return false; } + + num_fields = arg_num_fields; + fields = arg_fields; if ( arg_num_fields != 1 ) { Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); @@ -74,8 +77,9 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con return false; } - num_fields = arg_num_fields; - fields = arg_fields; +#ifdef DEBUG + Debug(DBG_INPUT, "Raw reader created, will perform first update"); +#endif switch ( mode ) { case MANUAL: @@ -87,6 +91,7 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con assert(false); } + return true; } diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 5f196648b6..4de5c3450e 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -16,10 +16,6 @@ sdf module A; -export { - redef enum Input::ID += { INPUT }; -} - type Val: record { s: string; }; @@ -30,6 +26,5 @@ event line(tpe: Input::Event, s: string) { event bro_init() { - Input::create_stream(A::INPUT, [$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM]); - Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); } From 842f635695d67f027ff2911aa41272ec9ceabf71 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 08:10:28 -0700 Subject: [PATCH 078/123] give EventDescripion field back to events --- src/input/Manager.cc | 25 ++++++++++++++----- .../scripts/base/frameworks/input/raw.bro | 3 ++- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index af82b676c6..7ac1de92c2 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -41,11 +41,14 @@ public: EnumVal* type; ReaderFrontend* reader; + RecordVal* description; + virtual ~Filter(); }; Manager::Filter::~Filter() { Unref(type); + Unref(description); delete(reader); } @@ -219,6 +222,8 @@ bool Manager::CreateStream(Filter* info, RecordVal* description) info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault info->name = name; info->source = source; + Ref(description); + info->description = description; DBG_LOG(DBG_INPUT, "Successfully created new input stream %s", name.c_str()); @@ -274,27 +279,33 @@ bool Manager::CreateEventStream(RecordVal* fval) { reporter->Error("events first attribute must be of type Input::Event"); return false; } + + if ( ! same_type((*args)[1], BifType::Record::Input::EventDescription, 0) ) + { + reporter->Error("events second attribute must be of type Input::EventDescription"); + return false; + } if ( want_record->InternalInt() == 0 ) { - if ( args->length() != fields->NumFields() + 1 ) { - reporter->Error("events has wrong number of arguments"); + if ( args->length() != fields->NumFields() + 2 ) { + reporter->Error("event has wrong number of arguments"); return false; } for ( int i = 0; i < fields->NumFields(); i++ ) { - if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { reporter->Error("Incompatible type for event"); return false; } } } else if ( want_record->InternalInt() == 1 ) { - if ( args->length() != 2 ) { - reporter->Error("events has wrong number of arguments"); + if ( args->length() != 3 ) { + reporter->Error("event has wrong number of arguments"); return false; } - if ( !same_type((*args)[1], fields ) ) { + if ( !same_type((*args)[2], fields ) ) { reporter->Error("Incompatible type for event"); return false; } @@ -965,6 +976,8 @@ int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const * // no tracking, send everything with a new event... //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); + Ref(filter->description); + out_vals.push_back(filter->description); int position = 0; if ( filter->want_record ) { diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 4de5c3450e..0399eb301d 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -20,7 +20,8 @@ type Val: record { s: string; }; -event line(tpe: Input::Event, s: string) { +event line(tpe: Input::Event, description: Input::EventDescription, s: string) { + print description; print s; } From e59aed6ce35afc89bdf0d8114a0a2a330dee83ec Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 08:31:19 -0700 Subject: [PATCH 079/123] for seth - reverse order of event arguments --- src/input/Manager.cc | 12 ++++++------ testing/btest/scripts/base/frameworks/input/raw.bro | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 7ac1de92c2..d0db846769 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -274,15 +274,15 @@ bool Manager::CreateEventStream(RecordVal* fval) { return false; } - if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) { - reporter->Error("events first attribute must be of type Input::Event"); + reporter->Error("events second attribute must be of type Input::Event"); return false; } - if ( ! same_type((*args)[1], BifType::Record::Input::EventDescription, 0) ) + if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) { - reporter->Error("events second attribute must be of type Input::EventDescription"); + reporter->Error("events first attribute must be of type Input::EventDescription"); return false; } @@ -973,11 +973,11 @@ int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const * Val *val; list out_vals; + Ref(filter->description); + out_vals.push_back(filter->description); // no tracking, send everything with a new event... //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); out_vals.push_back(type); - Ref(filter->description); - out_vals.push_back(filter->description); int position = 0; if ( filter->want_record ) { diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 0399eb301d..6b9fb8ef96 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -20,7 +20,7 @@ type Val: record { s: string; }; -event line(tpe: Input::Event, description: Input::EventDescription, s: string) { +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { print description; print s; } From 29f56b4986bfe3b4fff59458fe0aacfef572c1e6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 23:43:13 -0700 Subject: [PATCH 080/123] continue finetuning of interface + adjust tests. streaming + re-reading do not seem to work completely correctly + there are still some strange random crashes. --- .../scripts.base.frameworks.input.event/out | 49 ++++++++ .../scripts.base.frameworks.input.raw/out | 56 ++++++++++ .../out | 105 ++++++++++++++++++ .../scripts/base/frameworks/input/basic.bro | 2 +- .../scripts/base/frameworks/input/event.bro | 13 +-- .../frameworks/input/onecolumn-norecord.bro | 13 +-- .../frameworks/input/onecolumn-record.bro | 11 +- .../scripts/base/frameworks/input/port.bro | 15 +-- .../frameworks/input/predicate-stream.bro | 11 +- .../base/frameworks/input/predicate.bro | 12 +- .../scripts/base/frameworks/input/raw.bro | 2 + .../scripts/base/frameworks/input/reread.bro | 15 +-- .../scripts/base/frameworks/input/stream.bro | 12 +- .../base/frameworks/input/tableevent.bro | 15 +-- .../base/frameworks/input/twofilters.bro | 6 +- 15 files changed, 251 insertions(+), 86 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out index e32a2aea00..59070cd88e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -1,21 +1,70 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 1 T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 2 T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 3 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 4 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 5 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 6 F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::i; +print A::b; +}] Input::EVENT_NEW 7 T diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out index 2059013c5d..34a5599dc9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -1,8 +1,64 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index 54048a86b8..56b36a1a0e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,21 +1,126 @@ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=1] T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=2] T +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=3] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=4] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=5] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=6] F +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[2] = T, +[4] = F, +[6] = F, +[7] = T, +[1] = T, +[5] = F, +[3] = F +}, idx=, val=, want_record=F, ev=line +{ +print description; +print tpe; +print left; +print right; +}, pred=] Input::EVENT_NEW [i=7] T diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 827b1ce283..8d4028a12e 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -45,6 +45,6 @@ event bro_init() Input::remove("ssh"); } -event Input::update_finished(id: string) { +event Input::update_finished(name: string, source:string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 41eba1613c..dca75334d0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -19,16 +19,13 @@ module A; -export { - redef enum Input::ID += { INPUT }; -} - type Val: record { i: int; b: bool; }; -event line(tpe: Input::Event, i: int, b: bool) { +event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + print description; print tpe; print i; print b; @@ -36,6 +33,6 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index bcbba05a3e..d6c81cb2db 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -30,12 +26,11 @@ global servers: table[int] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 1c532ba6a9..ca1e956f35 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -14,10 +14,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -30,12 +26,11 @@ global servers: table[int] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$name="input", $source="input.log", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 801d6bac3f..88e86eb5dc 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -13,10 +13,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: addr; }; @@ -29,17 +25,14 @@ global servers: table[addr] of Val = table(); event bro_init() { - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]); print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; - Input::remove_tablefilter(A::INPUT, "input"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print servers[1.2.3.4]; print servers[1.2.3.5]; print servers[1.2.3.6]; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index f08aaef998..20c69131cb 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -23,10 +23,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -38,7 +34,7 @@ type Val: record { global servers: table[int] of Val = table(); global ct: int; -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { ct = ct + 1; if ( ct < 3 ) { return; @@ -75,9 +71,10 @@ event bro_init() { ct = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log", $mode=Input::STREAM]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, + Input::add_table([$source="input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); + Input::remove("input"); + } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 009911e6a8..278ac7418e 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -20,10 +20,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -37,13 +33,13 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); + Input::remove("input"); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 6b9fb8ef96..8ec6c12a78 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -22,10 +22,12 @@ type Val: record { event line(description: Input::EventDescription, tpe: Input::Event, s: string) { print description; + print tpe; print s; } event bro_init() { Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 742d68605b..0930cdcb34 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -62,10 +62,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -94,8 +90,9 @@ global outfile: file; global try: count; -event line(tpe: Input::Event, left: Idx, right: Val) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; + #print outfile, description; print outfile, tpe; print outfile, left; print outfile, right; @@ -106,8 +103,7 @@ event bro_init() outfile = open ("../out"); try = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::REREAD]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, $pred(typ: Input::Event, left: Idx, right: Val) = { print outfile, "============PREDICATE============"; print outfile, typ; @@ -119,7 +115,7 @@ event bro_init() } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { print outfile, "==========SERVERS============"; print outfile, servers; @@ -127,7 +123,6 @@ event Input::update_finished(id: Input::ID) { if ( try == 5 ) { print outfile, "done"; close(outfile); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } } diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index db368074aa..571a2273c1 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -28,10 +28,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -60,7 +56,7 @@ global outfile: file; global try: count; -event line(tpe: Input::Event, left: Idx, right: Val) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; print outfile, tpe; print outfile, left; @@ -73,8 +69,7 @@ event line(tpe: Input::Event, left: Idx, right: Val) { if ( try == 3 ) { print outfile, "done"; close(outfile); - Input::remove_tablefilter(A::INPUT, "ssh"); - Input::remove_stream(A::INPUT); + Input::remove("input"); } } @@ -83,7 +78,6 @@ event bro_init() outfile = open ("../out"); try = 0; # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="../input.log", $mode=Input::STREAM]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); + Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]); } diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 0c86ac94b8..e40485dd12 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,5 +1,5 @@ # -# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: bro -b %INPUT >out # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -18,12 +18,6 @@ redef InputAscii::empty_field = "EMPTY"; -module A; - -export { - redef enum Log::ID += { LOG }; -} - type Idx: record { i: int; }; @@ -34,7 +28,8 @@ type Val: record { global destination: table[int] of Val = table(); -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { + print description; print tpe; print left; print right; @@ -42,6 +37,6 @@ event line(tpe: Input::Event, left: Idx, right: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 260f73e58f..5e94aafba9 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -20,10 +20,6 @@ redef InputAscii::empty_field = "EMPTY"; module A; -export { - redef enum Input::ID += { INPUT }; -} - type Idx: record { i: int; }; @@ -49,7 +45,7 @@ event bro_init() Input::force_update(A::INPUT); } -event Input::update_finished(id: Input::ID) { +event Input::update_finished(name: string, source: string) { if ( done == T ) { return; } From 3286d013c9d37b2fbeb9bcdbf171d83fae12f168 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 16 Mar 2012 23:45:10 -0700 Subject: [PATCH 081/123] forgot two files. --- scripts/base/frameworks/input/main.bro | 2 +- src/input/Manager.cc | 38 +++++++++++++++----------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4f7f9983d1..e06dfae005 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -110,7 +110,7 @@ export { global force_update: function(id: string) : bool; ## Event that is called, when the update of a specific source is finished - global update_finished: event(id: string); + global update_finished: event(name: string, source:string); } @load base/input.bif diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d0db846769..28b6afe63f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -376,7 +376,7 @@ bool Manager::CreateTableStream(RecordVal* fval) { } TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); - // check if index fields match tabla description + // check if index fields match table description { int num = idx->NumFields(); const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); @@ -416,29 +416,35 @@ bool Manager::CreateTableStream(RecordVal* fval) { const type_list* args = etype->ArgTypes()->Types(); - if ( args->length() != 3 ) + if ( args->length() != 4 ) { - reporter->Error("Table event must take 3 arguments"); + reporter->Error("Table event must take 4 arguments"); return false; } - if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) { - reporter->Error("table events first attribute must be of type Input::Event"); + reporter->Error("table events first attribute must be of type Input::TableDescription"); return false; } - if ( ! same_type((*args)[1], idx) ) + if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events second attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[2], idx) ) { reporter->Error("table events index attributes do not match"); return false; } - if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) { reporter->Error("table events value attributes do not match"); return false; - } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[3], val->FieldType(0) ) ) { reporter->Error("table events value attribute does not match"); return false; } @@ -825,14 +831,15 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 3, ev, predidx, oldval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, predidx); + Ref(filter->description); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 3, ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); } } } @@ -936,7 +943,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } - SendEvent(handler, 1, new BroString(i->name)); + SendEvent(handler, 2, new BroString(i->name), new BroString(i->source)); } void Manager::Put(ReaderFrontend* reader, Value* *vals) { @@ -1080,14 +1087,13 @@ int Manager::PutTable(Filter* i, const Value* const *vals) { assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 3, ev, predidx, oldval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - Ref(valval); if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 3, ev, predidx); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 3, ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); } } } From bf597012f89bbcfd374e84e98a85b48d637254f6 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 18 Mar 2012 10:50:10 -0700 Subject: [PATCH 082/123] fix some stupid, not that easy to find bugs. Functionality seems to work completely again - including all tests passing. --- src/input/Manager.cc | 10 +- src/input/ReaderBackend.cc | 2 + src/input/readers/Ascii.cc | 8 +- .../scripts.base.frameworks.input.repeat/out | 160 +++++ .../scripts.base.frameworks.input.reread/out | 662 +++++++++++++----- .../out | 15 - .../scripts/base/frameworks/input/repeat.bro | 41 ++ .../scripts/base/frameworks/input/reread.bro | 6 +- .../base/frameworks/input/twofilters.bro | 100 --- 9 files changed, 690 insertions(+), 314 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.repeat/out delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out create mode 100644 testing/btest/scripts/base/frameworks/input/repeat.bro delete mode 100644 testing/btest/scripts/base/frameworks/input/twofilters.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 28b6afe63f..fb7ea6edca 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -837,7 +837,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { Ref(valval); if ( filter->num_val_fields == 0 ) { Ref(filter->description); - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); + SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); } else { SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); } @@ -898,7 +898,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { Ref(predidx); Ref(val); - bool result = CallPred(filter->pred, 3, ev, predidx, val); + bool result = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict @@ -943,7 +943,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } - SendEvent(handler, 2, new BroString(i->name), new BroString(i->source)); + SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); } void Manager::Put(ReaderFrontend* reader, Value* *vals) { @@ -1154,7 +1154,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - filterresult = CallPred(filter->pred, 3, ev, predidx, val); + filterresult = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); if ( filterresult == false ) { // keep it. @@ -1170,7 +1170,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { assert(val != 0); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(filter->event, 3, ev, idxval, val); + SendEvent(filter->event, 4, filter->description->Ref(), ev, idxval, val); } } diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index b33e19d297..0a6ff37dc2 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -177,6 +177,8 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, DisableFrontend(); } + disabled = !success; + return success; } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index bb59b3fc1d..a04a40e780 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -89,6 +89,9 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c mode = arg_mode; mtime = 0; + num_fields = arg_num_fields; + fields = arg_fields; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; @@ -106,9 +109,6 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c return false; } - num_fields = arg_num_fields; - fields = arg_fields; - switch ( mode ) { case MANUAL: case REREAD: @@ -480,7 +480,7 @@ bool Ascii::DoUpdate() { bool Ascii::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - + switch ( mode ) { case MANUAL: // yay, we do nothing :) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out new file mode 100644 index 0000000000..71de0d2570 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.repeat/out @@ -0,0 +1,160 @@ +input0 +input.log +{ +[1] = T +} +input1 +input.log +{ +[1] = T +} +input2 +input.log +{ +[1] = T +} +input3 +input.log +{ +[1] = T +} +input4 +input.log +{ +[1] = T +} +input5 +input.log +{ +[1] = T +} +input6 +input.log +{ +[1] = T +} +input7 +input.log +{ +[1] = T +} +input8 +input.log +{ +[1] = T +} +input9 +input.log +{ +[1] = T +} +input10 +input.log +{ +[1] = T +} +input11 +input.log +{ +[1] = T +} +input12 +input.log +{ +[1] = T +} +input13 +input.log +{ +[1] = T +} +input14 +input.log +{ +[1] = T +} +input15 +input.log +{ +[1] = T +} +input16 +input.log +{ +[1] = T +} +input17 +input.log +{ +[1] = T +} +input18 +input.log +{ +[1] = T +} +input19 +input.log +{ +[1] = T +} +input20 +input.log +{ +[1] = T +} +input21 +input.log +{ +[1] = T +} +input22 +input.log +{ +[1] = T +} +input23 +input.log +{ +[1] = T +} +input24 +input.log +{ +[1] = T +} +input25 +input.log +{ +[1] = T +} +input26 +input.log +{ +[1] = T +} +input27 +input.log +{ +[1] = T +} +input28 +input.log +{ +[1] = T +} +input29 +input.log +{ +[1] = T +} +input30 +input.log +{ +[1] = T +} +input31 +input.log +{ +[1] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index f244f11a73..545a1cb781 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -14,8 +14,44 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-42] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -59,8 +95,56 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-43] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -116,8 +200,56 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_CHANGED +Left [i=-43] +Right [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -173,8 +305,68 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-44] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -203,8 +395,80 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-45] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -233,8 +497,92 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-46] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -263,8 +611,104 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type Input::EVENT_NEW +Left [i=-47] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -293,22 +737,8 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ -Input::EVENT_NEW -[i=-48] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -==========SERVERS============ -{ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -393,10 +823,30 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]] -} -============PREDICATE============ -Input::EVENT_REMOVED -[i=-43] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-48] +Right [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -408,175 +858,10 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-46] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-47] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-45] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_REMOVED -[i=-42] -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-43] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-46] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-44] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-47] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-45] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Input::EVENT_REMOVED -[i=-42] -[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ==========SERVERS============ { -[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, 1, @@ -587,6 +872,5 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -} -done +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out deleted file mode 100644 index 5b1ee5e983..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out +++ /dev/null @@ -1,15 +0,0 @@ -VALID -VALID -VALID -VALID -VALID -VALID -VALID -MARK -VALID -VALID -VALID -VALID -VALID -VALID -VALID diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro new file mode 100644 index 0000000000..58ce9a1675 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -0,0 +1,41 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32}; + +event bro_init() +{ + for ( i in one_to_32 ) { + Input::add_table([$source="input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]); + Input::remove(fmt("input%d", i)); + } +} + +event Input::update_finished(name: string, source: string) { + print name; + print source; + print destination; +} diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 0930cdcb34..f33b060fe0 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -92,9 +92,13 @@ global try: count; event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { print outfile, "============EVENT============"; - #print outfile, description; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; print outfile, tpe; + print outfile, "Left"; print outfile, left; + print outfile, "Right"; print outfile, right; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro deleted file mode 100644 index 5e94aafba9..0000000000 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ /dev/null @@ -1,100 +0,0 @@ -# -# @TEST-EXEC: bro %INPUT >out -# @TEST-EXEC: btest-diff out - -@TEST-START-FILE input.log -#separator \x09 -#path ssh -#fields i b -#types int bool -1 T -2 T -3 F -4 F -5 F -6 F -7 T -@TEST-END-FILE - -redef InputAscii::empty_field = "EMPTY"; - -module A; - -type Idx: record { - i: int; -}; - -type Val: record { - b: bool; -}; - -global destination1: table[int] of Val = table(); -global destination2: table[int] of Val = table(); - -global done: bool = F; - -event bro_init() -{ - # first read in the old stuff into the table... - Input::create_stream(A::INPUT, [$source="input.log", $autostart=F]); - Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, - $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } - ]); - Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - - Input::force_update(A::INPUT); -} - -event Input::update_finished(name: string, source: string) { - if ( done == T ) { - return; - } - - done = T; - - if ( 1 in destination1 ) { - print "VALID"; - } - if ( 2 in destination1 ) { - print "VALID"; - } - if ( !(3 in destination1) ) { - print "VALID"; - } - if ( !(4 in destination1) ) { - print "VALID"; - } - if ( !(5 in destination1) ) { - print "VALID"; - } - if ( !(6 in destination1) ) { - print "VALID"; - } - if ( 7 in destination1 ) { - print "VALID"; - } - - print "MARK"; - - if ( 2 in destination2 ) { - print "VALID"; - } - if ( 2 in destination2 ) { - print "VALID"; - } - if ( 3 in destination2 ) { - print "VALID"; - } - if ( 4 in destination2 ) { - print "VALID"; - } - if ( 5 in destination2 ) { - print "VALID"; - } - if ( 6 in destination2 ) { - print "VALID"; - } - if ( 7 in destination2 ) { - print "VALID"; - } -} From aa6026c1a7d40b9fa8bc553ad4f42a7150a1cbeb Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 18 Mar 2012 10:52:23 -0700 Subject: [PATCH 083/123] forgot to undo this - this idea did not work, because records cannot reference themselves. --- src/input/Manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index fb7ea6edca..d0386fbb3f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -898,7 +898,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { Ref(predidx); Ref(val); - bool result = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); + bool result = CallPred(filter->pred, 3, ev, predidx, val); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict @@ -1154,7 +1154,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - filterresult = CallPred(filter->pred, 4, filter->description->Ref(), ev, predidx, val); + filterresult = CallPred(filter->pred, 3, ev, predidx, val); if ( filterresult == false ) { // keep it. From 88e0cea598e5e57d87150c05e3d59989b6102fee Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 18 Mar 2012 15:31:47 -0700 Subject: [PATCH 084/123] add execute-mode support to the raw reader - allows to directly call commands and read their output. Note that fdstream.h is from boost and has a separate license: * (C) Copyright Nicolai M. Josuttis 2001. * Permission to copy, use, modify, sell and distribute this software * is granted provided this copyright notice appears in all copies. * This software is provided "as is" without express or implied * warranty, and with no claim as to its suitability for any purpose. --- src/input/fdstream.h | 184 ++++++++++++++++++ src/input/readers/Raw.cc | 63 ++++-- src/input/readers/Raw.h | 4 + src/types.bif | 1 + .../base/frameworks/input/executeraw.bro | 33 ++++ 5 files changed, 267 insertions(+), 18 deletions(-) create mode 100644 src/input/fdstream.h create mode 100644 testing/btest/scripts/base/frameworks/input/executeraw.bro diff --git a/src/input/fdstream.h b/src/input/fdstream.h new file mode 100644 index 0000000000..585e03d10b --- /dev/null +++ b/src/input/fdstream.h @@ -0,0 +1,184 @@ +/* The following code declares classes to read from and write to + * file descriptore or file handles. + * + * See + * http://www.josuttis.com/cppcode + * for details and the latest version. + * + * - open: + * - integrating BUFSIZ on some systems? + * - optimized reading of multiple characters + * - stream for reading AND writing + * - i18n + * + * (C) Copyright Nicolai M. Josuttis 2001. + * Permission to copy, use, modify, sell and distribute this software + * is granted provided this copyright notice appears in all copies. + * This software is provided "as is" without express or implied + * warranty, and with no claim as to its suitability for any purpose. + * + * Version: Jul 28, 2002 + * History: + * Jul 28, 2002: bugfix memcpy() => memmove() + * fdinbuf::underflow(): cast for return statements + * Aug 05, 2001: first public version + */ +#ifndef BOOST_FDSTREAM_HPP +#define BOOST_FDSTREAM_HPP + +#include +#include +#include +// for EOF: +#include +// for memmove(): +#include + + +// low-level read and write functions +#ifdef _MSC_VER +# include +#else +# include +//extern "C" { +// int write (int fd, const char* buf, int num); +// int read (int fd, char* buf, int num); +//} +#endif + + +// BEGIN namespace BOOST +namespace boost { + + +/************************************************************ + * fdostream + * - a stream that writes on a file descriptor + ************************************************************/ + + +class fdoutbuf : public std::streambuf { + protected: + int fd; // file descriptor + public: + // constructor + fdoutbuf (int _fd) : fd(_fd) { + } + protected: + // write one character + virtual int_type overflow (int_type c) { + if (c != EOF) { + char z = c; + if (write (fd, &z, 1) != 1) { + return EOF; + } + } + return c; + } + // write multiple characters + virtual + std::streamsize xsputn (const char* s, + std::streamsize num) { + return write(fd,s,num); + } +}; + +class fdostream : public std::ostream { + protected: + fdoutbuf buf; + public: + fdostream (int fd) : std::ostream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +/************************************************************ + * fdistream + * - a stream that reads on a file descriptor + ************************************************************/ + +class fdinbuf : public std::streambuf { + protected: + int fd; // file descriptor + protected: + /* data buffer: + * - at most, pbSize characters in putback area plus + * - at most, bufSize characters in ordinary read buffer + */ + static const int pbSize = 4; // size of putback area + static const int bufSize = 1024; // size of the data buffer + char buffer[bufSize+pbSize]; // data buffer + + public: + /* constructor + * - initialize file descriptor + * - initialize empty data buffer + * - no putback area + * => force underflow() + */ + fdinbuf (int _fd) : fd(_fd) { + setg (buffer+pbSize, // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize); // end position + } + + protected: + // insert new characters into the buffer + virtual int_type underflow () { +#ifndef _MSC_VER + using std::memmove; +#endif + + // is read position before end of buffer? + if (gptr() < egptr()) { + return traits_type::to_int_type(*gptr()); + } + + /* process size of putback area + * - use number of characters read + * - but at most size of putback area + */ + int numPutback; + numPutback = gptr() - eback(); + if (numPutback > pbSize) { + numPutback = pbSize; + } + + /* copy up to pbSize characters previously read into + * the putback area + */ + memmove (buffer+(pbSize-numPutback), gptr()-numPutback, + numPutback); + + // read at most bufSize new characters + int num; + num = read (fd, buffer+pbSize, bufSize); + if (num <= 0) { + // ERROR or EOF + return EOF; + } + + // reset buffer pointers + setg (buffer+(pbSize-numPutback), // beginning of putback area + buffer+pbSize, // read position + buffer+pbSize+num); // end of buffer + + // return next character + return traits_type::to_int_type(*gptr()); + } +}; + +class fdistream : public std::istream { + protected: + fdinbuf buf; + public: + fdistream (int fd) : std::istream(0), buf(fd) { + rdbuf(&buf); + } +}; + + +} // END namespace boost + +#endif /*BOOST_FDSTREAM_HPP*/ diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index a83314c491..777acb5951 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -7,10 +7,12 @@ #include #include "../../threading/SerialTypes.h" +#include "../fdstream.h" #define MANUAL 0 #define REREAD 1 #define STREAM 2 +#define EXECUTE 3 #include #include @@ -23,6 +25,7 @@ using threading::Field; Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) { file = 0; + in = 0; //keyMap = new map(); @@ -41,9 +44,15 @@ Raw::~Raw() void Raw::DoFinish() { if ( file != 0 ) { - file->close(); - delete(file); + if ( mode != EXECUTE ) { + file->close(); + delete(file); + } else { // mode == EXECUTE + delete(in); + pclose(pfile); + } file = 0; + in = 0; } } @@ -53,15 +62,29 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con mode = arg_mode; mtime = 0; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) && ( mode != EXECUTE ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; } - file = new ifstream(path.c_str()); - if ( !file->is_open() ) { - Error(Fmt("Init: cannot open %s", fname.c_str())); - return false; + if ( mode != EXECUTE ) { + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + in = file; + + } else { // mode == EXECUTE + + pfile = popen(path.c_str(), "r"); + if ( pfile == NULL ) { + Error(Fmt("Could not execute command %s", path.c_str())); + return false; + } + + in = new boost::fdistream(fileno(pfile)); } num_fields = arg_num_fields; @@ -81,23 +104,14 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con Debug(DBG_INPUT, "Raw reader created, will perform first update"); #endif - switch ( mode ) { - case MANUAL: - case REREAD: - case STREAM: - DoUpdate(); - break; - default: - assert(false); - } - + DoUpdate(); return true; } bool Raw::GetLine(string& str) { - while ( getline(*file, str, separator[0]) ) { + while ( getline(*in, str, separator[0]) ) { return true; } @@ -141,6 +155,18 @@ bool Raw::DoUpdate() { return false; } + break; + case EXECUTE: + // re-execute it... + pclose(pfile); + + pfile = popen(fname.c_str(), "r"); + if ( pfile == NULL ) { + Error(Fmt("Could not execute command %s", fname.c_str())); + return false; + } + + in = new boost::fdistream(fileno(pfile)); break; default: assert(false); @@ -171,6 +197,7 @@ bool Raw::DoHeartbeat(double network_time, double current_time) switch ( mode ) { case MANUAL: + case EXECUTE: // yay, we do nothing :) break; case REREAD: diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index ace4e0ee88..55d14d956d 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -31,7 +31,11 @@ private: bool GetLine(string& str); + istream* in; ifstream* file; + + FILE* pfile; + string fname; // Options set from the script-level. diff --git a/src/types.bif b/src/types.bif index 26850bfa93..ebd206c6fa 100644 --- a/src/types.bif +++ b/src/types.bif @@ -186,6 +186,7 @@ enum Mode %{ MANUAL = 0, REREAD = 1, STREAM = 2, + EXECUTE = 3, %} module GLOBAL; diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro new file mode 100644 index 0000000000..85c1415bf3 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="wc input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $mode=Input::EXECUTE]); + Input::remove("input"); +} From 08e1771682da9d90a1deeae3918f3f2960b38772 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 20 Mar 2012 12:07:37 -0700 Subject: [PATCH 085/123] update to execute raw. support reading from commands by adppending | to the filename. support streaming reads from command. Fix something to make rearead work better. (magically happened) --- src/input/fdstream.h | 5 + src/input/readers/Raw.cc | 202 ++-- src/input/readers/Raw.h | 7 +- src/types.bif | 1 - .../out | 145 +++ .../scripts.base.frameworks.input.reread/out | 966 +++++++++++++----- .../out | 128 +++ .../out | 120 +++ .../base/frameworks/input/executeraw.bro | 2 +- .../frameworks/input/executestreamraw.bro | 58 ++ .../base/frameworks/input/rereadraw.bro | 34 + .../base/frameworks/input/streamraw.bro | 56 + 12 files changed, 1364 insertions(+), 360 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out create mode 100644 testing/btest/scripts/base/frameworks/input/executestreamraw.bro create mode 100644 testing/btest/scripts/base/frameworks/input/rereadraw.bro create mode 100644 testing/btest/scripts/base/frameworks/input/streamraw.bro diff --git a/src/input/fdstream.h b/src/input/fdstream.h index 585e03d10b..cda767dd52 100644 --- a/src/input/fdstream.h +++ b/src/input/fdstream.h @@ -35,10 +35,12 @@ #include + // low-level read and write functions #ifdef _MSC_VER # include #else +# include # include //extern "C" { // int write (int fd, const char* buf, int num); @@ -154,6 +156,9 @@ class fdinbuf : public std::streambuf { // read at most bufSize new characters int num; num = read (fd, buffer+pbSize, bufSize); + if ( num == EAGAIN ) { + return 0; + } if (num <= 0) { // ERROR or EOF return EOF; diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 777acb5951..fb9243e713 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -12,11 +12,11 @@ #define MANUAL 0 #define REREAD 1 #define STREAM 2 -#define EXECUTE 3 #include #include #include +#include using namespace input::reader; using threading::Value; @@ -44,52 +44,73 @@ Raw::~Raw() void Raw::DoFinish() { if ( file != 0 ) { - if ( mode != EXECUTE ) { - file->close(); - delete(file); - } else { // mode == EXECUTE - delete(in); - pclose(pfile); - } - file = 0; - in = 0; + Close(); } } +bool Raw::Open() +{ + if ( execute ) { + file = popen(fname.c_str(), "r"); + if ( file == NULL ) { + Error(Fmt("Could not execute command %s", fname.c_str())); + return false; + } + } else { + file = fopen(fname.c_str(), "r"); + if ( file == NULL ) { + Error(Fmt("Init: cannot open %s", fname.c_str())); + return false; + } + } + + in = new boost::fdistream(fileno(file)); + + if ( execute && mode == STREAM ) { + fcntl(fileno(file), F_SETFL, O_NONBLOCK); + } + + return true; +} + +bool Raw::Close() +{ + if ( file == NULL ) { + InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); + return false; + } + + if ( execute ) { + delete(in); + pclose(file); + } else { + delete(in); + fclose(file); + } + + in = NULL; + file = NULL; + + return true; +} + bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) { fname = path; mode = arg_mode; mtime = 0; - - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) && ( mode != EXECUTE ) ) { - Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); - return false; - } + execute = false; + firstrun = true; + bool result; - if ( mode != EXECUTE ) { - - file = new ifstream(path.c_str()); - if ( !file->is_open() ) { - Error(Fmt("Init: cannot open %s", fname.c_str())); - return false; - } - in = file; - - } else { // mode == EXECUTE - - pfile = popen(path.c_str(), "r"); - if ( pfile == NULL ) { - Error(Fmt("Could not execute command %s", path.c_str())); - return false; - } - - in = new boost::fdistream(fileno(pfile)); - } - num_fields = arg_num_fields; fields = arg_fields; + if ( path.length() == 0 ) { + Error("No source path provided"); + return false; + } + if ( arg_num_fields != 1 ) { Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); return false; @@ -100,12 +121,45 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con return false; } + // do Initialization + char last = path[path.length()-1]; + if ( last == '|' ) { + execute = true; + fname = path.substr(0, fname.length() - 1); + + if ( ( mode != MANUAL ) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s in execution mode", mode, fname.c_str())); + return false; + } + + result = Open(); + + } else { + execute = false; + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str())); + return false; + } + + result = Open(); + + } + + if ( result == false ) { + return result; + } + + #ifdef DEBUG Debug(DBG_INPUT, "Raw reader created, will perform first update"); #endif + // after initialization - do update DoUpdate(); +#ifdef DEBUG + Debug(DBG_INPUT, "First update went through"); +#endif return true; } @@ -121,56 +175,45 @@ bool Raw::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool Raw::DoUpdate() { - switch ( mode ) { - case REREAD: - // check if the file has changed - struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { - Error(Fmt("Could not get stat for %s", fname.c_str())); - return false; - } + if ( firstrun ) { + firstrun = false; + } else { + switch ( mode ) { + case REREAD: + // check if the file has changed + struct stat sb; + if ( stat(fname.c_str(), &sb) == -1 ) { + Error(Fmt("Could not get stat for %s", fname.c_str())); + return false; + } - if ( sb.st_mtime <= mtime ) { - // no change - return true; - } + if ( sb.st_mtime <= mtime ) { + // no change + return true; + } - mtime = sb.st_mtime; - // file changed. reread. + mtime = sb.st_mtime; + // file changed. reread. - // fallthrough - case MANUAL: - case STREAM: - - if ( file && file->is_open() ) { - if ( mode == STREAM ) { - file->clear(); // remove end of file evil bits + // fallthrough + case MANUAL: + case STREAM: + Debug(DBG_INPUT, "Updating"); + if ( mode == STREAM && file != NULL && in != NULL ) { + fpurge(file); + in->clear(); // remove end of file evil bits break; } - file->close(); - } - file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { - Error(Fmt("cannot open %s", fname.c_str())); - return false; - } - break; - case EXECUTE: - // re-execute it... - pclose(pfile); - - pfile = popen(fname.c_str(), "r"); - if ( pfile == NULL ) { - Error(Fmt("Could not execute command %s", fname.c_str())); - return false; - } - - in = new boost::fdistream(fileno(pfile)); - break; - default: - assert(false); + Close(); + if ( !Open() ) { + return false; + } + break; + default: + assert(false); + } } string line; @@ -195,9 +238,10 @@ bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); + Debug(DBG_INPUT, "Heartbeat"); + switch ( mode ) { case MANUAL: - case EXECUTE: // yay, we do nothing :) break; case REREAD: diff --git a/src/input/readers/Raw.h b/src/input/readers/Raw.h index 55d14d956d..1cbeff4f83 100644 --- a/src/input/readers/Raw.h +++ b/src/input/readers/Raw.h @@ -28,13 +28,14 @@ protected: private: virtual bool DoHeartbeat(double network_time, double current_time); + bool Open(); + bool Close(); bool GetLine(string& str); istream* in; - ifstream* file; - FILE* pfile; + FILE* file; string fname; @@ -45,6 +46,8 @@ private: string headerline; int mode; + bool execute; + bool firstrun; time_t mtime; diff --git a/src/types.bif b/src/types.bif index ebd206c6fa..26850bfa93 100644 --- a/src/types.bif +++ b/src/types.bif @@ -186,7 +186,6 @@ enum Mode %{ MANUAL = 0, REREAD = 1, STREAM = 2, - EXECUTE = 3, %} module GLOBAL; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out new file mode 100644 index 0000000000..06e28de441 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out @@ -0,0 +1,145 @@ +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +A::try = A::try + 1; +if (9 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 545a1cb781..46a30f387f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -303,81 +303,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ -[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-44] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_NEW @@ -393,93 +318,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ -[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-45] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_NEW @@ -495,105 +333,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============EVENT============ -Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ -[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]], -[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - -}, vc=[10, 20, 30], ve=[]] -}, idx=, val=, want_record=T, ev=line -{ -print A::outfile, ============EVENT============; -print A::outfile, Description; -print A::outfile, A::description; -print A::outfile, Type; -print A::outfile, A::tpe; -print A::outfile, Left; -print A::outfile, A::left; -print A::outfile, Right; -print A::outfile, A::right; -}, pred=anonymous-function -{ -print A::outfile, ============PREDICATE============; -print A::outfile, A::typ; -print A::outfile, A::left; -print A::outfile, A::right; -return (T); -}] -Type -Input::EVENT_NEW -Left -[i=-46] -Right -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============PREDICATE============ Input::EVENT_NEW @@ -609,6 +348,21 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_NEW +[i=-48] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description @@ -636,6 +390,387 @@ AA, BB }, se={ +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-45] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-46] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + }, vc=[10, 20, 30], ve=[]], [-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, @@ -720,21 +855,6 @@ AA, BB }, se={ -}, vc=[10, 20, 30], ve=[]] -============PREDICATE============ -Input::EVENT_NEW -[i=-48] -[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ -2, -4, -1, -3 -}, ss={ -CC, -AA, -BB -}, se={ - }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description @@ -873,4 +993,296 @@ BB }, se={ }, vc=[10, 20, 30], ve=[]], -[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, \ No newline at end of file +[-46] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-47] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-45] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_REMOVED +[i=-43] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-46] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-47] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-45] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-43] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-46] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-44] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-47] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-45] +Left +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-42] +Left +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +==========SERVERS============ +{ +[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +done diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out new file mode 100644 index 0000000000..d85c8f2e83 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out @@ -0,0 +1,128 @@ +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +q3r3057fdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdfs\d +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW + +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +dfsdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +sdf +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::description; +print A::tpe; +print A::s; +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out new file mode 100644 index 0000000000..937acf428e --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out @@ -0,0 +1,120 @@ +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +q3r3057fdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdfs\d +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW + +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +dfsdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +sdf +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +{ +print A::outfile, A::description; +print A::outfile, A::tpe; +print A::outfile, A::s; +if (3 == A::try) +{ +print A::outfile, done; +close(A::outfile); +Input::remove(input); +} + +}] +Input::EVENT_NEW +3rw43wRRERLlL#RWERERERE. diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 85c1415bf3..6fceebf885 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -28,6 +28,6 @@ event line(description: Input::EventDescription, tpe: Input::Event, s: string) { event bro_init() { - Input::add_event([$source="wc input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $mode=Input::EXECUTE]); + Input::add_event([$source="wc input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); Input::remove("input"); } diff --git a/testing/btest/scripts/base/frameworks/input/executestreamraw.bro b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro new file mode 100644 index 0000000000..d97a7b26a0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/executestreamraw.bro @@ -0,0 +1,58 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. + +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + try = try + 1; + + if ( try == 9 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="tail -f ../input.log |", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro new file mode 100644 index 0000000000..33361ad27e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + + +module A; + +type Val: record { + s: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print description; + print tpe; + print s; +} + +event bro_init() +{ + Input::add_event([$source="input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line]); + Input::force_update("input"); + Input::remove("input"); +} diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro new file mode 100644 index 0000000000..cc0afd5ae8 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -0,0 +1,56 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input2.log >> input.log +# @TEST-EXEC: sleep 3 +# @TEST-EXEC: cat input3.log >> input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF +@TEST-END-FILE + +@TEST-START-FILE input2.log +DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF +q3r3057fdf +@TEST-END-FILE + +@TEST-START-FILE input3.log +sdfs\d + +dfsdf +sdf +3rw43wRRERLlL#RWERERERE. +@TEST-END-FILE + +@load frameworks/communication/listen + +module A; + +type Val: record { + s: string; +}; + +global try: count; +global outfile: file; + +event line(description: Input::EventDescription, tpe: Input::Event, s: string) { + print outfile, description; + print outfile, tpe; + print outfile, s; + + if ( try == 3 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line]); +} From d39a389201180c730d6a7d24a2f0c6426ff07fe9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 20 Mar 2012 14:11:59 -0700 Subject: [PATCH 086/123] make optional fields possible for input framework. This do not have to be present in the input file and are marked as &optional in the record description. Those can e.g. be used to create field values on the file in a predicate while reading a file - example: Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } --- src/input/Manager.cc | 10 ++++- src/input/readers/Ascii.cc | 24 ++++++++-- src/input/readers/Ascii.h | 1 + src/threading/SerialTypes.cc | 4 +- src/threading/SerialTypes.h | 5 ++- .../out | 9 ++++ .../base/frameworks/input/optional.bro | 45 +++++++++++++++++++ 7 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.optional/out create mode 100644 testing/btest/scripts/base/frameworks/input/optional.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d0386fbb3f..f6ba6f9f49 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -492,7 +492,11 @@ bool Manager::CreateTableStream(RecordVal* fval) { Unref(pred); if ( valfields > 1 ) { - assert(filter->want_record); + if ( ! filter->want_record ) { + reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); + delete filter; + return false; + } } @@ -631,6 +635,10 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) { + field->optional = true; + } + fields->push_back(field); } } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a04a40e780..20ae79ab19 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -26,6 +26,7 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) @@ -33,10 +34,11 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, cons { position = arg_position; secondary_position = -1; + present = true; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), subtype(arg.subtype) + : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) { position = arg.position; secondary_position = arg.secondary_position; @@ -162,7 +164,15 @@ bool Ascii::ReadHeader(bool useCached) { map::iterator fit = ifields.find(field->name); if ( fit == ifields.end() ) { - Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + if ( field->optional ) { + // we do not really need this field. mark it as not present and always send an undef back. + FieldMapping f(field->name, field->type, field->subtype, -1); + f.present = false; + columnMap.push_back(f); + continue; + } + + Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); return false; } @@ -220,7 +230,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { } else if ( s == "F" ) { val->val.int_val = 0; } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); + Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); return false; } break; @@ -423,6 +433,14 @@ bool Ascii::DoUpdate() { fit != columnMap.end(); fit++ ){ + if ( ! fit->present ) { + // add non-present field + fields[fpos] = new Value((*fit).type, false); + fpos++; + continue; + } + + assert(fit->position >= 0 ); if ( (*fit).position > pos || (*fit).secondary_position > pos ) { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h index 40f92be717..a9b14768fb 100644 --- a/src/input/readers/Ascii.h +++ b/src/input/readers/Ascii.h @@ -19,6 +19,7 @@ struct FieldMapping { int position; // for ports: pos of the second field int secondary_position; + bool present; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/threading/SerialTypes.cc b/src/threading/SerialTypes.cc index 78556e5271..1d7255d695 100644 --- a/src/threading/SerialTypes.cc +++ b/src/threading/SerialTypes.cc @@ -13,7 +13,7 @@ bool Field::Read(SerializationFormat* fmt) int st; bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && - fmt->Read(&t, "type") && fmt->Read(&st, "subtype") ); + fmt->Read(&t, "type") && fmt->Read(&st, "subtype") && fmt->Read(&optional, "optional")); type = (TypeTag) t; subtype = (TypeTag) st; @@ -23,7 +23,7 @@ bool Field::Read(SerializationFormat* fmt) bool Field::Write(SerializationFormat* fmt) const { return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && - fmt->Write((int)subtype, "subtype")); + fmt->Write((int)subtype, "subtype"), fmt->Write(optional, "optional")); } Value::~Value() diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index ac34f3e476..bee84f2b54 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -24,17 +24,18 @@ struct Field { string secondary_name; TypeTag type; //! Type of the field. TypeTag subtype; //! Inner type for sets. + bool optional; //! needed by input framework. Is the field optional or does it have to be present in the input data /** * Constructor. */ - Field() { subtype = TYPE_VOID; } + Field() { subtype = TYPE_VOID; optional = false; } /** * Copy constructor. */ Field(const Field& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), type(other.type), subtype(other.subtype), optional(other.optional) { } /** * Unserializes a field. diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.optional/out b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out new file mode 100644 index 0000000000..7a304fc918 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.optional/out @@ -0,0 +1,9 @@ +{ +[2] = [b=T, notb=F], +[4] = [b=F, notb=T], +[6] = [b=F, notb=T], +[7] = [b=T, notb=F], +[1] = [b=T, notb=F], +[5] = [b=F, notb=T], +[3] = [b=F, notb=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro new file mode 100644 index 0000000000..c354f7c3ab --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -0,0 +1,45 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + notb: bool &optional; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} From 51ddc9f572a1ab8524ecc27ec5ee6de494d2fc56 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 21 Mar 2012 15:51:21 -0700 Subject: [PATCH 087/123] fix bug that crashed input framework when creating already existing stream (tried to free not yet alloccated data) + write twotables test --- src/input/Manager.cc | 48 ++- .../out | 349 ++++++++++++++++++ .../base/frameworks/input/twotables.bro | 113 ++++++ 3 files changed, 500 insertions(+), 10 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twotables/out create mode 100644 testing/btest/scripts/base/frameworks/input/twotables.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index f6ba6f9f49..f62e87d937 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -43,14 +43,24 @@ public: RecordVal* description; + Filter(); virtual ~Filter(); }; -Manager::Filter::~Filter() { - Unref(type); - Unref(description); +Manager::Filter::Filter() { + type = 0; + reader = 0; + description = 0; +} - delete(reader); +Manager::Filter::~Filter() { + if ( type ) + Unref(type); + if ( description ) + Unref(description); + + if ( reader ) + delete(reader); } class Manager::TableFilter: public Manager::Filter { @@ -85,28 +95,46 @@ public: bool want_record; EventFilter(); + ~EventFilter(); }; -Manager::TableFilter::TableFilter() { +Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { filter_type = TABLE_FILTER; tab = 0; itype = 0; rtype = 0; + + currDict = 0; + lastDict = 0; + + pred = 0; } -Manager::EventFilter::EventFilter() { +Manager::EventFilter::EventFilter() : Manager::Filter::Filter() { + fields = 0; filter_type = EVENT_FILTER; } +Manager::EventFilter::~EventFilter() { + if ( fields ) { + Unref(fields); + } +} + Manager::TableFilter::~TableFilter() { - Unref(tab); - Unref(itype); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); if ( rtype ) // can be 0 for sets Unref(rtype); - delete currDict; - delete lastDict; + if ( currDict != 0 ) + delete currDict; + + if ( lastDict != 0 ) + delete lastDict; } struct ReaderDefinition { diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out new file mode 100644 index 0000000000..a61a4a2993 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -0,0 +1,349 @@ +============PREDICATE============ +Input::EVENT_NEW +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE 2============ +Input::EVENT_NEW +[i=-43] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-42] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============EVENT============ +Description +[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh2, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE 2============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-43] +Right +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} +============PREDICATE============ +Input::EVENT_NEW +[i=-44] +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============PREDICATE============ +Input::EVENT_REMOVED +[i=-42] +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +}, idx=, val=, want_record=T, ev=line +{ +print A::outfile, ============EVENT============; +print A::outfile, Description; +print A::outfile, A::description; +print A::outfile, Type; +print A::outfile, A::tpe; +print A::outfile, Left; +print A::outfile, A::left; +print A::outfile, Right; +print A::outfile, A::right; +}, pred=anonymous-function +{ +print A::outfile, ============PREDICATE============; +print A::outfile, A::typ; +print A::outfile, A::left; +print A::outfile, A::right; +return (T); +}] +Type +Input::EVENT_NEW +Left +[i=-44] +Right +[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +============EVENT============ +Description +Input::EVENT_REMOVED +Type +[i=-42] +Left +[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +Right +==========SERVERS============ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-44] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro new file mode 100644 index 0000000000..6f18e0e939 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -0,0 +1,113 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: btest-bg-wait -k 2 +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); + +global outfile: file; + +global try: count; + +event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) { + print outfile, "============EVENT============"; + print outfile, "Description"; + print outfile, description; + print outfile, "Type"; + print outfile, tpe; + print outfile, "Left"; + print outfile, left; + print outfile, "Right"; + print outfile, right; +} + +event bro_init() +{ + outfile = open ("../out"); + try = 0; + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); + Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line, + $pred(typ: Input::Event, left: Idx, right: Val) = { + print outfile, "============PREDICATE 2============"; + print outfile, typ; + print outfile, left; + print outfile, right; + return T; + } + ]); +} + + +event Input::update_finished(name: string, source: string) { + print outfile, "==========SERVERS============"; + print outfile, servers; + + try = try + 1; + if ( try == 5 ) { + print outfile, "done"; + close(outfile); + Input::remove("input"); + } +} From 0db89bed3ba6248354315f6d3f1b77ab84e65a41 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 10:33:49 -0700 Subject: [PATCH 088/123] fix crash when deleting data from source where there are no events or predicates... (that happens when all testcases are too complicated and use all features..) --- src/input/Manager.cc | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index f62e87d937..c1fa060b2b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -915,18 +915,20 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { ListVal * idx = 0; Val *val = 0; + + Val* predidx = 0; + EnumVal* ev = 0; + int startpos = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); val = filter->tab->Lookup(idx); assert(val != 0); + predidx = ListValToRecordVal(idx, filter->itype, &startpos); + ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); } - int startpos = 0; - Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - if ( filter->pred ) { // ask predicate, if we want to expire this element... @@ -953,8 +955,10 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { SendEvent(filter->event, 3, ev, predidx, val); } - Unref(predidx); - Unref(ev); + if ( predidx ) // if we have a filter or an event... + Unref(predidx); + if ( ev ) + Unref(ev); filter->tab->Delete(ih->idxkey); filter->lastDict->Remove(lastDictIdxKey); // deletex in next line @@ -1321,6 +1325,7 @@ RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, RecordVal* rec = new RecordVal(request_type->AsRecordType()); + assert(list != 0); int maxpos = list->Length(); for ( int i = 0; i < request_type->NumFields(); i++ ) { From 14c6c4004289e715101dcddc1a42955ac6f86e8c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 10:59:36 -0700 Subject: [PATCH 089/123] fix crash when all value fields of imported table are uninitialized. --- .../out | 4 ++ .../base/frameworks/input/emptyvals.bro | 37 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out create mode 100644 testing/btest/scripts/base/frameworks/input/emptyvals.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out new file mode 100644 index 0000000000..f75248cf97 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.emptyvals/out @@ -0,0 +1,4 @@ +{ +[2] = [b=], +[1] = [b=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro new file mode 100644 index 0000000000..77659d13ec --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +##types bool int +T 1 +- 2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global servers: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::remove("ssh"); +} + +event Input::update_finished(name: string, source:string) { + print servers; +} From 5f5209fcfb12cf96dbb9bb027a93133657f0d9ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 11:00:51 -0700 Subject: [PATCH 090/123] ...forgotten file. --- src/input/Manager.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index c1fa060b2b..32b60f05f2 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -760,8 +760,13 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { hash_t valhash = 0; if ( filter->num_val_fields > 0 ) { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); - valhash = valhashkey->Hash(); - delete(valhashkey); + if ( valhashkey == 0 ) { + // empty line. index, but no values. + // hence we also have no hash value... + } else { + valhash = valhashkey->Hash(); + delete(valhashkey); + } } InputHash *h = filter->lastDict->Lookup(idxhash); From 7e4cbbc0735a13343062c9bc18e01f8d8d7b17fb Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 12:45:11 -0700 Subject: [PATCH 091/123] remove forgotten debug statements --- src/input/readers/Raw.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index fb9243e713..f416fbe94a 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -198,7 +198,6 @@ bool Raw::DoUpdate() { // fallthrough case MANUAL: case STREAM: - Debug(DBG_INPUT, "Updating"); if ( mode == STREAM && file != NULL && in != NULL ) { fpurge(file); in->clear(); // remove end of file evil bits @@ -238,8 +237,6 @@ bool Raw::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - Debug(DBG_INPUT, "Heartbeat"); - switch ( mode ) { case MANUAL: // yay, we do nothing :) From 6c4a40f176c74735d33bb8afe66ac429d9f416ed Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 13:09:53 -0700 Subject: [PATCH 092/123] missing include on linux --- src/input/Manager.cc | 2 +- src/input/readers/Raw.cc | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 32b60f05f2..d8bf63505b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1540,7 +1540,7 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); break; case IPv6: - length += sizeof(val->val.addr_val.in.in6); + length = sizeof(val->val.addr_val.in.in6); memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); break; default: diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f416fbe94a..cba0a29f3e 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -17,6 +17,7 @@ #include #include #include +#include using namespace input::reader; using threading::Value; @@ -199,7 +200,7 @@ bool Raw::DoUpdate() { case MANUAL: case STREAM: if ( mode == STREAM && file != NULL && in != NULL ) { - fpurge(file); + //fpurge(file); in->clear(); // remove end of file evil bits break; } From f73de0bc8c531f9b2ec0d06b9c63054dc4cf3589 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 15:11:42 -0700 Subject: [PATCH 093/123] fix small memory leak (field description given to readers was never freed). --- src/input/ReaderBackend.cc | 13 +++++++++++++ src/input/ReaderBackend.h | 3 +++ src/input/readers/Ascii.cc | 1 - 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 0a6ff37dc2..ce79ecfd39 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -169,6 +169,9 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, source = arg_source; SetName("InputReader/"+source); + num_fields = arg_num_fields; + fields = arg_fields; + // disable if DoInit returns error. int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); @@ -188,6 +191,16 @@ void ReaderBackend::Finish() disabled = true; DisableFrontend(); SendOut(new ReaderFinishedMessage(frontend)); + + if ( fields != 0 ) { + + for ( unsigned int i = 0; i < num_fields; i++ ) { + delete(fields[i]); + } + + delete[] (fields); + fields = 0; + } } bool ReaderBackend::Update() diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 28fd99f2b9..5742f72368 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -228,6 +228,9 @@ private: char* buf; unsigned int buf_len; bool autostart; + + unsigned int num_fields; + const threading::Field* const * fields; // raw mapping }; } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 20ae79ab19..553a4ada81 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -73,7 +73,6 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) Ascii::~Ascii() { DoFinish(); - } void Ascii::DoFinish() From 94d439b0cba75fee93a3b08bb3b57041e694b46e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 17:17:41 -0700 Subject: [PATCH 094/123] enable predicate modification of index of value which is currently being added/removed Todo: test if this works for removal ( I think it should ). --- src/input/Manager.cc | 45 +++++++++++++++-- src/input/Manager.h | 1 + .../out | 4 ++ .../base/frameworks/input/predicatemodify.bro | 50 +++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicatemodify.bro diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d8bf63505b..2201c69995 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -692,6 +692,31 @@ bool Manager::ForceUpdate(const string &name) return true; // update is async :( } + +Val* Manager::RecordValToIndexVal(RecordVal *r) { + Val* idxval; + + RecordType *type = r->Type()->AsRecordType(); + + int num_fields = type->NumFields(); + + if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) { + idxval = r->Lookup(0); + } else { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < num_fields; j++ ) { + Val* rval = r->Lookup(j); + assert(rval != 0); + l->Append(r->LookupWithDefault(j)); + } + idxval = l; + } + + + return idxval; +} + + Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { Val* idxval; int position = 0; @@ -788,8 +813,8 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } - Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; + RecordVal* predidx = 0; int position = filter->num_idx_fields; if ( filter->num_val_fields == 0 ) { @@ -806,8 +831,10 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { if ( filter->pred ) { EnumVal* ev; //Ref(idxval); - int startpos = 0; - Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); + int startpos = 0; + //Val* predidx = ListValToRecordVal(idxval->AsListVal(), filter->itype, &startpos); + predidx = ValueToRecordVal(vals, filter->itype, &startpos); + //ValueToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -818,13 +845,14 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool result; if ( filter->num_val_fields > 0 ) { // we have values - result = CallPred(filter->pred, 3, ev, predidx, valval); + result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval); } else { // no values - result = CallPred(filter->pred, 2, ev, predidx); + result = CallPred(filter->pred, 2, ev, predidx->Ref()); } if ( result == false ) { + Unref(predidx); if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... delete(filter->currDict->RemoveEntry(idxhash)); @@ -839,6 +867,13 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } + Val* idxval; + if ( predidx != 0 ) { + idxval = RecordValToIndexVal(predidx); + Unref(predidx); + } else { + idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); + } Val* oldval = 0; if ( updated == true ) { assert(filter->num_val_fields > 0); diff --git a/src/input/Manager.h b/src/input/Manager.h index 71169c4bc2..c6dd40bd95 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -158,6 +158,7 @@ private: // Converts a threading::value to a record type. mostly used by ValueToVal RecordVal* ValueToRecordVal(const threading::Value* const *vals, RecordType *request_type, int* position); + Val* RecordValToIndexVal(RecordVal *r); // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out new file mode 100644 index 0000000000..c648e63710 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodify/out @@ -0,0 +1,4 @@ +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=T, s=testmodified] +} diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro new file mode 100644 index 0000000000..c3198d8483 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -0,0 +1,50 @@ +# +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; + ss: string; +}; + +type Val: record { + b: bool; + s: string; +}; + +global servers: table[int, string] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::add_table([$source="input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( left$i == 1 ) { + right$s = "testmodified"; + } + + if ( left$i == 2 ) { + left$ss = "idxmodified"; + } + return T; + } + ]); + Input::remove("input"); +} + +event Input::update_finished(name: string, source: string) { + print servers; +} From 03116d779eef498d2fcc0ab7e2822a94cbfb43f4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 22 Mar 2012 18:08:59 -0700 Subject: [PATCH 095/123] one unref to many ... apparently --- src/input/Manager.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 2201c69995..8eaca07e78 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -705,8 +705,8 @@ Val* Manager::RecordValToIndexVal(RecordVal *r) { } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < num_fields; j++ ) { - Val* rval = r->Lookup(j); - assert(rval != 0); + //Val* rval = r->Lookup(j); + //assert(rval != 0); l->Append(r->LookupWithDefault(j)); } idxval = l; @@ -870,7 +870,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { Val* idxval; if ( predidx != 0 ) { idxval = RecordValToIndexVal(predidx); - Unref(predidx); + // I think there is an unref missing here. But if I insert is, it crashes :) } else { idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); } From 315948dbc8fc3b34a0add6a46686814474ff2fb7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 23 Mar 2012 11:40:59 -0700 Subject: [PATCH 096/123] add test for update functionality of tables where a predicate modifies values / indexes. Seems to work fine for all cases... --- .../out | 23 ++++ .../input/predicatemodifyandreread.bro | 107 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out new file mode 100644 index 0000000000..0adccc1856 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicatemodifyandreread/out @@ -0,0 +1,23 @@ +Update_finished for input, try 1 +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=T, s=testmodified] +} +Update_finished for input, try 2 +{ +[2, idxmodified] = [b=T, s=test2], +[1, idx1] = [b=F, s=testmodified] +} +Update_finished for input, try 3 +{ +[2, idxmodified] = [b=F, s=test2], +[1, idx1] = [b=F, s=testmodified] +} +Update_finished for input, try 4 +{ +[2, idxmodified] = [b=F, s=test2] +} +Update_finished for input, try 5 +{ +[1, idx1] = [b=T, s=testmodified] +} diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro new file mode 100644 index 0000000000..1606ff6a27 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -0,0 +1,107 @@ +# +# @TEST-EXEC: cp input1.log input.log +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input2.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input3.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input4.log input.log +# @TEST-EXEC: sleep 2 +# @TEST-EXEC: cp input5.log input.log +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff out +# + +@TEST-START-FILE input1.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input2.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 F test1 idx1 +2 T test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input3.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 F test1 idx1 +2 F test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input4.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +2 F test2 idx2 +@TEST-END-FILE + +@TEST-START-FILE input5.log +#separator \x09 +#path ssh +#fields i b s ss +#types int bool string string +1 T test1 idx1 +@TEST-END-FILE + +@load frameworks/communication/listen + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; + ss: string; +}; + +type Val: record { + b: bool; + s: string; +}; + +global servers: table[int, string] of Val = table(); +global outfile: file; +global try: count; + +event bro_init() +{ + try = 0; + outfile = open ("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $mode=Input::REREAD, + $pred(typ: Input::Event, left: Idx, right: Val) = { + if ( left$i == 1 ) { + right$s = "testmodified"; + } + + if ( left$i == 2 ) { + left$ss = "idxmodified"; + } + return T; + } + ]); +} + +event Input::update_finished(name: string, source: string) { + try = try + 1; + print outfile, fmt("Update_finished for %s, try %d", name, try); + print outfile, servers; + + if ( try == 5 ) { + close (outfile); + Input::remove("input"); + } +} From 872ad195f789a155c3dd79a4f3786388203f3dce Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 23 Mar 2012 12:30:54 -0700 Subject: [PATCH 097/123] prevent several remove operations for the same thread to be queued and output errors in that case. --- src/input/Manager.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 8eaca07e78..6d97c2f50b 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -33,6 +33,7 @@ class Manager::Filter { public: string name; string source; + bool removed; int mode; @@ -51,6 +52,7 @@ Manager::Filter::Filter() { type = 0; reader = 0; description = 0; + removed = false; } Manager::Filter::~Filter() { @@ -597,6 +599,13 @@ bool Manager::RemoveStream(const string &name) { return false; // not found } + if ( i->removed ) { + reporter->Error("Stream %s is already queued for removal. Ignoring", name.c_str()); + return false; + } + + i->removed = true; + i->reader->Finish(); #ifdef DEBUG From 9732859d44ea66098f541fd879c0781b8362b718 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 12:20:39 -0700 Subject: [PATCH 098/123] add first simple benchmark reader (it simply spews random data, amount of lines specified in source). --- src/CMakeLists.txt | 2 +- src/input/Manager.cc | 9 +- src/input/readers/Ascii.cc | 10 +- src/input/readers/Benchmark.cc | 198 +++++++++++++++++++++++++++++++++ src/input/readers/Benchmark.h | 47 ++++++++ src/types.bif | 1 + 6 files changed, 256 insertions(+), 11 deletions(-) create mode 100644 src/input/readers/Benchmark.cc create mode 100644 src/input/readers/Benchmark.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9ec76f8d2..9b075decd5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -425,7 +425,7 @@ set(bro_SRCS input/ReaderFrontend.cc input/readers/Ascii.cc input/readers/Raw.cc - + input/readers/Benchmark.cc ${dns_SRCS} ${openssl_SRCS} diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6d97c2f50b..f8ad493e11 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -7,6 +7,7 @@ #include "ReaderBackend.h" #include "readers/Ascii.h" #include "readers/Raw.h" +#include "readers/Benchmark.h" #include "Event.h" #include "EventHandler.h" @@ -149,6 +150,7 @@ struct ReaderDefinition { ReaderDefinition input_readers[] = { { BifEnum::Input::READER_ASCII, "Ascii", 0, reader::Ascii::Instantiate }, { BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate }, + { BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate }, // End marker { BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 } @@ -600,7 +602,7 @@ bool Manager::RemoveStream(const string &name) { } if ( i->removed ) { - reporter->Error("Stream %s is already queued for removal. Ignoring", name.c_str()); + reporter->Error("Stream %s is already queued for removal. Ignoring remove.", name.c_str()); return false; } @@ -690,6 +692,11 @@ bool Manager::ForceUpdate(const string &name) reporter->Error("Stream %s not found", name.c_str()); return false; } + + if ( i->removed ) { + reporter->Error("Stream %s is already queued for removal. Ignoring force update.", name.c_str()); + return false; + } i->reader->Update(); diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index 553a4ada81..17391afe73 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -110,15 +110,7 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c return false; } - switch ( mode ) { - case MANUAL: - case REREAD: - case STREAM: - DoUpdate(); - break; - default: - assert(false); - } + DoUpdate(); return true; } diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc new file mode 100644 index 0000000000..b48074c146 --- /dev/null +++ b/src/input/readers/Benchmark.cc @@ -0,0 +1,198 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Benchmark.h" +#include "NetVar.h" + +#include "../../threading/SerialTypes.h" + +#define MANUAL 0 +#define REREAD 1 +#define STREAM 2 + +#include +#include +#include + +using namespace input::reader; +using threading::Value; +using threading::Field; + + + +Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) +{ +} + +Benchmark::~Benchmark() +{ + DoFinish(); +} + +void Benchmark::DoFinish() +{ +} + +bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) +{ + mode = arg_mode; + + num_fields = arg_num_fields; + fields = arg_fields; + num_lines = atoi(path.c_str()); + + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); + return false; + } + + DoUpdate(); + + return true; +} + +string Benchmark::RandomString(const int len) { + string s; + + s.reserve(len); + + static const char values[] = + "0123456789!@#$%^&*()-_=+{}[]\\|" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz"; + + for (int i = 0; i < len; ++i) { + s[i] = values[rand() / (RAND_MAX / sizeof(values))]; + } + + return s; +} + +// read the entire file and send appropriate thingies back to InputMgr +bool Benchmark::DoUpdate() { + for ( int i = 0; i < num_lines; i++ ) { + Value** field = new Value*[num_fields]; + for (unsigned int j = 0; j < num_fields; j++ ) { + field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); + } + SendEntry(field); + } + + EndCurrentSend(); + + return true; +} + +threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { + Value* val = new Value(type, true); + + // basically construct something random from the fields that we want. + + switch ( type ) { + case TYPE_ENUM: + assert(false); // no enums, please. + case TYPE_STRING: + val->val.string_val = new string(RandomString(10)); + break; + + case TYPE_BOOL: + val->val.int_val = 1; // we never lie. + break; + + case TYPE_INT: + val->val.int_val = rand(); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = random(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + val->val.uint_val = rand(); + break; + + case TYPE_PORT: + val->val.port_val.port = rand() / (RAND_MAX / 60000); + val->val.port_val.proto = TRANSPORT_UNKNOWN; + break; + + case TYPE_SUBNET: { + val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); + val->val.subnet_val.length = 16; + } + break; + + case TYPE_ADDR: + val->val.addr_val = StringToAddr("192.168.17.1"); + break; + + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { + // how many entries do we have... + unsigned int length = rand() / (RAND_MAX / 15); + + Value** lvals = new Value* [length]; + + if ( type == TYPE_TABLE ) { + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } else if ( type == TYPE_VECTOR ) { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } else { + assert(false); + } + + if ( length == 0 ) + break; //empty + + for ( unsigned int pos = 0; pos < length; pos++ ) { + + Value* newval = EntryToVal(subtype, TYPE_ENUM); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d", type)); + return 0; + } + + return val; + +} + + +bool Benchmark::DoHeartbeat(double network_time, double current_time) +{ + ReaderBackend::DoHeartbeat(network_time, current_time); + + switch ( mode ) { + case MANUAL: + // yay, we do nothing :) + break; + case REREAD: + case STREAM: + Update(); // call update and not DoUpdate, because update actually checks disabled. + break; + default: + assert(false); + } + + return true; +} + diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h new file mode 100644 index 0000000000..5a82c5b726 --- /dev/null +++ b/src/input/readers/Benchmark.h @@ -0,0 +1,47 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUT_READERS_BENCHMARK_H +#define INPUT_READERS_BENCHMARK_H + + +#include "../ReaderBackend.h" + +namespace input { namespace reader { + +class Benchmark : public ReaderBackend { +public: + Benchmark(ReaderFrontend* frontend); + ~Benchmark(); + + static ReaderBackend* Instantiate(ReaderFrontend* frontend) { return new Benchmark(frontend); } + +protected: + + virtual bool DoInit(string path, int mode, int arg_num_fields, const threading::Field* const* fields); + + virtual void DoFinish(); + + virtual bool DoUpdate(); + +private: + + virtual bool DoHeartbeat(double network_time, double current_time); + + unsigned int num_fields; + + const threading::Field* const * fields; // raw mapping + + threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); + + int mode; + int num_lines; + + string RandomString(const int len); + +}; + + +} +} + +#endif /* INPUT_READERS_BENCHMARK_H */ diff --git a/src/types.bif b/src/types.bif index 26850bfa93..682170e3a6 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,6 +174,7 @@ enum Reader %{ READER_DEFAULT, READER_ASCII, READER_RAW, + READER_BENCHMARK, %} enum Event %{ From 016a2540a5afc867bbfb68dd92e5dad667bf920d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 12:41:59 -0700 Subject: [PATCH 099/123] ...and spread out streaming reads over time. --- src/input/readers/Benchmark.cc | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index b48074c146..a4cf5f6818 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -51,9 +51,7 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel } string Benchmark::RandomString(const int len) { - string s; - - s.reserve(len); + string s(len, ' '); static const char values[] = "0123456789!@#$%^&*()-_=+{}[]\\|" @@ -74,10 +72,19 @@ bool Benchmark::DoUpdate() { for (unsigned int j = 0; j < num_fields; j++ ) { field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); } - SendEntry(field); + + if ( mode == STREAM ) { + // do not do tracking, spread out elements over the second that we have... + Put(field); + usleep(900000/num_lines); + } else { + SendEntry(field); + } } - EndCurrentSend(); + //if ( mode != STREAM ) { // well, does not really make sense in the streaming sense - but I like getting the event. + EndCurrentSend(); + //} return true; } From 28f3fa01444b0c5595bafc6585320cdd11168b0f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 13:52:58 -0700 Subject: [PATCH 100/123] make time types always return current time for benchmark reader --- src/input/readers/Benchmark.cc | 13 ++++++++++++- src/input/readers/Benchmark.h | 2 ++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index a4cf5f6818..07ee7eb9bc 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -65,6 +65,14 @@ string Benchmark::RandomString(const int len) { return s; } +double Benchmark::CurrTime() { + struct timeval tv; + assert ( gettimeofday(&tv, 0) >= 0 ); + + return double(tv.tv_sec) + double(tv.tv_usec) / 1e6; +} + + // read the entire file and send appropriate thingies back to InputMgr bool Benchmark::DoUpdate() { for ( int i = 0; i < num_lines; i++ ) { @@ -109,8 +117,11 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { val->val.int_val = rand(); break; - case TYPE_DOUBLE: case TYPE_TIME: + val->val.double_val = CurrTime(); + break; + + case TYPE_DOUBLE: case TYPE_INTERVAL: val->val.double_val = random(); break; diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index 5a82c5b726..e8de4ac773 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -29,6 +29,8 @@ private: unsigned int num_fields; + double CurrTime(); + const threading::Field* const * fields; // raw mapping threading::Value* EntryToVal(TypeTag Type, TypeTag subtype); From 6a60f484f9faa3925cfc297e38f1f06561f41607 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 29 Mar 2012 09:03:33 -0700 Subject: [PATCH 101/123] make heart beat interval for threading configureable from scripting layer --- scripts/base/init-bare.bro | 8 ++++++++ src/const.bif | 1 + src/threading/Manager.cc | 6 +++++- src/threading/Manager.h | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 9f4e0355f0..77f90cae5f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1484,6 +1484,14 @@ export { }; } # end export +module Threading; + +export { + ## The heart beat interval used by the threading framework. + ## Changing this should usually not be neccessary and will break several tests. + const heart_beat_interval = 1.0 &redef; +} + module GLOBAL; ## An NTP message. diff --git a/src/const.bif b/src/const.bif index bc960caeb6..aadb1e1ab7 100644 --- a/src/const.bif +++ b/src/const.bif @@ -12,3 +12,4 @@ const NFS3::return_data: bool; const NFS3::return_data_max: count; const NFS3::return_data_first_only: bool; +const Threading::heart_beat_interval: double; diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index d008d2e5e8..8546ca3948 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -1,5 +1,6 @@ #include "Manager.h" +#include "NetVar.h" using namespace threading; @@ -11,6 +12,9 @@ Manager::Manager() next_beat = 0; terminating = false; idle = false; + + heart_beat_interval = double(BifConst::Threading::heart_beat_interval); + DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval); } Manager::~Manager() @@ -73,7 +77,7 @@ void Manager::GetFds(int* read, int* write, int* except) double Manager::NextTimestamp(double* network_time) { if ( ::network_time && ! next_beat ) - next_beat = ::network_time + HEART_BEAT_INTERVAL; + next_beat = ::network_time + heart_beat_interval; // fprintf(stderr, "N %.6f %.6f did_process=%d next_next=%.6f\n", ::network_time, timer_mgr->Time(), (int)did_process, next_beat); diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 7d9ba766d4..29729f6a7a 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -120,7 +120,7 @@ protected: virtual const char* Tag() { return "threading::Manager"; } private: - static const int HEART_BEAT_INTERVAL = 1; + int heart_beat_interval; typedef std::list all_thread_list; all_thread_list all_threads; From ead30e423d0316e1e2e05ae5334f7771d1582f5c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 30 Mar 2012 08:40:38 -0700 Subject: [PATCH 102/123] change type of heart_beat_interval to interval (makes much more sese) --- scripts/base/init-bare.bro | 2 +- src/const.bif | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 3a5b2023dd..4637580337 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1489,7 +1489,7 @@ module Threading; export { ## The heart beat interval used by the threading framework. ## Changing this should usually not be neccessary and will break several tests. - const heart_beat_interval = 1.0 &redef; + const heart_beat_interval = 1.0 secs &redef; } module GLOBAL; diff --git a/src/const.bif b/src/const.bif index aadb1e1ab7..f9e5f61644 100644 --- a/src/const.bif +++ b/src/const.bif @@ -12,4 +12,4 @@ const NFS3::return_data: bool; const NFS3::return_data_max: count; const NFS3::return_data_first_only: bool; -const Threading::heart_beat_interval: double; +const Threading::heart_beat_interval: interval; From 355b85fcd7fc638ad54cf8b8d3614fe3c6ecc9e5 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 30 Mar 2012 09:08:08 -0700 Subject: [PATCH 103/123] most of the stuff we should need for benchmarking. next: search memory leaks, after 1.5million simulated inputs we are leaking about 1Gb of ram... --- scripts/base/frameworks/input/__load__.bro | 1 + .../base/frameworks/input/readers/benchmark.bro | 8 ++++++++ src/input.bif | 3 +++ src/input/ReaderBackend.cc | 7 ++++++- src/input/readers/Benchmark.cc | 15 ++++++++++++++- src/input/readers/Benchmark.h | 2 ++ 6 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/benchmark.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index b41fe5e95f..0e7d8ffb73 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1,4 +1,5 @@ @load ./main @load ./readers/ascii @load ./readers/raw +@load ./readers/benchmark diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro new file mode 100644 index 0000000000..3293201cea --- /dev/null +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -0,0 +1,8 @@ +##! Interface for the ascii input reader. + +module InputBenchmark; + +export { + ## multiplication factor for each second + const factor = 1 &redef; +} diff --git a/src/input.bif b/src/input.bif index 1157b7b62b..e4ecf4d020 100644 --- a/src/input.bif +++ b/src/input.bif @@ -45,3 +45,6 @@ const unset_field: string; module InputRaw; const record_separator: string; + +module InputBenchmark; +const factor: count; diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index ce79ecfd39..f0b4f8e7e9 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -58,7 +58,12 @@ public: name(name), num_vals(num_vals), val(val) {} virtual bool Process() { - return input_mgr->SendEvent(name, num_vals, val); + bool success = input_mgr->SendEvent(name, num_vals, val); + + if ( !success ) + reporter->Error("SendEvent for event %s failed", name.c_str()); + + return true; // we do not want to die if sendEvent fails because the event did not return. } private: diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 07ee7eb9bc..de77ba1afa 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -21,6 +21,7 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) { + multiplication_factor = int(BifConst::InputBenchmark::factor); } Benchmark::~Benchmark() @@ -198,13 +199,25 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { bool Benchmark::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - + num_lines = num_lines*multiplication_factor; + switch ( mode ) { case MANUAL: // yay, we do nothing :) break; case REREAD: case STREAM: + if ( multiplication_factor != 1 ) { + // we have to document at what time we changed the factor to what value. + Value** v = new Value*[2]; + v[0] = new Value(TYPE_COUNT, true); + v[0]->val.uint_val = num_lines; + v[1] = new Value(TYPE_TIME, true); + v[1]->val.double_val = CurrTime(); + + SendEvent("lines_changed", 2, v); + } + Update(); // call update and not DoUpdate, because update actually checks disabled. break; default: diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index e8de4ac773..e0d3f124af 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -38,6 +38,8 @@ private: int mode; int num_lines; + int multiplication_factor; + string RandomString(const int len); }; From 3405cbdfbd5eb9c00dc4f782e9cc875da3c26d8f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 26 Mar 2012 19:53:01 -0700 Subject: [PATCH 104/123] Introducing - the check if a thread queue might have data. Without locks. Who needs those anyways. --- src/threading/Manager.cc | 6 ++++++ src/threading/MsgThread.h | 6 ++++++ src/threading/Queue.h | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index db86caa26f..6a539861be 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -82,6 +82,12 @@ double Manager::NextTimestamp(double* network_time) // If we had something to process last time (or out heartbeat // is due), we want to check for more asap. return timer_mgr->Time(); + + for ( msg_thread_list::iterator i = msg_threads.begin(); i != msg_threads.end(); i++ ) + { + if ( (*i)->MightHaveOut() ) + return timer_mgr->Time(); + } return -1.0; } diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 28c7690dfa..4220230a71 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -261,6 +261,12 @@ private: */ bool HasOut() { return queue_out.Ready(); } + /** + * Returns true if there might be at least one message pending for the main + * thread. + */ + bool MightHaveOut() { return queue_out.MaybeReady(); } + Queue queue_in; Queue queue_out; diff --git a/src/threading/Queue.h b/src/threading/Queue.h index a25f897d23..64d6e7cd93 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -53,6 +53,11 @@ public: */ bool Ready(); + /** + * Returns true if the next Get() operation might succeed. + */ + bool MaybeReady() { return ( ( read_ptr - write_ptr) != 0 ); } + /** * Returns the number of queued items not yet retrieved. */ From 579a10d060241f99275e72505230eb642ee2cc47 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 27 Mar 2012 09:18:01 -0700 Subject: [PATCH 105/123] make benchmark reader more configureable --- .../frameworks/input/readers/benchmark.bro | 8 ++++- src/input.bif | 4 ++- src/input/readers/Benchmark.cc | 30 +++++++++++++++---- src/input/readers/Benchmark.h | 5 +++- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index 3293201cea..c6a6e88fca 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -4,5 +4,11 @@ module InputBenchmark; export { ## multiplication factor for each second - const factor = 1 &redef; + const factor = 1.0 &redef; + + ## spread factor between lines + const spread = 0 &redef; + + ## spreading where usleep = 1000000 / autospread * num_lines + const autospread = 0.0 &redef; } diff --git a/src/input.bif b/src/input.bif index e4ecf4d020..059a7ec8bf 100644 --- a/src/input.bif +++ b/src/input.bif @@ -47,4 +47,6 @@ module InputRaw; const record_separator: string; module InputBenchmark; -const factor: count; +const factor: double; +const spread: count; +const autospread: double; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index de77ba1afa..d8de8c2538 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -21,7 +21,11 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) { - multiplication_factor = int(BifConst::InputBenchmark::factor); + multiplication_factor = double(BifConst::InputBenchmark::factor); + autospread = double(BifConst::InputBenchmark::autospread); + spread = int(BifConst::InputBenchmark::spread); + autospread_time = 0; + } Benchmark::~Benchmark() @@ -40,6 +44,9 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel num_fields = arg_num_fields; fields = arg_fields; num_lines = atoi(path.c_str()); + + if ( autospread != 0.0 ) + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); @@ -85,15 +92,21 @@ bool Benchmark::DoUpdate() { if ( mode == STREAM ) { // do not do tracking, spread out elements over the second that we have... Put(field); - usleep(900000/num_lines); } else { SendEntry(field); } + + if ( spread != 0 ) + usleep(spread); + + if ( autospread_time != 0 ) { + usleep( autospread_time ); + } } - //if ( mode != STREAM ) { // well, does not really make sense in the streaming sense - but I like getting the event. + if ( mode != STREAM ) { EndCurrentSend(); - //} + } return true; } @@ -199,7 +212,7 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { bool Benchmark::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); - num_lines = num_lines*multiplication_factor; + num_lines = (int) ( (double) num_lines*multiplication_factor); switch ( mode ) { case MANUAL: @@ -217,8 +230,15 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } + + if ( autospread != 0.0 ) { + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); + // because executing this in every loop is apparently too expensive. + } Update(); // call update and not DoUpdate, because update actually checks disabled. + + SendEvent("HeartbeatDone", 0, 0); break; default: assert(false); diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index e0d3f124af..f0bd0c752d 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -38,7 +38,10 @@ private: int mode; int num_lines; - int multiplication_factor; + double multiplication_factor; + int spread; + double autospread; + int autospread_time; string RandomString(const int len); From ed5374b6d7b4d4601d8e8ee11f547692ebd3fffc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 09:35:45 -0700 Subject: [PATCH 106/123] and another option for the benchmark reader (constant addition of lines) --- scripts/base/frameworks/input/readers/benchmark.bro | 3 +++ src/input.bif | 1 + src/input/readers/Benchmark.cc | 13 +++++++++++++ src/input/readers/Benchmark.h | 1 + 4 files changed, 18 insertions(+) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index c6a6e88fca..0f3553b117 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -11,4 +11,7 @@ export { ## spreading where usleep = 1000000 / autospread * num_lines const autospread = 0.0 &redef; + + ## addition factor for each heartbeat + const addfactor = 0 &redef; } diff --git a/src/input.bif b/src/input.bif index 059a7ec8bf..798759ab66 100644 --- a/src/input.bif +++ b/src/input.bif @@ -50,3 +50,4 @@ module InputBenchmark; const factor: double; const spread: count; const autospread: double; +const addfactor: count; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index d8de8c2538..a17c8a7ff6 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -24,6 +24,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) multiplication_factor = double(BifConst::InputBenchmark::factor); autospread = double(BifConst::InputBenchmark::autospread); spread = int(BifConst::InputBenchmark::spread); + add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; } @@ -213,6 +214,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) { ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); + num_lines += add; switch ( mode ) { case MANUAL: @@ -230,6 +232,17 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } + + if ( add != 0 ) { + // we have to document at what time we changed the factor to what value. + Value** v = new Value*[2]; + v[0] = new Value(TYPE_COUNT, true); + v[0]->val.uint_val = num_lines; + v[1] = new Value(TYPE_TIME, true); + v[1]->val.double_val = CurrTime(); + + SendEvent("lines_changed", 2, v); + } if ( autospread != 0.0 ) { autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index f0bd0c752d..182adcd1af 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -42,6 +42,7 @@ private: int spread; double autospread; int autospread_time; + int add; string RandomString(const int len); From 719540414f6bcf80bd1efee9d6dcfce0b2ba7e7e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 09:41:00 -0700 Subject: [PATCH 107/123] repair general stupidity --- src/input/readers/Benchmark.cc | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index a17c8a7ff6..a914f1a2e8 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -222,7 +222,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - if ( multiplication_factor != 1 ) { + if ( multiplication_factor != 1 || add != 0 ) { // we have to document at what time we changed the factor to what value. Value** v = new Value*[2]; v[0] = new Value(TYPE_COUNT, true); @@ -233,17 +233,6 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) SendEvent("lines_changed", 2, v); } - if ( add != 0 ) { - // we have to document at what time we changed the factor to what value. - Value** v = new Value*[2]; - v[0] = new Value(TYPE_COUNT, true); - v[0]->val.uint_val = num_lines; - v[1] = new Value(TYPE_TIME, true); - v[1]->val.double_val = CurrTime(); - - SendEvent("lines_changed", 2, v); - } - if ( autospread != 0.0 ) { autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); // because executing this in every loop is apparently too expensive. From b47620e501944acddde0277fd2e9c7ab4de6cfec Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 30 Mar 2012 09:18:44 -0700 Subject: [PATCH 108/123] add a couple more configuration options --- .../frameworks/input/readers/benchmark.bro | 6 +++++ src/input.bif | 2 ++ src/input/readers/Benchmark.cc | 23 +++++++++++++++---- src/input/readers/Benchmark.h | 3 +++ 4 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index 0f3553b117..b5adc70861 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -14,4 +14,10 @@ export { ## addition factor for each heartbeat const addfactor = 0 &redef; + + ## stop spreading at x lines per heartbeat + const stopspreadat = 0 &redef; + + ## 1 -> enable timed spreading + const timedspread = 0 &redef; } diff --git a/src/input.bif b/src/input.bif index 798759ab66..63cbb2796d 100644 --- a/src/input.bif +++ b/src/input.bif @@ -51,3 +51,5 @@ const factor: double; const spread: count; const autospread: double; const addfactor: count; +const stopspreadat: count; +const timedspread: count; diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index a914f1a2e8..391fdd7435 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -26,6 +26,8 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) spread = int(BifConst::InputBenchmark::spread); add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; + stopspreadat = int(BifConst::InputBenchmark::stopspreadat); + timedspread = int(BifConst::InputBenchmark::timedspread); } @@ -54,6 +56,7 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel return false; } + heartbeatstarttime = CurrTime(); DoUpdate(); return true; @@ -97,12 +100,23 @@ bool Benchmark::DoUpdate() { SendEntry(field); } - if ( spread != 0 ) - usleep(spread); + if ( stopspreadat == 0 || num_lines < stopspreadat ) { + if ( spread != 0 ) + usleep(spread); - if ( autospread_time != 0 ) { - usleep( autospread_time ); + if ( autospread_time != 0 ) + usleep( autospread_time ); } + + if ( timedspread == 1 ) { + double diff; + do { + diff = CurrTime() - heartbeatstarttime; + //printf("%d %f\n", i, diff); + } while ( diff < i/(num_lines + (num_lines * 0.15) ) ); + //} while ( diff < 0.8); + } + } if ( mode != STREAM ) { @@ -215,6 +229,7 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); num_lines += add; + heartbeatstarttime = CurrTime(); switch ( mode ) { case MANUAL: diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index 182adcd1af..e5dca66889 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -43,6 +43,9 @@ private: double autospread; int autospread_time; int add; + int stopspreadat; + double heartbeatstarttime; + int timedspread; string RandomString(const int len); From 1170a877693f01d03963227c5d3a2f1aeeec53e1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 15:37:32 -0700 Subject: [PATCH 109/123] make benchmark reader hartbeat inverval aware fix small memleak on tablereader destruction make timespread better configureable --- .../frameworks/input/readers/benchmark.bro | 2 +- src/input.bif | 2 +- src/input/Manager.cc | 11 ++++++---- src/input/readers/Benchmark.cc | 21 +++++++++++++++---- src/input/readers/Benchmark.h | 2 +- src/threading/Manager.h | 8 ++++++- 6 files changed, 34 insertions(+), 12 deletions(-) diff --git a/scripts/base/frameworks/input/readers/benchmark.bro b/scripts/base/frameworks/input/readers/benchmark.bro index b5adc70861..fe44914271 100644 --- a/scripts/base/frameworks/input/readers/benchmark.bro +++ b/scripts/base/frameworks/input/readers/benchmark.bro @@ -19,5 +19,5 @@ export { const stopspreadat = 0 &redef; ## 1 -> enable timed spreading - const timedspread = 0 &redef; + const timedspread = 0.0 &redef; } diff --git a/src/input.bif b/src/input.bif index 63cbb2796d..0749ac0287 100644 --- a/src/input.bif +++ b/src/input.bif @@ -52,4 +52,4 @@ const spread: count; const autospread: double; const addfactor: count; const stopspreadat: count; -const timedspread: count; +const timedspread: double; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index f8ad493e11..218a9209ee 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -133,11 +133,15 @@ Manager::TableFilter::~TableFilter() { if ( rtype ) // can be 0 for sets Unref(rtype); - if ( currDict != 0 ) + if ( currDict != 0 ) { + currDict->Clear(); delete currDict; + } - if ( lastDict != 0 ) + if ( lastDict != 0 ) { + lastDict->Clear();; delete lastDict; + } } struct ReaderDefinition { @@ -898,6 +902,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } //i->tab->Assign(idxval, valval); + assert(idxval); HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); @@ -1067,8 +1072,6 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) { } int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { - bool updated = false; - assert(i); assert(i->filter_type == EVENT_FILTER); diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index 391fdd7435..118b57f616 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -13,6 +13,8 @@ #include #include +#include "../../threading/Manager.h" + using namespace input::reader; using threading::Value; using threading::Field; @@ -27,7 +29,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) add = int(BifConst::InputBenchmark::addfactor); autospread_time = 0; stopspreadat = int(BifConst::InputBenchmark::stopspreadat); - timedspread = int(BifConst::InputBenchmark::timedspread); + timedspread = double(BifConst::InputBenchmark::timedspread); } @@ -87,7 +89,8 @@ double Benchmark::CurrTime() { // read the entire file and send appropriate thingies back to InputMgr bool Benchmark::DoUpdate() { - for ( int i = 0; i < num_lines; i++ ) { + int linestosend = num_lines * threading::Manager::HEART_BEAT_INTERVAL; + for ( int i = 0; i < linestosend; i++ ) { Value** field = new Value*[num_fields]; for (unsigned int j = 0; j < num_fields; j++ ) { field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); @@ -108,12 +111,13 @@ bool Benchmark::DoUpdate() { usleep( autospread_time ); } - if ( timedspread == 1 ) { + if ( timedspread != 0.0 ) { double diff; do { diff = CurrTime() - heartbeatstarttime; //printf("%d %f\n", i, diff); - } while ( diff < i/(num_lines + (num_lines * 0.15) ) ); + //} while ( diff < i/threading::Manager::HEART_BEAT_INTERVAL*(num_lines + (num_lines * timedspread) ) ); + } while ( diff/threading::Manager::HEART_BEAT_INTERVAL < i/(linestosend + (linestosend * timedspread) ) ); //} while ( diff < 0.8); } @@ -226,6 +230,15 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { bool Benchmark::DoHeartbeat(double network_time, double current_time) { + /* + * This does not work the way I envisioned it, because the queueing is the problem. + printf("%f\n", CurrTime() - current_time); + if ( CurrTime() - current_time > 0.25 ) { + // event has hung for a time. refuse. + SendEvent("EndBenchmark", 0, 0); + return true; + } */ + ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); num_lines += add; diff --git a/src/input/readers/Benchmark.h b/src/input/readers/Benchmark.h index e5dca66889..ca248586da 100644 --- a/src/input/readers/Benchmark.h +++ b/src/input/readers/Benchmark.h @@ -45,7 +45,7 @@ private: int add; int stopspreadat; double heartbeatstarttime; - int timedspread; + double timedspread; string RandomString(const int len); diff --git a/src/threading/Manager.h b/src/threading/Manager.h index 7d9ba766d4..d5d78b288a 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -9,6 +9,10 @@ #include "BasicThread.h" #include "MsgThread.h" +namespace input { namespace reader { + class Benchmark; +}} + namespace threading { /** @@ -80,6 +84,7 @@ public: protected: friend class BasicThread; friend class MsgThread; + friend class input::reader::Benchmark; // needs heartbeat /** * Registers a new basic thread with the manager. This is @@ -118,9 +123,10 @@ protected: * Part of the IOSource interface. */ virtual const char* Tag() { return "threading::Manager"; } + + static const int HEART_BEAT_INTERVAL = 10; private: - static const int HEART_BEAT_INTERVAL = 1; typedef std::list all_thread_list; all_thread_list all_threads; From 7a71a74994a1271384e6fdc5b6c54e82e1a6761c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 16:31:11 -0700 Subject: [PATCH 110/123] fix largest leak in manager. --- src/input/Manager.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 218a9209ee..ed59900608 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1663,10 +1663,11 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { position += CopyValue(data, position, val); } - hash_t key = HashKey::HashBytes(data, length); + HashKey *key = new HashKey(data, length); + delete data; assert(position == length); - return new HashKey(data, length, key, true); + return key; } // convert threading value to Bro value From b7bbda724404273e38865b02b8e6c4dc767cdc51 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 28 Mar 2012 23:18:40 -0700 Subject: [PATCH 111/123] fix a couple more leaks. But - still leaking quite a lot with tables. --- src/input/Manager.cc | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index ed59900608..009fdb0bbb 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -25,9 +25,15 @@ using threading::Field; struct InputHash { hash_t valhash; - HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. + HashKey* idxkey; + ~InputHash(); }; +InputHash::~InputHash() { + if ( idxkey ) + delete idxkey; +} + declare(PDict, InputHash); class Manager::Filter { @@ -821,6 +827,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); + delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } else { assert( filter->num_val_fields > 0 ); @@ -855,7 +862,6 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { //Val* predidx = ListValToRecordVal(idxval->AsListVal(), filter->itype, &startpos); predidx = ValueToRecordVal(vals, filter->itype, &startpos); //ValueToRecordVal(vals, filter->itype, &startpos); - Ref(valval); if ( updated ) { ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); @@ -865,7 +871,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { bool result; if ( filter->num_val_fields > 0 ) { // we have values - result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval); + result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval->Ref()); } else { // no values result = CallPred(filter->pred, 2, ev, predidx->Ref()); @@ -876,10 +882,12 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... delete(filter->currDict->RemoveEntry(idxhash)); + delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } else { // keep old one filter->currDict->Insert(idxhash, h); + delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } } @@ -916,8 +924,10 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { if ( filter->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); + Unref(idxval); // asssign does not consume idxval. filter->currDict->Insert(idxhash, ih); + delete idxhash; if ( filter->event ) { EnumVal* ev; @@ -931,12 +941,11 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - Ref(valval); if ( filter->num_val_fields == 0 ) { Ref(filter->description); SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); } else { - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval); + SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); } } } From 8e526a7f835f514cf864c6bda172485e35b0ac60 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 29 Mar 2012 01:09:11 -0700 Subject: [PATCH 112/123] fix memory leak for tables... nearly completely. There is still a tiny where I have not yet found where the delete could be missing. For big table imports the memory footprint is significant nevertheless -- with tables of > 200000 entries, memory consumption can apparently reach in excess of 1.5Gb - and on a first glance this seems legitimate. (The reason for this is probably that we use several hash tables to keep the performance impact small). --- src/input/Manager.cc | 45 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 009fdb0bbb..a1a3410f5e 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -34,6 +34,11 @@ InputHash::~InputHash() { delete idxkey; } +static void input_hash_delete_func(void* val) { + InputHash* h = (InputHash*) val; + delete h; +} + declare(PDict, InputHash); class Manager::Filter { @@ -170,6 +175,14 @@ Manager::Manager() { } +Manager::~Manager() { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { + delete s->second; + delete s->first; + } + +} + ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { ReaderDefinition* ir = input_readers; @@ -527,7 +540,9 @@ bool Manager::CreateTableStream(RecordVal* fval) { filter->itype = idx->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); + filter->currDict->SetDeleteFunc(input_hash_delete_func); filter->lastDict = new PDict(InputHash); + filter->lastDict->SetDeleteFunc(input_hash_delete_func); filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault @@ -820,20 +835,20 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } } - InputHash *h = filter->lastDict->Lookup(idxhash); + InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( filter->num_val_fields == 0 || h->valhash == valhash ) { - // ok, exact duplicate + // ok, exact duplicate, move entry to new dicrionary and do nothing else. filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); delete idxhash; return filter->num_val_fields + filter->num_idx_fields; } else { assert( filter->num_val_fields > 0 ); - // updated + // entry was updated in some way filter->lastDict->Remove(idxhash); - delete(h); + // keep h for predicates updated = true; } @@ -881,8 +896,10 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { Unref(predidx); if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(filter->currDict->RemoveEntry(idxhash)); + // (but why should it be in there? assert this). + assert ( filter->currDict->RemoveEntry(idxhash) == 0 ); delete idxhash; + delete h; return filter->num_val_fields + filter->num_idx_fields; } else { // keep old one @@ -893,6 +910,12 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { } } + + // now we don't need h anymore - if we are here, the entry is updated and a new h is created. + if ( h ) { + delete h; + h = 0; + } Val* idxval; @@ -1014,6 +1037,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { Unref(predidx); Unref(ev); filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); + delete lastDictIdxKey; continue; } } @@ -1030,8 +1054,9 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { if ( ev ) Unref(ev); - filter->tab->Delete(ih->idxkey); - filter->lastDict->Remove(lastDictIdxKey); // deletex in next line + Unref(filter->tab->Delete(ih->idxkey)); + filter->lastDict->Remove(lastDictIdxKey); // delete in next line + delete lastDictIdxKey; delete(ih); } @@ -1040,6 +1065,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { filter->lastDict = filter->currDict; filter->currDict = new PDict(InputHash); + filter->currDict->SetDeleteFunc(input_hash_delete_func); #ifdef DEBUG DBG_LOG(DBG_INPUT, "EndCurrentSend complete for stream %s, queueing update_finished event", @@ -1284,9 +1310,12 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { // only if filter = true -> no filtering if ( filterresult ) { - success = ( filter->tab->Delete(idxval) != 0 ); + Val* retptr = filter->tab->Delete(idxval); + success = ( retptr != 0 ); if ( !success ) { reporter->Error("Internal error while deleting values from input table"); + } else { + Unref(retptr); } } } else if ( i->filter_type == EVENT_FILTER ) { From 384fc730d472a69d921b18fa57a6db559a8faedd Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 1 Apr 2012 17:13:51 -0700 Subject: [PATCH 113/123] fix heart_beat_interval -- initialization in constructor does not work anymore (probably due to change in init ordering?) --- src/threading/Manager.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index ec7ab34d14..6c14fd65ca 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -60,6 +60,12 @@ void Manager::KillThreads() void Manager::AddThread(BasicThread* thread) { + if ( heart_beat_interval == 0 ) { + // sometimes initialization does not seem to work from constructor + heart_beat_interval = double(BifConst::Threading::heart_beat_interval); + DBG_LOG(DBG_THREADING, "Heart beat interval set to %f", heart_beat_interval); + } + DBG_LOG(DBG_THREADING, "Adding thread %s ...", thread->Name().c_str()); all_threads.push_back(thread); idle = false; From 25affe2c826c8ba93069685d6b64f68612c971e3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 3 Apr 2012 00:52:41 +0200 Subject: [PATCH 114/123] fix missing get call for heart beat in benchmark reader. --- src/input/readers/Benchmark.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index f0cebd2dc1..9bba7d7831 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -30,6 +30,7 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) autospread_time = 0; stopspreadat = int(BifConst::InputBenchmark::stopspreadat); timedspread = double(BifConst::InputBenchmark::timedspread); + heart_beat_interval = double(BifConst::Threading::heart_beat_interval); } From a5cc98bb5d189dcc9ad04516b6a583bae4ea1508 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 10 Apr 2012 13:57:09 -0700 Subject: [PATCH 115/123] fix memory leak in tables and vectors that are read into tables --- src/input/Manager.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index a1a3410f5e..102fd78d6f 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -1795,8 +1795,12 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(ValueToVal( val->val.set_val.vals[i], type ), 0); + Val* assignval = ValueToVal( val->val.set_val.vals[i], type ); + t->Assign(assignval, 0); + Unref(assignval); // idex is not consumed by assign. } + + Unref(s); return t; break; } @@ -1809,6 +1813,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { for ( int i = 0; i < val->val.vector_val.size; i++ ) { v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); } + Unref(vt); return v; } From 1967f6f81caec93c1d85c851e3229a9197fe5cf1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 15 Apr 2012 15:11:39 -0700 Subject: [PATCH 116/123] rename a couple of structures and make the names in manager fit the api more. This should it make easier for other people to understand what is going on without having knowledge of an "internal api * means * in external api" mapping. --- src/input/Manager.cc | 98 ++++++++++++++++++++++++-------------------- src/input/Manager.h | 24 +++++------ 2 files changed, 65 insertions(+), 57 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 102fd78d6f..8c8d6d8ba3 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -23,6 +23,11 @@ using namespace input; using threading::Value; using threading::Field; +/** + * InputHashes are used as Dictionaries to store the value and index hashes for all lines currently stored in a table. + * Index hash is stored as HashKey*, because it is thrown into other Bro functions that need the complex structure of it. + * For everything we do (with values), we just take the hash_t value and compare it directly with == + */ struct InputHash { hash_t valhash; HashKey* idxkey; @@ -41,7 +46,10 @@ static void input_hash_delete_func(void* val) { declare(PDict, InputHash); -class Manager::Filter { +/** + * Base stuff that every stream can do + */ +class Manager::Stream { public: string name; string source; @@ -49,25 +57,25 @@ public: int mode; - FilterType filter_type; // to distinguish between event and table filters + StreamType filter_type; // to distinguish between event and table filters EnumVal* type; ReaderFrontend* reader; RecordVal* description; - Filter(); - virtual ~Filter(); + Stream(); + virtual ~Stream(); }; -Manager::Filter::Filter() { +Manager::Stream::Stream() { type = 0; reader = 0; description = 0; removed = false; } -Manager::Filter::~Filter() { +Manager::Stream::~Stream() { if ( type ) Unref(type); if ( description ) @@ -77,7 +85,7 @@ Manager::Filter::~Filter() { delete(reader); } -class Manager::TableFilter: public Manager::Filter { +class Manager::TableStream: public Manager::Stream { public: unsigned int num_idx_fields; @@ -96,11 +104,11 @@ public: EventHandlerPtr event; - TableFilter(); - ~TableFilter(); + TableStream(); + ~TableStream(); }; -class Manager::EventFilter: public Manager::Filter { +class Manager::EventStream: public Manager::Stream { public: EventHandlerPtr event; @@ -108,11 +116,11 @@ public: unsigned int num_fields; bool want_record; - EventFilter(); - ~EventFilter(); + EventStream(); + ~EventStream(); }; -Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { +Manager::TableStream::TableStream() : Manager::Stream::Stream() { filter_type = TABLE_FILTER; tab = 0; @@ -125,18 +133,18 @@ Manager::TableFilter::TableFilter() : Manager::Filter::Filter() { pred = 0; } -Manager::EventFilter::EventFilter() : Manager::Filter::Filter() { +Manager::EventStream::EventStream() : Manager::Stream::Stream() { fields = 0; filter_type = EVENT_FILTER; } -Manager::EventFilter::~EventFilter() { +Manager::EventStream::~EventStream() { if ( fields ) { Unref(fields); } } -Manager::TableFilter::~TableFilter() { +Manager::TableStream::~TableStream() { if ( tab ) Unref(tab); if ( itype ) @@ -176,7 +184,7 @@ Manager::Manager() } Manager::~Manager() { - for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { delete s->second; delete s->first; } @@ -233,7 +241,7 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) } // create a new input reader object to be used at whomevers leisure lateron. -bool Manager::CreateStream(Filter* info, RecordVal* description) +bool Manager::CreateStream(Stream* info, RecordVal* description) { ReaderDefinition* ir = input_readers; @@ -249,7 +257,7 @@ bool Manager::CreateStream(Filter* info, RecordVal* description) Unref(name_val); { - Filter *i = FindFilter(name); + Stream *i = FindStream(name); if ( i != 0 ) { reporter->Error("Trying create already existing input stream %s", name.c_str()); return false; @@ -296,7 +304,7 @@ bool Manager::CreateEventStream(RecordVal* fval) { return false; } - EventFilter* filter = new EventFilter(); + EventStream* filter = new EventStream(); { bool res = CreateStream(filter, fval); if ( res == false ) { @@ -412,7 +420,7 @@ bool Manager::CreateTableStream(RecordVal* fval) { return false; } - TableFilter* filter = new TableFilter(); + TableStream* filter = new TableStream(); { bool res = CreateStream(filter, fval); if ( res == false ) { @@ -620,7 +628,7 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) bool Manager::RemoveStream(const string &name) { - Filter *i = FindFilter(name); + Stream *i = FindStream(name); if ( i == 0 ) { return false; // not found @@ -644,7 +652,7 @@ bool Manager::RemoveStream(const string &name) { } bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->Error("Stream not found in RemoveStreamContinuation"); @@ -712,7 +720,7 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co bool Manager::ForceUpdate(const string &name) { - Filter *i = FindFilter(name); + Stream *i = FindStream(name); if ( i == 0 ) { reporter->Error("Stream %s not found", name.c_str()); return false; @@ -786,7 +794,7 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in SendEntry"); return; @@ -797,7 +805,7 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { readFields = SendEntryTable(i, vals); } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(i, type, vals); + readFields = SendEventStreamEvent(i, type, vals); } else { assert(false); } @@ -808,13 +816,13 @@ void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { delete [] vals; } -int Manager::SendEntryTable(Filter* i, const Value* const *vals) { +int Manager::SendEntryTable(Stream* i, const Value* const *vals) { bool updated = false; assert(i); assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; HashKey* idxhash = HashValues(filter->num_idx_fields, vals); @@ -979,7 +987,7 @@ int Manager::SendEntryTable(Filter* i, const Value* const *vals) { void Manager::EndCurrentSend(ReaderFrontend* reader) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in EndCurrentSend"); return; @@ -996,7 +1004,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = filter->lastDict->InitForIteration(); @@ -1083,7 +1091,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { } void Manager::Put(ReaderFrontend* reader, Value* *vals) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in Put"); return; @@ -1094,7 +1102,7 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) { readFields = PutTable(i, vals); } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - readFields = SendEventFilterEvent(i, type, vals); + readFields = SendEventStreamEvent(i, type, vals); } else { assert(false); } @@ -1106,11 +1114,11 @@ void Manager::Put(ReaderFrontend* reader, Value* *vals) { } -int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const *vals) { +int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) { assert(i); assert(i->filter_type == EVENT_FILTER); - EventFilter* filter = (EventFilter*) i; + EventStream* filter = (EventStream*) i; Val *val; list out_vals; @@ -1143,11 +1151,11 @@ int Manager::SendEventFilterEvent(Filter* i, EnumVal* type, const Value* const * } -int Manager::PutTable(Filter* i, const Value* const *vals) { +int Manager::PutTable(Stream* i, const Value* const *vals) { assert(i); assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; @@ -1244,7 +1252,7 @@ int Manager::PutTable(Filter* i, const Value* const *vals) { // Todo:: perhaps throw some kind of clear-event? void Manager::Clear(ReaderFrontend* reader) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in Clear"); return; @@ -1256,14 +1264,14 @@ void Manager::Clear(ReaderFrontend* reader) { #endif assert(i->filter_type == TABLE_FILTER); - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; filter->tab->RemoveAll(); } // put interface: delete old entry from table. bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { - Filter *i = FindFilter(reader); + Stream *i = FindStream(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader in Delete"); return false; @@ -1273,7 +1281,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { int readVals = 0; if ( i->filter_type == TABLE_FILTER ) { - TableFilter* filter = (TableFilter*) i; + TableStream* filter = (TableStream*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; @@ -1320,7 +1328,7 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { } } else if ( i->filter_type == EVENT_FILTER ) { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - readVals = SendEventFilterEvent(i, type, vals); + readVals = SendEventStreamEvent(i, type, vals); success = true; } else { assert(false); @@ -1840,9 +1848,9 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { return NULL; } -Manager::Filter* Manager::FindFilter(const string &name) +Manager::Stream* Manager::FindStream(const string &name) { - for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { if ( (*s).second->name == name ) { @@ -1853,9 +1861,9 @@ Manager::Filter* Manager::FindFilter(const string &name) return 0; } -Manager::Filter* Manager::FindFilter(ReaderFrontend* reader) +Manager::Stream* Manager::FindStream(ReaderFrontend* reader) { - map::iterator s = readers.find(reader); + map::iterator s = readers.find(reader); if ( s != readers.end() ) { return s->second; } diff --git a/src/input/Manager.h b/src/input/Manager.h index c6dd40bd95..8f09828988 100644 --- a/src/input/Manager.h +++ b/src/input/Manager.h @@ -107,25 +107,25 @@ protected: // Functions are called from the ReaderBackend to notify the manager, that a filter has been removed // or a stream has been closed. // Used to prevent race conditions where data for a specific filter is still in the queue when the - // RemoveFilter directive is executed by the main thread. + // RemoveStream directive is executed by the main thread. // This makes sure all data that has ben queued for a filter is still received. bool RemoveStreamContinuation(ReaderFrontend* reader); private: - class Filter; - class TableFilter; - class EventFilter; + class Stream; + class TableStream; + class EventStream; - bool CreateStream(Filter*, RecordVal* description); + bool CreateStream(Stream*, RecordVal* description); // SendEntry implementation for Tablefilter - int SendEntryTable(Filter* i, const threading::Value* const *vals); + int SendEntryTable(Stream* i, const threading::Value* const *vals); // Put implementation for Tablefilter - int PutTable(Filter* i, const threading::Value* const *vals); + int PutTable(Stream* i, const threading::Value* const *vals); // SendEntry and Put implementation for Eventfilter - int SendEventFilterEvent(Filter* i, EnumVal* type, const threading::Value* const *vals); + int SendEventStreamEvent(Stream* i, EnumVal* type, const threading::Value* const *vals); // Checks is a bro type can be used for data reading. The equivalend in threading cannot be used, because we have support different types // from the log framework @@ -163,12 +163,12 @@ private: // Converts a Bro ListVal to a RecordVal given the record type RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); - Filter* FindFilter(const string &name); - Filter* FindFilter(ReaderFrontend* reader); + Stream* FindStream(const string &name); + Stream* FindStream(ReaderFrontend* reader); - enum FilterType { TABLE_FILTER, EVENT_FILTER }; + enum StreamType { TABLE_FILTER, EVENT_FILTER }; - map readers; + map readers; }; From 48e05621c082983c8eb103499a4151cd320eedbd Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 16 Apr 2012 14:49:24 -0700 Subject: [PATCH 117/123] update description to current interface. But this needs to get longer... --- doc/input.rst | 128 +++++++++++++++++++++----------------------------- 1 file changed, 54 insertions(+), 74 deletions(-) diff --git a/doc/input.rst b/doc/input.rst index e201af9fed..d9fe8aa6b8 100644 --- a/doc/input.rst +++ b/doc/input.rst @@ -20,11 +20,9 @@ very similar to the abstracts used in the logging framework: Input Streams An input stream corresponds to a single input source (usually a textfile). It defined the information necessary - to find the source (e.g. the filename) - - Filters - Each input stream has a set of filters attached to it, that - determine exaclty what kind of information is read. + to find the source (e.g. the filename), the reader that it used + to get data from it (see below). + It also defines exactly what data is read from the input source. There are two different kind of streams, event streams and table streams. By default, event streams generate an event for each line read @@ -41,28 +39,37 @@ very similar to the abstracts used in the logging framework: one event per line. -Basics -====== +Event Streams +============= For examples, please look at the unit tests in ``testing/btest/scripts/base/frameworks/input/``. -A very basic example to open an input stream is: +Event Streams are streams that generate an event for each line in of the input source. + +For example, a simple stream retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: .. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool) { + # work with event data + } - module Foo; - - export { - # Create an ID for our new stream - redef enum Input::ID += { INPUT }; + event bro_init { + Input::add_event([$source="input.log", $name="input", $fields=Val, $ev=line]); } - event bro_init() { - Input::create_stream(FOO::INPUT, [$source="input.log"]); - } +The fields that can be set for an event stream are: -The fields that can be set when creating a stream are: + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). ``source`` A mandatory string identifying the source of the data. @@ -81,49 +88,9 @@ The fields that can be set when creating a stream are: ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new data is added to the file. - ``autostart`` - If set to yes, the first update operation is triggered automatically after the first filter has been added to the stream. - This has to be set to false if several filters are added to the input source. - In this case Input::force_update has to be called manually once after all filters have been added. - -Filters -======= - -Each filter defines the data fields that it wants to receive from the respective -input file. Depending on the type of filter, events or a table are created from -the data in the source file. - -Event Filters -------------- - -Event filters are filters that generate an event for each line in of the input source. - -For example, a simple filter retrieving the fields ``i`` and ``b`` from an inputSource -could be defined as follows: - -.. code:: bro - - type Val: record { - i: int; - b: bool; - }; - - event line(tpe: Input::Event, i: int, b: bool) { - # work with event data - } - - event bro_init { - # Input stream definition, etc - ... - - Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); - } - -The fields that can be set for an event filter are: - ``name`` - A mandatory name for the filter that can later be used - to manipulate it further. + A mandatory name for the stream that can later be used + to remove it. ``fields`` Name of a record type containing the fields, which should be retrieved from @@ -138,16 +105,14 @@ The fields that can be set for an event filter are: been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information for event filters, the value is always ``NEW`` at the moment. - ``want_record`` - Boolean value, that defines if the event wants to receive the fields inside of - a single record value, or individually (default). -Table Filters -------------- -Table filters are the second, more complex type of filter. +Table Streams +============= -Table filters store the information they read from an input source in a bro table. For example, +Table streams are the second, more complex type of input streams. + +Table streams store the information they read from an input source in a bro table. For example, when reading a file that contains ip addresses and connection attemt information one could use an approach similar to this: @@ -164,18 +129,33 @@ an approach similar to this: global conn_attempts: table[addr] of count = table(); event bro_init { - # Input stream definitions, etc. - ... - - Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); - - # read the file after all filters have been set (only needed if autostart is set to false) - Input::force_update(Foo::INPUT); + Input::add_table([$source="input.txt", $name="input", $idx=Idx, $val=Val, $destination=conn_attempts]); } The table conn_attempts will then contain the information about connection attemps. -The possible fields that can be set for an table filter are: +The possible fields that can be set for an table stream are: + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + ``mode`` + The mode in which the stream is opened. Possible values are ``MANUAL``, ``REREAD`` and ``STREAM``. + Default is ``MANUAL``. + ``MANUAL`` means, that the files is not updated after it has been read. Changes to the file will not + be reflected in the data bro knows. + ``REREAD`` means that the whole file is read again each time a change is found. This should be used for + files that are mapped to a table where individual lines can change. + ``STREAM`` means that the data from the file is streamed. Events / table entries will be generated as new + data is added to the file. ``name`` A mandatory name for the filter that can later be used From 82a6f3832ae10c5a9b5a1646422511c9fce1b041 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 May 2012 13:51:50 -0700 Subject: [PATCH 118/123] fix two memory leaks which occured when one used filters. --- src/input/Manager.cc | 6 +++++- src/input/readers/Ascii.cc | 4 ++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index bd6cd34991..6b179f66a1 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -750,7 +750,7 @@ Val* Manager::RecordValToIndexVal(RecordVal *r) { int num_fields = type->NumFields(); if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) { - idxval = r->Lookup(0); + idxval = r->LookupWithDefault(0); } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < num_fields; j++ ) { @@ -902,6 +902,7 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { if ( result == false ) { Unref(predidx); + Unref(valval); if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... // (but why should it be in there? assert this). @@ -956,6 +957,9 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { Ref(oldval); // otherwise it is no longer accessible after the assignment filter->tab->Assign(idxval, k, valval); Unref(idxval); // asssign does not consume idxval. + if ( predidx != 0 ) { + Unref(predidx); + } filter->currDict->Insert(idxhash, ih); delete idxhash; diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a1119ed253..a167408a0e 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -101,12 +101,16 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c file = new ifstream(path.c_str()); if ( !file->is_open() ) { Error(Fmt("Init: cannot open %s", fname.c_str())); + delete(file); + file = 0; return false; } if ( ReadHeader(false) == false ) { Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); file->close(); + delete(file); + file = 0; return false; } From 2034c10e9766d833e7bd32d011c5557ab1f90a7f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 10:33:22 -0700 Subject: [PATCH 119/123] make input framework source (hopefully) adhere to the usual indentation style. No functional changes. --- src/input/Manager.cc | 1224 ++++++++++++++++++-------------- src/input/ReaderBackend.cc | 148 ++-- src/input/ReaderFrontend.cc | 40 +- src/input/readers/Ascii.cc | 314 ++++---- src/input/readers/Benchmark.cc | 118 ++- src/input/readers/Raw.cc | 155 ++-- 6 files changed, 1102 insertions(+), 897 deletions(-) diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 6b179f66a1..0fde16b87d 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -24,25 +24,30 @@ using threading::Value; using threading::Field; /** - * InputHashes are used as Dictionaries to store the value and index hashes for all lines currently stored in a table. - * Index hash is stored as HashKey*, because it is thrown into other Bro functions that need the complex structure of it. - * For everything we do (with values), we just take the hash_t value and compare it directly with == + * InputHashes are used as Dictionaries to store the value and index hashes for all + * lines currently stored in a table. Index hash is stored as HashKey*, because it is + * thrown into other Bro functions that need the complex structure of it. + * For everything we do (with values), we just take the hash_t value and compare it + * directly with == */ -struct InputHash { +struct InputHash + { hash_t valhash; HashKey* idxkey; ~InputHash(); -}; + }; -InputHash::~InputHash() { +InputHash::~InputHash() + { if ( idxkey ) delete idxkey; -} + } -static void input_hash_delete_func(void* val) { +static void input_hash_delete_func(void* val) + { InputHash* h = (InputHash*) val; delete h; -} + } declare(PDict, InputHash); @@ -68,14 +73,16 @@ public: virtual ~Stream(); }; -Manager::Stream::Stream() { +Manager::Stream::Stream() + { type = 0; reader = 0; description = 0; removed = false; -} + } -Manager::Stream::~Stream() { +Manager::Stream::~Stream() + { if ( type ) Unref(type); if ( description ) @@ -83,7 +90,7 @@ Manager::Stream::~Stream() { if ( reader ) delete(reader); -} + } class Manager::TableStream: public Manager::Stream { public: @@ -120,7 +127,8 @@ public: ~EventStream(); }; -Manager::TableStream::TableStream() : Manager::Stream::Stream() { +Manager::TableStream::TableStream() : Manager::Stream::Stream() + { filter_type = TABLE_FILTER; tab = 0; @@ -131,20 +139,22 @@ Manager::TableStream::TableStream() : Manager::Stream::Stream() { lastDict = 0; pred = 0; -} + } -Manager::EventStream::EventStream() : Manager::Stream::Stream() { +Manager::EventStream::EventStream() : Manager::Stream::Stream() + { fields = 0; filter_type = EVENT_FILTER; -} + } -Manager::EventStream::~EventStream() { - if ( fields ) { +Manager::EventStream::~EventStream() + { + if ( fields ) Unref(fields); - } -} + } -Manager::TableStream::~TableStream() { +Manager::TableStream::~TableStream() + { if ( tab ) Unref(tab); if ( itype ) @@ -152,22 +162,24 @@ Manager::TableStream::~TableStream() { if ( rtype ) // can be 0 for sets Unref(rtype); - if ( currDict != 0 ) { + if ( currDict != 0 ) + { currDict->Clear(); delete currDict; - } + } - if ( lastDict != 0 ) { + if ( lastDict != 0 ) + { lastDict->Clear();; delete lastDict; - } -} + } + } struct ReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages bool (*init)(); // optional one-time inifializing function - ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances + ReaderBackend* (*factory)(ReaderFrontend* frontend); // factory function for creating instances }; ReaderDefinition input_readers[] = { @@ -180,49 +192,55 @@ ReaderDefinition input_readers[] = { }; Manager::Manager() -{ -} - -Manager::~Manager() { - for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - delete s->second; - delete s->first; + { } -} - -ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) { - ReaderDefinition* ir = input_readers; - - while ( true ) { - if ( ir->type == BifEnum::Input::READER_DEFAULT ) +Manager::~Manager() + { + for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { - reporter->Error("The reader that was requested was not found and could not be initialized."); - return 0; + delete s->second; + delete s->first; } - if ( ir->type != type ) { + } + +ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) + { + ReaderDefinition* ir = input_readers; + + while ( true ) + { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + reporter->Error("The reader that was requested was not found and could not be initialized."); + return 0; + } + + if ( ir->type != type ) + { // no, didn't find the right one... ++ir; continue; - } + } // call init function of writer if presnt if ( ir->init ) - { + { if ( (*ir->init)() ) { - //clear it to be not called again - ir->init = 0; - } else { + //clear it to be not called again + ir->init = 0; + } + else { // ohok. init failed, kill factory for all eternity ir->factory = 0; DBG_LOG(DBG_LOGGING, "Failed to init input class %s", ir->name); return 0; } - } + } if ( !ir->factory ) // no factory? @@ -230,7 +248,8 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) // all done. break. break; - } + } + assert(ir->factory); ReaderBackend* backend = (*ir->factory)(frontend); @@ -238,31 +257,34 @@ ReaderBackend* Manager::CreateBackend(ReaderFrontend* frontend, bro_int_t type) frontend->ty_name = ir->name; return backend; -} + } // create a new input reader object to be used at whomevers leisure lateron. bool Manager::CreateStream(Stream* info, RecordVal* description) -{ + { ReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) - { + if ( ! ( same_type(rtype, BifType::Record::Input::TableDescription, 0) + || same_type(rtype, BifType::Record::Input::EventDescription, 0) ) ) + { reporter->Error("Streamdescription argument not of right type for new input stream"); return false; - } + } Val* name_val = description->LookupWithDefault(rtype->FieldOffset("name")); string name = name_val->AsString()->CheckString(); Unref(name_val); - { + { Stream *i = FindStream(name); - if ( i != 0 ) { - reporter->Error("Trying create already existing input stream %s", name.c_str()); + if ( i != 0 ) + { + reporter->Error("Trying create already existing input stream %s", + name.c_str()); return false; + } } - } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); @@ -293,25 +315,27 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) return true; -} + } -bool Manager::CreateEventStream(RecordVal* fval) { +bool Manager::CreateEventStream(RecordVal* fval) + { RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::EventDescription, 0) ) - { + { reporter->Error("filter argument not of right type"); return false; - } + } EventStream* filter = new EventStream(); - { + { bool res = CreateStream(filter, fval); - if ( res == false ) { + if ( res == false ) + { delete filter; return false; + } } - } RecordType *fields = fval->LookupWithDefault(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); @@ -322,77 +346,87 @@ bool Manager::CreateEventStream(RecordVal* fval) { Func* event = event_val->AsFunc(); Unref(event_val); - { + { FuncType* etype = event->FType()->AsFuncType(); - if ( ! etype->IsEvent() ) { + if ( ! etype->IsEvent() ) + { reporter->Error("stream event is a function, not an event"); return false; - } + } const type_list* args = etype->ArgTypes()->Types(); - if ( args->length() < 2 ) { + if ( args->length() < 2 ) + { reporter->Error("event takes not enough arguments"); return false; - } + } if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) - { + { reporter->Error("events second attribute must be of type Input::Event"); return false; - } + } if ( ! same_type((*args)[0], BifType::Record::Input::EventDescription, 0) ) - { + { reporter->Error("events first attribute must be of type Input::EventDescription"); return false; - } + } - if ( want_record->InternalInt() == 0 ) { - if ( args->length() != fields->NumFields() + 2 ) { + if ( want_record->InternalInt() == 0 ) + { + if ( args->length() != fields->NumFields() + 2 ) + { reporter->Error("event has wrong number of arguments"); return false; - } + } - for ( int i = 0; i < fields->NumFields(); i++ ) { - if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) { + for ( int i = 0; i < fields->NumFields(); i++ ) + { + if ( !same_type((*args)[i+2], fields->FieldType(i) ) ) + { reporter->Error("Incompatible type for event"); return false; + } } - } - } else if ( want_record->InternalInt() == 1 ) { - if ( args->length() != 3 ) { + } + else if ( want_record->InternalInt() == 1 ) + { + if ( args->length() != 3 ) + { reporter->Error("event has wrong number of arguments"); return false; - } + } - if ( !same_type((*args)[2], fields ) ) { + if ( !same_type((*args)[2], fields ) ) + { reporter->Error("Incompatible type for event"); return false; - } + } - } else { + } + else assert(false); - } - } + } vector fieldsV; // vector, because UnrollRecordType needs it bool status = !UnrollRecordType(&fieldsV, fields, ""); - if ( status ) { + if ( status ) + { reporter->Error("Problem unrolling"); return false; - } + } Field** logf = new Field*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) logf[i] = fieldsV[i]; - } Unref(fields); // ref'd by lookupwithdefault filter->num_fields = fieldsV.size(); @@ -412,56 +446,64 @@ bool Manager::CreateEventStream(RecordVal* fval) { return true; } -bool Manager::CreateTableStream(RecordVal* fval) { +bool Manager::CreateTableStream(RecordVal* fval) + { RecordType* rtype = fval->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::TableDescription, 0) ) - { + { reporter->Error("filter argument not of right type"); return false; - } + } TableStream* filter = new TableStream(); - { + { bool res = CreateStream(filter, fval); - if ( res == false ) { + if ( res == false ) + { delete filter; return false; + } } - } Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); RecordType *idx = fval->LookupWithDefault(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = 0; - if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) { + + if ( fval->LookupWithDefault(rtype->FieldOffset("val")) != 0 ) + { val = fval->LookupWithDefault(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); Unref(val); // The lookupwithdefault in the if-clause ref'ed val. - } + } + TableVal *dst = fval->LookupWithDefault(rtype->FieldOffset("destination"))->AsTableVal(); // check if index fields match table description - { + { int num = idx->NumFields(); const type_list* tl = dst->Type()->AsTableType()->IndexTypes(); loop_over_list(*tl, j) { - if ( j >= num ) { + if ( j >= num ) + { reporter->Error("Table type has more indexes than index definition"); return false; - } + } - if ( !same_type(idx->FieldType(j), (*tl)[j]) ) { + if ( !same_type(idx->FieldType(j), (*tl)[j]) ) + { reporter->Error("Table type does not match index type"); return false; - } + } } - if ( num != j ) { + if ( num != j ) + { reporter->Error("Table has less elements than index definition"); return false; + } } - } Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -469,51 +511,57 @@ bool Manager::CreateTableStream(RecordVal* fval) { Func* event = event_val ? event_val->AsFunc() : 0; Unref(event_val); - if ( event ) { + if ( event ) + { FuncType* etype = event->FType()->AsFuncType(); - if ( ! etype->IsEvent() ) { + if ( ! etype->IsEvent() ) + { reporter->Error("stream event is a function, not an event"); return false; - } + } const type_list* args = etype->ArgTypes()->Types(); if ( args->length() != 4 ) - { + { reporter->Error("Table event must take 4 arguments"); return false; - } + } if ( ! same_type((*args)[0], BifType::Record::Input::TableDescription, 0) ) - { + { reporter->Error("table events first attribute must be of type Input::TableDescription"); return false; - } + } if ( ! same_type((*args)[1], BifType::Enum::Input::Event, 0) ) - { + { reporter->Error("table events second attribute must be of type Input::Event"); return false; - } + } if ( ! same_type((*args)[2], idx) ) - { + { reporter->Error("table events index attributes do not match"); return false; - } + } if ( want_record->InternalInt() == 1 && ! same_type((*args)[3], val) ) - { + { reporter->Error("table events value attributes do not match"); return false; - } else if ( want_record->InternalInt() == 0 && !same_type((*args)[3], val->FieldType(0) ) ) { + } + else if ( want_record->InternalInt() == 0 + && !same_type((*args)[3], val->FieldType(0) ) ) + { reporter->Error("table events value attribute does not match"); return false; - } + } + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); - } + } vector fieldsV; // vector, because we don't know the length beforehands @@ -529,16 +577,16 @@ bool Manager::CreateTableStream(RecordVal* fval) { if ( !val ) assert(valfields == 0); - if ( status ) { + if ( status ) + { reporter->Error("Problem unrolling"); return false; - } + } Field** fields = new Field*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) fields[i] = fieldsV[i]; - } filter->pred = pred ? pred->AsFunc() : 0; filter->num_idx_fields = idxfields; @@ -556,13 +604,15 @@ bool Manager::CreateTableStream(RecordVal* fval) { Unref(want_record); // ref'd by lookupwithdefault Unref(pred); - if ( valfields > 1 ) { - if ( ! filter->want_record ) { + if ( valfields > 1 ) + { + if ( ! filter->want_record ) + { reporter->Error("Stream %s does not want a record (want_record=F), but has more then one value field. Aborting", filter->name.c_str()); delete filter; return false; + } } - } assert(filter->reader); @@ -574,7 +624,7 @@ bool Manager::CreateTableStream(RecordVal* fval) { filter->name.c_str()); return true; -} + } bool Manager::IsCompatibleType(BroType* t, bool atomic_only) @@ -582,7 +632,7 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t ) return false; - switch ( t->Tag() ) { + switch ( t->Tag() ) { case TYPE_BOOL: case TYPE_INT: case TYPE_COUNT: @@ -624,20 +674,21 @@ bool Manager::IsCompatibleType(BroType* t, bool atomic_only) } return false; -} + } -bool Manager::RemoveStream(const string &name) { +bool Manager::RemoveStream(const string &name) + { Stream *i = FindStream(name); - if ( i == 0 ) { + if ( i == 0 ) return false; // not found - } - if ( i->removed ) { + if ( i->removed ) + { reporter->Error("Stream %s is already queued for removal. Ignoring remove.", name.c_str()); return false; - } + } i->removed = true; @@ -649,54 +700,66 @@ bool Manager::RemoveStream(const string &name) { #endif return true; -} - -bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) { - Stream *i = FindStream(reader); - - if ( i == 0 ) { - reporter->Error("Stream not found in RemoveStreamContinuation"); - return false; } +bool Manager::RemoveStreamContinuation(ReaderFrontend* reader) + { + Stream *i = FindStream(reader); + + if ( i == 0 ) + { + reporter->Error("Stream not found in RemoveStreamContinuation"); + return false; + } #ifdef DEBUG DBG_LOG(DBG_INPUT, "Successfully executed removal of stream %s", i->name.c_str()); #endif + readers.erase(reader); delete(i); return true; -} + } -bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { - for ( int i = 0; i < rec->NumFields(); i++ ) +bool Manager::UnrollRecordType(vector *fields, + const RecordType *rec, const string& nameprepend) { - if ( !IsCompatibleType(rec->FieldType(i)) ) { + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( !IsCompatibleType(rec->FieldType(i)) ) + { reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag())); return false; - } + } if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) - { + { string prep = nameprepend + rec->FieldName(i) + "."; if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) - { + { return false; - } + } - } else { + } + else + { Field* field = new Field(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); - if ( field->type == TYPE_TABLE ) { + if ( field->type == TYPE_TABLE ) + { field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); - } else if ( field->type == TYPE_VECTOR ) { + } + else if ( field->type == TYPE_VECTOR ) + { field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); - } else if ( field->type == TYPE_PORT && - rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { + } else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) + { // we have an annotation for the second column Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); @@ -705,31 +768,32 @@ bool Manager::UnrollRecordType(vector *fields, const RecordType *rec, co assert(c->Type()->Tag() == TYPE_STRING); field->secondary_name = c->AsStringVal()->AsString()->CheckString(); - } + } - if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) { + if ( rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL ) ) field->optional = true; - } fields->push_back(field); + } } - } return true; -} + } bool Manager::ForceUpdate(const string &name) -{ + { Stream *i = FindStream(name); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->Error("Stream %s not found", name.c_str()); return false; - } + } - if ( i->removed ) { + if ( i->removed ) + { reporter->Error("Stream %s is already queued for removal. Ignoring force update.", name.c_str()); return false; - } + } i->reader->Update(); @@ -742,31 +806,34 @@ bool Manager::ForceUpdate(const string &name) } -Val* Manager::RecordValToIndexVal(RecordVal *r) { +Val* Manager::RecordValToIndexVal(RecordVal *r) + { Val* idxval; RecordType *type = r->Type()->AsRecordType(); int num_fields = type->NumFields(); - if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) { + if ( num_fields == 1 && type->FieldDecl(0)->type->Tag() != TYPE_RECORD ) + { idxval = r->LookupWithDefault(0); - } else { + } + else + { ListVal *l = new ListVal(TYPE_ANY); - for ( int j = 0 ; j < num_fields; j++ ) { - //Val* rval = r->Lookup(j); - //assert(rval != 0); + for ( int j = 0 ; j < num_fields; j++ ) l->Append(r->LookupWithDefault(j)); - } + idxval = l; - } + } return idxval; -} + } -Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) { +Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Value* const *vals) + { Val* idxval; int position = 0; @@ -776,47 +843,54 @@ Val* Manager::ValueToIndexVal(int num_fields, const RecordType *type, const Valu position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); - for ( int j = 0 ; j < type->NumFields(); j++ ) { - if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { - l->Append(ValueToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); - } else { + for ( int j = 0 ; j < type->NumFields(); j++ ) + { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) + l->Append(ValueToRecordVal(vals, + type->FieldType(j)->AsRecordType(), &position)); + else + { l->Append(ValueToVal(vals[position], type->FieldType(j))); position++; + } } - } idxval = l; - } + } assert ( position == num_fields ); return idxval; -} + } -void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) { +void Manager::SendEntry(ReaderFrontend* reader, Value* *vals) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in SendEntry"); return; - } + } int readFields; - if ( i->filter_type == TABLE_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) readFields = SendEntryTable(i, vals); - } else if ( i->filter_type == EVENT_FILTER ) { + else if ( i->filter_type == EVENT_FILTER ) + { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); - } else { + } + else assert(false); - } - for ( int i = 0; i < readFields; i++ ) { + for ( int i = 0; i < readFields; i++ ) delete vals[i]; - } - delete [] vals; -} -int Manager::SendEntryTable(Stream* i, const Value* const *vals) { + delete [] vals; + } + +int Manager::SendEntryTable(Stream* i, const Value* const *vals) + { bool updated = false; assert(i); @@ -826,59 +900,66 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { HashKey* idxhash = HashValues(filter->num_idx_fields, vals); - if ( idxhash == 0 ) { + if ( idxhash == 0 ) + { reporter->Error("Could not hash line. Ignoring"); return filter->num_val_fields + filter->num_idx_fields; - } + } hash_t valhash = 0; - if ( filter->num_val_fields > 0 ) { + if ( filter->num_val_fields > 0 ) + { HashKey* valhashkey = HashValues(filter->num_val_fields, vals+filter->num_idx_fields); if ( valhashkey == 0 ) { // empty line. index, but no values. // hence we also have no hash value... - } else { + } + else + { valhash = valhashkey->Hash(); delete(valhashkey); + } } - } InputHash *h = filter->lastDict->Lookup(idxhash); - if ( h != 0 ) { + if ( h != 0 ) + { // seen before - if ( filter->num_val_fields == 0 || h->valhash == valhash ) { + if ( filter->num_val_fields == 0 || h->valhash == valhash ) + { // ok, exact duplicate, move entry to new dicrionary and do nothing else. filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); delete idxhash; return filter->num_val_fields + filter->num_idx_fields; - } else { + } + else + { assert( filter->num_val_fields > 0 ); // entry was updated in some way filter->lastDict->Remove(idxhash); // keep h for predicates updated = true; + } } - } Val* valval; RecordVal* predidx = 0; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 0 ) { + if ( filter->num_val_fields == 0 ) valval = 0; - } else if ( filter->num_val_fields == 1 && !filter->want_record ) { + else if ( filter->num_val_fields == 1 && !filter->want_record ) valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); - } else { + else valval = ValueToRecordVal(vals, filter->rtype, &position); - } - // call filter first to determine if we really add / change the entry - if ( filter->pred ) { + if ( filter->pred ) + { EnumVal* ev; //Ref(idxval); int startpos = 0; @@ -886,68 +967,74 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { predidx = ValueToRecordVal(vals, filter->itype, &startpos); //ValueToRecordVal(vals, filter->itype, &startpos); - if ( updated ) { + if ( updated ) ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); - } else { + else ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - } bool result; - if ( filter->num_val_fields > 0 ) { // we have values + if ( filter->num_val_fields > 0 ) // we have values result = CallPred(filter->pred, 3, ev, predidx->Ref(), valval->Ref()); - } else { - // no values + else // no values result = CallPred(filter->pred, 2, ev, predidx->Ref()); - } - if ( result == false ) { + if ( result == false ) + { Unref(predidx); Unref(valval); - if ( !updated ) { + if ( !updated ) + { // throw away. Hence - we quit. And remove the entry from the current dictionary... // (but why should it be in there? assert this). assert ( filter->currDict->RemoveEntry(idxhash) == 0 ); delete idxhash; delete h; return filter->num_val_fields + filter->num_idx_fields; - } else { + } + else + { // keep old one filter->currDict->Insert(idxhash, h); delete idxhash; return filter->num_val_fields + filter->num_idx_fields; + } } - } - } + } // now we don't need h anymore - if we are here, the entry is updated and a new h is created. - if ( h ) { + if ( h ) + { delete h; h = 0; - } + } Val* idxval; - if ( predidx != 0 ) { + if ( predidx != 0 ) + { idxval = RecordValToIndexVal(predidx); // I think there is an unref missing here. But if I insert is, it crashes :) - } else { + } + else idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); - } + Val* oldval = 0; - if ( updated == true ) { + if ( updated == true ) + { assert(filter->num_val_fields > 0); // in that case, we need the old value to send the event (if we send an event). oldval = filter->tab->Lookup(idxval, false); - } + } //i->tab->Assign(idxval, valval); assert(idxval); HashKey* k = filter->tab->ComputeHash(idxval); - if ( !k ) { + if ( !k ) + { reporter->InternalError("could not hash"); assert(false); - } + } InputHash* ih = new InputHash(); ih->idxkey = new HashKey(k->Key(), k->Size(), k->Hash()); @@ -955,57 +1042,64 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals) { if ( filter->event && updated ) Ref(oldval); // otherwise it is no longer accessible after the assignment + filter->tab->Assign(idxval, k, valval); Unref(idxval); // asssign does not consume idxval. - if ( predidx != 0 ) { + if ( predidx != 0 ) Unref(predidx); - } filter->currDict->Insert(idxhash, ih); delete idxhash; - if ( filter->event ) { + if ( filter->event ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - if ( updated ) { // in case of update send back the old value. + if ( updated ) + { // in case of update send back the old value. assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); - } else { + } + else + { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( filter->num_val_fields == 0 ) { + if ( filter->num_val_fields == 0 ) + { Ref(filter->description); SendEvent(filter->event, 3, filter->description->Ref(), ev, predidx); - } else { + } + else SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); + } - } - } + } return filter->num_val_fields + filter->num_idx_fields; -} + } -void Manager::EndCurrentSend(ReaderFrontend* reader) { +void Manager::EndCurrentSend(ReaderFrontend* reader) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + + if ( i == 0 ) + { reporter->InternalError("Unknown reader in EndCurrentSend"); return; - } + } #ifdef DEBUG DBG_LOG(DBG_INPUT, "Got EndCurrentSend stream %s", i->name.c_str()); #endif - if ( i->filter_type == EVENT_FILTER ) { - // nothing to do.. + if ( i->filter_type == EVENT_FILTER ) // nothing to do.. return; - } assert(i->filter_type == TABLE_FILTER); TableStream* filter = (TableStream*) i; @@ -1016,8 +1110,8 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - + while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) + { ListVal * idx = 0; Val *val = 0; @@ -1025,16 +1119,18 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { EnumVal* ev = 0; int startpos = 0; - if ( filter->pred || filter->event ) { + if ( filter->pred || filter->event ) + { idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); val = filter->tab->Lookup(idx); assert(val != 0); predidx = ListValToRecordVal(idx, filter->itype, &startpos); ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - } + } - if ( filter->pred ) { + if ( filter->pred ) + { // ask predicate, if we want to expire this element... Ref(ev); @@ -1043,7 +1139,8 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { bool result = CallPred(filter->pred, 3, ev, predidx, val); - if ( result == false ) { + if ( result == false ) + { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... Unref(predidx); @@ -1051,15 +1148,16 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); delete lastDictIdxKey; continue; + } } - } - if ( filter->event ) { + if ( filter->event ) + { Ref(predidx); Ref(val); Ref(ev); SendEvent(filter->event, 3, ev, predidx, val); - } + } if ( predidx ) // if we have a filter or an event... Unref(predidx); @@ -1070,9 +1168,9 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { filter->lastDict->Remove(lastDictIdxKey); // delete in next line delete lastDictIdxKey; delete(ih); - } + } - filter->lastDict->Clear(); // should be empty->->-> but->->-> well->->-> who knows->->-> + filter->lastDict->Clear(); // should be empt. buti- well... who knows... delete(filter->lastDict); filter->lastDict = filter->currDict; @@ -1086,39 +1184,40 @@ void Manager::EndCurrentSend(ReaderFrontend* reader) { // Send event that the current update is indeed finished. EventHandler* handler = event_registry->Lookup("Input::update_finished"); - if ( handler == 0 ) { + if ( handler == 0 ) reporter->InternalError("Input::update_finished not found!"); - } - SendEvent(handler, 2, new StringVal(i->name.c_str()), new StringVal(i->source.c_str())); -} + } -void Manager::Put(ReaderFrontend* reader, Value* *vals) { +void Manager::Put(ReaderFrontend* reader, Value* *vals) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in Put"); return; - } + } int readFields; - if ( i->filter_type == TABLE_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) readFields = PutTable(i, vals); - } else if ( i->filter_type == EVENT_FILTER ) { + else if ( i->filter_type == EVENT_FILTER ) + { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); readFields = SendEventStreamEvent(i, type, vals); - } else { + } + else assert(false); - } - for ( int i = 0; i < readFields; i++ ) { + for ( int i = 0; i < readFields; i++ ) delete vals[i]; - } + delete [] vals; + } -} - -int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) { +int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const *vals) + { assert(i); assert(i->filter_type == EVENT_FILTER); @@ -1133,29 +1232,37 @@ int Manager::SendEventStreamEvent(Stream* i, EnumVal* type, const Value* const * out_vals.push_back(type); int position = 0; - if ( filter->want_record ) { + if ( filter->want_record ) + { RecordVal * r = ValueToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); - } else { - for ( int j = 0; j < filter->fields->NumFields(); j++) { + } + else + { + for ( int j = 0; j < filter->fields->NumFields(); j++) + { Val* val = 0; - if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { - val = ValueToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); - } else { + if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) + val = ValueToRecordVal(vals, + filter->fields->FieldType(j)->AsRecordType(), + &position); + else + { val = ValueToVal(vals[position], filter->fields->FieldType(j)); position++; - } + } out_vals.push_back(val); + } } - } SendEvent(filter->event, out_vals); return filter->fields->NumFields(); -} + } -int Manager::PutTable(Stream* i, const Value* const *vals) { +int Manager::PutTable(Stream* i, const Value* const *vals) + { assert(i); assert(i->filter_type == TABLE_FILTER); @@ -1165,102 +1272,115 @@ int Manager::PutTable(Stream* i, const Value* const *vals) { Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 0 ) { + if ( filter->num_val_fields == 0 ) valval = 0; - } else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) { + else if ( filter->num_val_fields == 1 && filter->want_record == 0 ) valval = ValueToVal(vals[position], filter->rtype->FieldType(0)); - } else { + else valval = ValueToRecordVal(vals, filter->rtype, &position); - } // if we have a subscribed event, we need to figure out, if this is an update or not // same for predicates - if ( filter->pred || filter->event ) { + if ( filter->pred || filter->event ) + { bool updated = false; Val* oldval = 0; - if ( filter->num_val_fields > 0 ) { + if ( filter->num_val_fields > 0 ) + { // in that case, we need the old value to send the event (if we send an event). oldval = filter->tab->Lookup(idxval, false); - } + } - if ( oldval != 0 ) { + if ( oldval != 0 ) + { // it is an update updated = true; Ref(oldval); // have to do that, otherwise it may disappear in assign - } + } // predicate if we want the update or not - if ( filter->pred ) { + if ( filter->pred ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); Ref(valval); - if ( updated ) { - ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); - } else { - ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - } + if ( updated ) + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + BifType::Enum::Input::Event); + else + ev = new EnumVal(BifEnum::Input::EVENT_NEW, + BifType::Enum::Input::Event); bool result; - if ( filter->num_val_fields > 0 ) { // we have values + if ( filter->num_val_fields > 0 ) // we have values result = CallPred(filter->pred, 3, ev, predidx, valval); - } else { - // no values + else // no values result = CallPred(filter->pred, 2, ev, predidx); - } - if ( result == false ) { + if ( result == false ) + { // do nothing Unref(idxval); Unref(valval); Unref(oldval); return filter->num_val_fields + filter->num_idx_fields; - } + } - } + } filter->tab->Assign(idxval, valval); - if ( filter->event ) { + if ( filter->event ) + { EnumVal* ev; int startpos = 0; Val* predidx = ValueToRecordVal(vals, filter->itype, &startpos); - if ( updated ) { // in case of update send back the old value. + if ( updated ) + { + // in case of update send back the old value. assert ( filter->num_val_fields > 0 ); - ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, + BifType::Enum::Input::Event); assert ( oldval != 0 ); - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, oldval); - } else { - ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); - if ( filter->num_val_fields == 0 ) { - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx); - } else { - SendEvent(filter->event, 4, filter->description->Ref(), ev, predidx, valval->Ref()); + SendEvent(filter->event, 4, filter->description->Ref(), + ev, predidx, oldval); + } + else + { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, + BifType::Enum::Input::Event); + if ( filter->num_val_fields == 0 ) + SendEvent(filter->event, 4, filter->description->Ref(), + ev, predidx); + else + SendEvent(filter->event, 4, filter->description->Ref(), + ev, predidx, valval->Ref()); } + } - } - - - } else { - // no predicates or other stuff + } + else // no predicates or other stuff filter->tab->Assign(idxval, valval); - } + return filter->num_idx_fields + filter->num_val_fields; -} + } // Todo:: perhaps throw some kind of clear-event? -void Manager::Clear(ReaderFrontend* reader) { +void Manager::Clear(ReaderFrontend* reader) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in Clear"); return; - } + } #ifdef DEBUG DBG_LOG(DBG_INPUT, "Got Clear for stream %s", @@ -1271,30 +1391,35 @@ void Manager::Clear(ReaderFrontend* reader) { TableStream* filter = (TableStream*) i; filter->tab->RemoveAll(); -} + } // put interface: delete old entry from table. -bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { +bool Manager::Delete(ReaderFrontend* reader, Value* *vals) + { Stream *i = FindStream(reader); - if ( i == 0 ) { + if ( i == 0 ) + { reporter->InternalError("Unknown reader in Delete"); return false; - } + } bool success = false; int readVals = 0; - if ( i->filter_type == TABLE_FILTER ) { + if ( i->filter_type == TABLE_FILTER ) + { TableStream* filter = (TableStream*) i; Val* idxval = ValueToIndexVal(filter->num_idx_fields, filter->itype, vals); assert(idxval != 0); readVals = filter->num_idx_fields + filter->num_val_fields; bool filterresult = true; - if ( filter->pred || filter->event ) { + if ( filter->pred || filter->event ) + { Val *val = filter->tab->Lookup(idxval); - if ( filter->pred ) { + if ( filter->pred ) + { Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); int startpos = 0; @@ -1302,62 +1427,68 @@ bool Manager::Delete(ReaderFrontend* reader, Value* *vals) { filterresult = CallPred(filter->pred, 3, ev, predidx, val); - if ( filterresult == false ) { + if ( filterresult == false ) + { // keep it. Unref(idxval); success = true; + } + } - } - // only if filter = true -> no filtering - if ( filterresult && filter->event ) { + if ( filterresult && filter->event ) + { Ref(idxval); assert(val != 0); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(filter->event, 4, filter->description->Ref(), ev, idxval, val); + } } - } // only if filter = true -> no filtering - if ( filterresult ) { + if ( filterresult ) + { Val* retptr = filter->tab->Delete(idxval); success = ( retptr != 0 ); - if ( !success ) { + if ( !success ) reporter->Error("Internal error while deleting values from input table"); - } else { + else Unref(retptr); } - } - } else if ( i->filter_type == EVENT_FILTER ) { + + } + else if ( i->filter_type == EVENT_FILTER ) + { EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); readVals = SendEventStreamEvent(i, type, vals); success = true; - } else { + } + else + { assert(false); return false; - } + } - for ( int i = 0; i < readVals; i++ ) { + for ( int i = 0; i < readVals; i++ ) delete vals[i]; - } + delete [] vals; return success; -} + } bool Manager::CallPred(Func* pred_func, const int numvals, ...) -{ + { bool result; val_list vl(numvals); va_list lP; va_start(lP, numvals); for ( int i = 0; i < numvals; i++ ) - { vl.append( va_arg(lP, Val*) ); - } + va_end(lP); Val* v = pred_func->Call(&vl); @@ -1365,120 +1496,131 @@ bool Manager::CallPred(Func* pred_func, const int numvals, ...) Unref(v); return(result); -} + } bool Manager::SendEvent(const string& name, const int num_vals, Value* *vals) -{ + { EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) { + if ( handler == 0 ) + { reporter->Error("Event %s not found", name.c_str()); return false; - } + } RecordType *type = handler->FType()->Args(); int num_event_vals = type->NumFields(); - if ( num_vals != num_event_vals ) { + if ( num_vals != num_event_vals ) + { reporter->Error("Wrong number of values for event %s", name.c_str()); return false; - } + } val_list* vl = new val_list; - for ( int i = 0; i < num_vals; i++) { + for ( int i = 0; i < num_vals; i++) vl->append(ValueToVal(vals[i], type->FieldType(i))); - } mgr.Dispatch(new Event(handler, vl)); - for ( int i = 0; i < num_vals; i++ ) { + for ( int i = 0; i < num_vals; i++ ) delete vals[i]; - } + delete [] vals; return true; } void Manager::SendEvent(EventHandlerPtr ev, const int numvals, ...) -{ + { val_list* vl = new val_list; va_list lP; va_start(lP, numvals); for ( int i = 0; i < numvals; i++ ) - { vl->append( va_arg(lP, Val*) ); - } + va_end(lP); mgr.QueueEvent(ev, vl, SOURCE_LOCAL); -} - -void Manager::SendEvent(EventHandlerPtr ev, list events) -{ - val_list* vl = new val_list; - - for ( list::iterator i = events.begin(); i != events.end(); i++ ) { - vl->append( *i ); } - mgr.QueueEvent(ev, vl, SOURCE_LOCAL); -} +void Manager::SendEvent(EventHandlerPtr ev, list events) + { + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) + { + vl->append( *i ); + } -// Convert a bro list value to a bro record value. I / we could think about moving this functionality to val.cc -RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); + } + +// Convert a bro list value to a bro record value. +// I / we could think about moving this functionality to val.cc +RecordVal* Manager::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) + { assert(position != 0 ); // we need the pointer to point to data; - if ( request_type->Tag() != TYPE_RECORD ) { + if ( request_type->Tag() != TYPE_RECORD ) + { reporter->InternalError("ListValToRecordVal called on non-record-value."); return 0; - } + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); assert(list != 0); int maxpos = list->Length(); - for ( int i = 0; i < request_type->NumFields(); i++ ) { + for ( int i = 0; i < request_type->NumFields(); i++ ) + { assert ( (*position) <= maxpos ); Val* fieldVal = 0; - if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); - } else { + else + { fieldVal = list->Index(*position); (*position)++; - } + } rec->Assign(i, fieldVal); - } + } return rec; -} + } // Convert a threading value to a record value -RecordVal* Manager::ValueToRecordVal(const Value* const *vals, RecordType *request_type, int* position) { +RecordVal* Manager::ValueToRecordVal(const Value* const *vals, + RecordType *request_type, int* position) + { assert(position != 0); // we need the pointer to point to data. - if ( request_type->Tag() != TYPE_RECORD ) { + if ( request_type->Tag() != TYPE_RECORD ) + { reporter->InternalError("ValueToRecordVal called on non-record-value."); return 0; - } + } RecordVal* rec = new RecordVal(request_type->AsRecordType()); - for ( int i = 0; i < request_type->NumFields(); i++ ) { + for ( int i = 0; i < request_type->NumFields(); i++ ) + { Val* fieldVal = 0; - if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); - } else { + else + { fieldVal = ValueToVal(vals[*position], request_type->FieldType(i)); (*position)++; - } + } rec->Assign(i, fieldVal); - } + } return rec; -} + } // Count the length of the values // used to create a correct length buffer for hashing later @@ -1495,7 +1637,7 @@ int Manager::GetValueLength(const Value* val) { case TYPE_COUNT: case TYPE_COUNTER: length += sizeof(val->val.uint_val); - break; + break; case TYPE_PORT: length += sizeof(val->val.port_val.port); @@ -1517,48 +1659,48 @@ int Manager::GetValueLength(const Value* val) { case TYPE_ADDR: { - switch ( val->val.addr_val.family ) { - case IPv4: - length += sizeof(val->val.addr_val.in.in4); - break; - case IPv6: - length += sizeof(val->val.addr_val.in.in6); - break; - default: - assert(false); - } - + switch ( val->val.addr_val.family ) { + case IPv4: + length += sizeof(val->val.addr_val.in.in4); + break; + case IPv6: + length += sizeof(val->val.addr_val.in.in6); + break; + default: + assert(false); + } } break; case TYPE_SUBNET: { - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - length += sizeof(val->val.subnet_val.prefix.in.in4)+sizeof(val->val.subnet_val.length); - break; - case IPv6: - length += sizeof(val->val.subnet_val.prefix.in.in6)+sizeof(val->val.subnet_val.length); - break; - default: - assert(false); - } - + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + length += sizeof(val->val.subnet_val.prefix.in.in4)+ + sizeof(val->val.subnet_val.length); + break; + case IPv6: + length += sizeof(val->val.subnet_val.prefix.in.in6)+ + sizeof(val->val.subnet_val.length); + break; + default: + assert(false); + } } break; - case TYPE_TABLE: { - for ( int i = 0; i < val->val.set_val.size; i++ ) { + case TYPE_TABLE: + { + for ( int i = 0; i < val->val.set_val.size; i++ ) length += GetValueLength(val->val.set_val.vals[i]); - } break; } - case TYPE_VECTOR: { + case TYPE_VECTOR: + { int j = val->val.vector_val.size; - for ( int i = 0; i < j; i++ ) { + for ( int i = 0; i < j; i++ ) length += GetValueLength(val->val.vector_val.vals[i]); - } break; } @@ -1572,7 +1714,8 @@ int Manager::GetValueLength(const Value* val) { // Given a threading::value, copy the raw data bytes into *data and return how many bytes were copied. // Used for hashing the values for lookup in the bro table -int Manager::CopyValue(char *data, const int startpos, const Value* val) { +int Manager::CopyValue(char *data, const int startpos, const Value* val) + { assert( val->present ); // presence has to be checked elsewhere switch ( val->type ) { @@ -1588,11 +1731,14 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { return sizeof(val->val.uint_val); break; - case TYPE_PORT: { + case TYPE_PORT: + { int length = 0; - memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); + memcpy(data+startpos, (const void*) &(val->val.port_val.port), + sizeof(val->val.port_val.port)); length += sizeof(val->val.port_val.port); - memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), sizeof(val->val.port_val.proto)); + memcpy(data+startpos+length, (const void*) &(val->val.port_val.proto), + sizeof(val->val.port_val.proto)); length += sizeof(val->val.port_val.proto); return length; break; @@ -1602,7 +1748,8 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + memcpy(data+startpos, (const void*) &(val->val.double_val), + sizeof(val->val.double_val)); return sizeof(val->val.double_val); break; @@ -1616,64 +1763,66 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { case TYPE_ADDR: { - int length; - switch ( val->val.addr_val.family ) { - case IPv4: - length = sizeof(val->val.addr_val.in.in4); - memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in4), length); - break; - case IPv6: - length = sizeof(val->val.addr_val.in.in6); - memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in6), length); - break; - default: - assert(false); - } - - return length; - + int length; + switch ( val->val.addr_val.family ) { + case IPv4: + length = sizeof(val->val.addr_val.in.in4); + memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in4), length); + break; + case IPv6: + length = sizeof(val->val.addr_val.in.in6); + memcpy(data + startpos, (const char*) &(val->val.addr_val.in.in6), length); + break; + default: + assert(false); + } + return length; } break; case TYPE_SUBNET: { - int length; - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - length = sizeof(val->val.addr_val.in.in4); - memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); - break; - case IPv6: - length = sizeof(val->val.addr_val.in.in6); - memcpy(data + startpos, (const char*) &(val->val.subnet_val.prefix.in.in4), length); - break; - default: - assert(false); - } - int lengthlength = sizeof(val->val.subnet_val.length); - memcpy(data + startpos + length , (const char*) &(val->val.subnet_val.length), lengthlength); - length += lengthlength; - return length; - + int length; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + length = sizeof(val->val.addr_val.in.in4); + memcpy(data + startpos, + (const char*) &(val->val.subnet_val.prefix.in.in4), length); + break; + case IPv6: + length = sizeof(val->val.addr_val.in.in6); + memcpy(data + startpos, + (const char*) &(val->val.subnet_val.prefix.in.in4), length); + break; + default: + assert(false); + } + int lengthlength = sizeof(val->val.subnet_val.length); + memcpy(data + startpos + length , + (const char*) &(val->val.subnet_val.length), lengthlength); + length += lengthlength; + return length; } break; - case TYPE_TABLE: { + case TYPE_TABLE: + { int length = 0; int j = val->val.set_val.size; - for ( int i = 0; i < j; i++ ) { + for ( int i = 0; i < j; i++ ) length += CopyValue(data, startpos+length, val->val.set_val.vals[i]); - } + return length; break; } - case TYPE_VECTOR: { + case TYPE_VECTOR: + { int length = 0; int j = val->val.vector_val.size; - for ( int i = 0; i < j; i++ ) { + for ( int i = 0; i < j; i++ ) length += CopyValue(data, startpos+length, val->val.vector_val.vals[i]); - } + return length; break; } @@ -1685,52 +1834,57 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val) { assert(false); return 0; -} + } // Hash num_elements threading values and return the HashKey for them. At least one of the vals has to be ->present. -HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) { +HashKey* Manager::HashValues(const int num_elements, const Value* const *vals) + { int length = 0; - for ( int i = 0; i < num_elements; i++ ) { + for ( int i = 0; i < num_elements; i++ ) + { const Value* val = vals[i]; if ( val->present ) length += GetValueLength(val); - } + } - if ( length == 0 ) { + if ( length == 0 ) + { reporter->Error("Input reader sent line where all elements are null values. Ignoring line"); return NULL; - } + } int position = 0; char *data = (char*) malloc(length); - if ( data == 0 ) { + if ( data == 0 ) reporter->InternalError("Could not malloc?"); - } - for ( int i = 0; i < num_elements; i++ ) { + + for ( int i = 0; i < num_elements; i++ ) + { const Value* val = vals[i]; if ( val->present ) position += CopyValue(data, position, val); - } + } HashKey *key = new HashKey(data, length); delete data; assert(position == length); return key; -} + } // convert threading value to Bro value -Val* Manager::ValueToVal(const Value* val, BroType* request_type) { +Val* Manager::ValueToVal(const Value* val, BroType* request_type) + { - if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) + { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; - } + } - if ( !val->present ) { + if ( !val->present ) return 0; // unset field - } switch ( val->type ) { case TYPE_BOOL: @@ -1762,72 +1916,73 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { case TYPE_ADDR: { - IPAddr* addr; - switch ( val->val.addr_val.family ) { - case IPv4: - addr = new IPAddr(val->val.addr_val.in.in4); - break; - case IPv6: - addr = new IPAddr(val->val.addr_val.in.in6); - break; - default: - assert(false); - } - AddrVal* addrval = new AddrVal(*addr); - delete addr; - return addrval; - + IPAddr* addr; + switch ( val->val.addr_val.family ) { + case IPv4: + addr = new IPAddr(val->val.addr_val.in.in4); + break; + case IPv6: + addr = new IPAddr(val->val.addr_val.in.in6); + break; + default: + assert(false); + } + AddrVal* addrval = new AddrVal(*addr); + delete addr; + return addrval; } case TYPE_SUBNET: { - IPAddr* addr; - switch ( val->val.subnet_val.prefix.family ) { - case IPv4: - addr = new IPAddr(val->val.subnet_val.prefix.in.in4); - break; - case IPv6: - addr = new IPAddr(val->val.subnet_val.prefix.in.in6); - break; - default: - assert(false); - } - SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); - delete addr; - return subnetval; - + IPAddr* addr; + switch ( val->val.subnet_val.prefix.family ) { + case IPv4: + addr = new IPAddr(val->val.subnet_val.prefix.in.in4); + break; + case IPv6: + addr = new IPAddr(val->val.subnet_val.prefix.in.in6); + break; + default: + assert(false); } + SubNetVal* subnetval = new SubNetVal(*addr, val->val.subnet_val.length); + delete addr; + return subnetval; break; + } - case TYPE_TABLE: { + case TYPE_TABLE: + { // all entries have to have the same type... BroType* type = request_type->AsTableType()->Indices()->PureType(); TypeList* set_index = new TypeList(type->Ref()); set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) { + for ( int i = 0; i < val->val.set_val.size; i++ ) + { Val* assignval = ValueToVal( val->val.set_val.vals[i], type ); t->Assign(assignval, 0); Unref(assignval); // idex is not consumed by assign. - } + } Unref(s); return t; break; } - case TYPE_VECTOR: { + case TYPE_VECTOR: + { // all entries have to have the same type... BroType* type = request_type->AsVectorType()->YieldType(); VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + for ( int i = 0; i < val->val.vector_val.size; i++ ) v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); - } + Unref(vt); return v; - + break; } case TYPE_ENUM: { @@ -1836,9 +1991,10 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { string module = extract_module_name(val->val.string_val->c_str()); string var = extract_var_name(val->val.string_val->c_str()); bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); - if ( index == -1 ) { - reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); - } + if ( index == -1 ) + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", + module.c_str(), var.c_str()); + return new EnumVal(index, request_type->Ref()->AsEnumType() ); break; } @@ -1850,26 +2006,24 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) { assert(false); return NULL; -} + } Manager::Stream* Manager::FindStream(const string &name) { for ( map::iterator s = readers.begin(); s != readers.end(); ++s ) { if ( (*s).second->name == name ) - { return (*s).second; } - } return 0; } Manager::Stream* Manager::FindStream(ReaderFrontend* reader) -{ + { map::iterator s = readers.find(reader); - if ( s != readers.end() ) { + if ( s != readers.end() ) return s->second; - } + return 0; -} + } diff --git a/src/input/ReaderBackend.cc b/src/input/ReaderBackend.cc index 27401ffcb8..c625301383 100644 --- a/src/input/ReaderBackend.cc +++ b/src/input/ReaderBackend.cc @@ -15,10 +15,11 @@ public: : threading::OutputMessage("Put", reader), val(val) {} - virtual bool Process() { + virtual bool Process() + { input_mgr->Put(Object(), val); return true; - } + } private: Value* *val; @@ -30,9 +31,10 @@ public: : threading::OutputMessage("Delete", reader), val(val) {} - virtual bool Process() { + virtual bool Process() + { return input_mgr->Delete(Object(), val); - } + } private: Value* *val; @@ -43,10 +45,11 @@ public: ClearMessage(ReaderFrontend* reader) : threading::OutputMessage("Clear", reader) {} - virtual bool Process() { + virtual bool Process() + { input_mgr->Clear(Object()); return true; - } + } private: }; @@ -57,14 +60,15 @@ public: : threading::OutputMessage("SendEvent", reader), name(name), num_vals(num_vals), val(val) {} - virtual bool Process() { + virtual bool Process() + { bool success = input_mgr->SendEvent(name, num_vals, val); if ( !success ) reporter->Error("SendEvent for event %s failed", name.c_str()); return true; // we do not want to die if sendEvent fails because the event did not return. - } + } private: const string name; @@ -78,10 +82,11 @@ public: : threading::OutputMessage("SendEntry", reader), val(val) { } - virtual bool Process() { + virtual bool Process() + { input_mgr->SendEntry(Object(), val); return true; - } + } private: Value* *val; @@ -92,10 +97,11 @@ public: EndCurrentSendMessage(ReaderFrontend* reader) : threading::OutputMessage("EndCurrentSend", reader) {} - virtual bool Process() { + virtual bool Process() + { input_mgr->EndCurrentSend(Object()); return true; - } + } private: }; @@ -105,9 +111,10 @@ public: ReaderClosedMessage(ReaderFrontend* reader) : threading::OutputMessage("ReaderClosed", reader) {} - virtual bool Process() { + virtual bool Process() + { return input_mgr->RemoveStreamContinuation(Object()); - } + } private: }; @@ -119,12 +126,16 @@ public: DisableMessage(ReaderFrontend* writer) : threading::OutputMessage("Disable", writer) {} - virtual bool Process() { Object()->SetDisable(); return true; } + virtual bool Process() + { + Object()->SetDisable(); + return true; + } }; ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() -{ + { buf = 0; buf_len = 1024; disabled = true; // disabled will be set correcty in init. @@ -132,45 +143,45 @@ ReaderBackend::ReaderBackend(ReaderFrontend* arg_frontend) : MsgThread() frontend = arg_frontend; SetName(frontend->Name()); -} + } ReaderBackend::~ReaderBackend() -{ - -} + { + } void ReaderBackend::Put(Value* *val) -{ + { SendOut(new PutMessage(frontend, val)); -} + } void ReaderBackend::Delete(Value* *val) -{ + { SendOut(new DeleteMessage(frontend, val)); -} + } void ReaderBackend::Clear() -{ + { SendOut(new ClearMessage(frontend)); -} + } void ReaderBackend::SendEvent(const string& name, const int num_vals, Value* *vals) -{ + { SendOut(new SendEventMessage(frontend, name, num_vals, vals)); -} + } void ReaderBackend::EndCurrentSend() -{ + { SendOut(new EndCurrentSendMessage(frontend)); -} + } void ReaderBackend::SendEntry(Value* *vals) -{ + { SendOut(new SendEntryMessage(frontend, vals)); -} + } -bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, const threading::Field* const* arg_fields) -{ +bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, + const threading::Field* const* arg_fields) + { source = arg_source; SetName("InputReader/"+source); @@ -180,89 +191,90 @@ bool ReaderBackend::Init(string arg_source, int mode, const int arg_num_fields, // disable if DoInit returns error. int success = DoInit(arg_source, mode, arg_num_fields, arg_fields); - if ( !success ) { + if ( !success ) + { Error("Init failed"); DisableFrontend(); - } + } disabled = !success; return success; -} + } void ReaderBackend::Close() -{ + { DoClose(); disabled = true; DisableFrontend(); SendOut(new ReaderClosedMessage(frontend)); - if ( fields != 0 ) { - - for ( unsigned int i = 0; i < num_fields; i++ ) { + if ( fields != 0 ) + { + for ( unsigned int i = 0; i < num_fields; i++ ) delete(fields[i]); - } delete[] (fields); fields = 0; + } } -} bool ReaderBackend::Update() -{ + { if ( disabled ) return false; bool success = DoUpdate(); - if ( !success ) { + if ( !success ) DisableFrontend(); - } return success; -} + } void ReaderBackend::DisableFrontend() -{ - disabled = true; // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now + { + disabled = true; + // we also set disabled here, because there still may be other messages queued and we will dutifully ignore these from now SendOut(new DisableMessage(frontend)); -} + } bool ReaderBackend::DoHeartbeat(double network_time, double current_time) -{ + { MsgThread::DoHeartbeat(network_time, current_time); - return true; -} - -TransportProto ReaderBackend::StringToProto(const string &proto) { - if ( proto == "unknown" ) { - return TRANSPORT_UNKNOWN; - } else if ( proto == "tcp" ) { - return TRANSPORT_TCP; - } else if ( proto == "udp" ) { - return TRANSPORT_UDP; - } else if ( proto == "icmp" ) { - return TRANSPORT_ICMP; } +TransportProto ReaderBackend::StringToProto(const string &proto) + { + if ( proto == "unknown" ) + return TRANSPORT_UNKNOWN; + else if ( proto == "tcp" ) + return TRANSPORT_TCP; + else if ( proto == "udp" ) + return TRANSPORT_UDP; + else if ( proto == "icmp" ) + return TRANSPORT_ICMP; + Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str())); return TRANSPORT_UNKNOWN; -} + } // more or less verbose copy from IPAddr.cc -- which uses reporter -Value::addr_t ReaderBackend::StringToAddr(const string &s) { +Value::addr_t ReaderBackend::StringToAddr(const string &s) + { Value::addr_t val; if ( s.find(':') == std::string::npos ) // IPv4. { val.family = IPv4; - if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) { + if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 ) + { Error(Fmt("Bad addres: %s", s.c_str())); memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr)); - } + } } @@ -277,6 +289,6 @@ Value::addr_t ReaderBackend::StringToAddr(const string &s) { } return val; -} + } } diff --git a/src/input/ReaderFrontend.cc b/src/input/ReaderFrontend.cc index 6b3c2e6a67..f61fd357b9 100644 --- a/src/input/ReaderFrontend.cc +++ b/src/input/ReaderFrontend.cc @@ -46,19 +46,23 @@ public: }; -ReaderFrontend::ReaderFrontend(bro_int_t type) { +ReaderFrontend::ReaderFrontend(bro_int_t type) + { disabled = initialized = false; ty_name = ""; backend = input_mgr->CreateBackend(this, type); assert(backend); backend->Start(); -} + } -ReaderFrontend::~ReaderFrontend() { -} +ReaderFrontend::~ReaderFrontend() + { + } -void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, const threading::Field* const* fields) { +void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, + const threading::Field* const* fields) + { if ( disabled ) return; @@ -69,39 +73,43 @@ void ReaderFrontend::Init(string arg_source, int mode, const int num_fields, con initialized = true; backend->SendIn(new InitMessage(backend, arg_source, mode, num_fields, fields)); -} + } -void ReaderFrontend::Update() { +void ReaderFrontend::Update() + { if ( disabled ) return; - if ( !initialized ) { + if ( !initialized ) + { reporter->Error("Tried to call update on uninitialized reader"); return; - } + } backend->SendIn(new UpdateMessage(backend)); -} + } -void ReaderFrontend::Close() { +void ReaderFrontend::Close() + { if ( disabled ) return; - if ( !initialized ) { + if ( !initialized ) + { reporter->Error("Tried to call finish on uninitialized reader"); return; - } + } backend->SendIn(new CloseMessage(backend)); -} + } string ReaderFrontend::Name() const -{ + { if ( source.size() ) return ty_name; return ty_name + "/" + source; -} + } } diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc index a167408a0e..c798c21a5e 100644 --- a/src/input/readers/Ascii.cc +++ b/src/input/readers/Ascii.cc @@ -23,69 +23,73 @@ using threading::Field; FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) : name(arg_name), type(arg_type) -{ + { position = arg_position; secondary_position = -1; present = true; -} + } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, + const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) -{ + { position = arg_position; secondary_position = -1; present = true; -} + } FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype), present(arg.present) -{ + { position = arg.position; secondary_position = arg.secondary_position; -} + } -FieldMapping FieldMapping::subType() { +FieldMapping FieldMapping::subType() + { return FieldMapping(name, subtype, position); -} + } Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) -{ + { file = 0; - //keyMap = new map(); - separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); - if ( separator.size() != 1 ) { + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), + BifConst::InputAscii::separator->Len()); + if ( separator.size() != 1 ) Error("separator length has to be 1. Separator will be truncated."); - } - set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); - if ( set_separator.size() != 1 ) { + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), + BifConst::InputAscii::set_separator->Len()); + if ( set_separator.size() != 1 ) Error("set_separator length has to be 1. Separator will be truncated."); - } - empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); - - unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), + BifConst::InputAscii::empty_field->Len()); + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), + BifConst::InputAscii::unset_field->Len()); + } Ascii::~Ascii() -{ + { DoClose(); -} + } void Ascii::DoClose() -{ - if ( file != 0 ) { + { + if ( file != 0 ) + { file->close(); delete(file); file = 0; + } } -} bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) -{ + { fname = path; mode = arg_mode; mtime = 0; @@ -93,124 +97,135 @@ bool Ascii::DoInit(string path, int arg_mode, int arg_num_fields, const Field* c num_fields = arg_num_fields; fields = arg_fields; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; - } + } file = new ifstream(path.c_str()); - if ( !file->is_open() ) { + if ( !file->is_open() ) + { Error(Fmt("Init: cannot open %s", fname.c_str())); delete(file); file = 0; return false; - } + } - if ( ReadHeader(false) == false ) { + if ( ReadHeader(false) == false ) + { Error(Fmt("Init: cannot open %s; headers are incorrect", fname.c_str())); file->close(); delete(file); file = 0; return false; - } + } DoUpdate(); return true; -} + } -bool Ascii::ReadHeader(bool useCached) { +bool Ascii::ReadHeader(bool useCached) + { // try to read the header line... string line; map ifields; - if ( !useCached ) { - if ( !GetLine(line) ) { + if ( !useCached ) + { + if ( !GetLine(line) ) + { Error("could not read first line"); return false; - } - - + } headerline = line; - - } else { + } + else line = headerline; - } // construct list of field names. istringstream splitstream(line); int pos=0; - while ( splitstream ) { + while ( splitstream ) + { string s; if ( !getline(splitstream, s, separator[0])) break; ifields[s] = pos; pos++; - } + } //printf("Updating fields from description %s\n", line.c_str()); columnMap.clear(); - for ( unsigned int i = 0; i < num_fields; i++ ) { + for ( unsigned int i = 0; i < num_fields; i++ ) + { const Field* field = fields[i]; map::iterator fit = ifields.find(field->name); - if ( fit == ifields.end() ) { - if ( field->optional ) { + if ( fit == ifields.end() ) + { + if ( field->optional ) + { // we do not really need this field. mark it as not present and always send an undef back. FieldMapping f(field->name, field->type, field->subtype, -1); f.present = false; columnMap.push_back(f); continue; - } + } Error(Fmt("Did not find requested field %s in input data file %s.", field->name.c_str(), fname.c_str())); return false; - } + } FieldMapping f(field->name, field->type, field->subtype, ifields[field->name]); - if ( field->secondary_name != "" ) { + if ( field->secondary_name != "" ) + { map::iterator fit2 = ifields.find(field->secondary_name); - if ( fit2 == ifields.end() ) { + if ( fit2 == ifields.end() ) + { Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); return false; - } + } f.secondary_position = ifields[field->secondary_name]; - } + } columnMap.push_back(f); - } + } // well, that seems to have worked... return true; -} + } -bool Ascii::GetLine(string& str) { - while ( getline(*file, str) ) { - if ( str[0] != '#' ) { +bool Ascii::GetLine(string& str) + { + while ( getline(*file, str) ) + { + if ( str[0] != '#' ) return true; - } - if ( str.compare(0,8, "#fields\t") == 0 ) { + if ( str.compare(0,8, "#fields\t") == 0 ) + { str = str.substr(8); return true; + } } - } return false; -} - - -Value* Ascii::EntryToVal(string s, FieldMapping field) { - - if ( s.compare(unset_field) == 0 ) { // field is not set... - return new Value(field.type, false); } + +Value* Ascii::EntryToVal(string s, FieldMapping field) + { + + if ( s.compare(unset_field) == 0 ) // field is not set... + return new Value(field.type, false); + Value* val = new Value(field.type, true); switch ( field.type ) { @@ -220,14 +235,15 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { break; case TYPE_BOOL: - if ( s == "T" ) { + if ( s == "T" ) val->val.int_val = 1; - } else if ( s == "F" ) { + else if ( s == "F" ) val->val.int_val = 0; - } else { + else + { Error(Fmt("Field: %s Invalid value for boolean: %s", field.name.c_str(), s.c_str())); return false; - } + } break; case TYPE_INT: @@ -250,7 +266,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { val->val.port_val.proto = TRANSPORT_UNKNOWN; break; - case TYPE_SUBNET: { + case TYPE_SUBNET: + { size_t pos = s.find("/"); if ( pos == s.npos ) { Error(Fmt("Invalid value for subnet: %s", s.c_str())); @@ -261,8 +278,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.prefix = StringToAddr(addr); val->val.subnet_val.length = width; - } break; + } case TYPE_ADDR: val->val.addr_val = StringToAddr(s); @@ -287,47 +304,56 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { Value** lvals = new Value* [length]; - if ( field.type == TYPE_TABLE ) { + if ( field.type == TYPE_TABLE ) + { val->val.set_val.vals = lvals; val->val.set_val.size = length; - } else if ( field.type == TYPE_VECTOR ) { + } + else if ( field.type == TYPE_VECTOR ) + { val->val.vector_val.vals = lvals; val->val.vector_val.size = length; - } else { + } + else + { assert(false); - } + } if ( length == 0 ) break; //empty istringstream splitstream(s); - while ( splitstream ) { + while ( splitstream ) + { string element; if ( !getline(splitstream, element, set_separator[0]) ) break; - if ( pos >= length ) { - Error(Fmt("Internal error while parsing set. pos %d >= length %d. Element: %s", pos, length, element.c_str())); + if ( pos >= length ) + { + Error(Fmt("Internal error while parsing set. pos %d >= length %d." + " Element: %s", pos, length, element.c_str())); break; - } + } Value* newval = EntryToVal(element, field.subType()); - if ( newval == 0 ) { + if ( newval == 0 ) + { Error("Error while reading set"); return 0; - } + } lvals[pos] = newval; pos++; - - } + } - if ( pos != length ) { + if ( pos != length ) + { Error("Internal error while parsing set: did not find all elements"); return 0; - } + } break; } @@ -340,24 +366,23 @@ Value* Ascii::EntryToVal(string s, FieldMapping field) { } return val; - -} + } // read the entire file and send appropriate thingies back to InputMgr -bool Ascii::DoUpdate() { +bool Ascii::DoUpdate() + { switch ( mode ) { case REREAD: // check if the file has changed struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { + if ( stat(fname.c_str(), &sb) == -1 ) + { Error(Fmt("Could not get stat for %s", fname.c_str())); return false; - } + } - if ( sb.st_mtime <= mtime ) { - // no change + if ( sb.st_mtime <= mtime ) // no change return true; - } mtime = sb.st_mtime; // file changed. reread. @@ -366,57 +391,56 @@ bool Ascii::DoUpdate() { case MANUAL: case STREAM: - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) - if ( file && file->is_open() ) { - if ( mode == STREAM ) { + // dirty, fix me. (well, apparently after trying seeking, etc + // - this is not that bad) + if ( file && file->is_open() ) + { + if ( mode == STREAM ) + { file->clear(); // remove end of file evil bits if ( !ReadHeader(true) ) // in case filters changed - { return false; // header reading failed - } + break; - } + } file->close(); - } + } file = new ifstream(fname.c_str()); - if ( !file->is_open() ) { + if ( !file->is_open() ) + { Error(Fmt("cannot open %s", fname.c_str())); return false; - } + } - if ( ReadHeader(false) == false ) { + if ( ReadHeader(false) == false ) + { return false; - } + } break; default: assert(false); - } - - - // - - // file->seekg(0, ios::beg); // do not forget clear. - - + } string line; - while ( GetLine(line ) ) { + while ( GetLine(line ) ) + { // split on tabs istringstream splitstream(line); map stringfields; int pos = 0; - while ( splitstream ) { + while ( splitstream ) + { string s; if ( !getline(splitstream, s, separator[0]) ) break; stringfields[pos] = s; pos++; - } + } pos--; // for easy comparisons of max element. @@ -426,69 +450,60 @@ bool Ascii::DoUpdate() { int fpos = 0; for ( vector::iterator fit = columnMap.begin(); fit != columnMap.end(); - fit++ ){ + fit++ ) + { - if ( ! fit->present ) { + if ( ! fit->present ) + { // add non-present field fields[fpos] = new Value((*fit).type, false); fpos++; continue; - } + } assert(fit->position >= 0 ); - if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + if ( (*fit).position > pos || (*fit).secondary_position > pos ) + { Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; - } + } Value* val = EntryToVal(stringfields[(*fit).position], *fit); - if ( val == 0 ) { + if ( val == 0 ) + { Error("Could not convert String value to Val"); return false; - } + } - if ( (*fit).secondary_position != -1 ) { + if ( (*fit).secondary_position != -1 ) + { // we have a port definition :) assert(val->type == TYPE_PORT ); // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]); - } + } fields[fpos] = val; fpos++; - } + } //printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields); assert ( (unsigned int) fpos == num_fields ); - if ( mode == STREAM ) { + if ( mode == STREAM ) Put(fields); - } else { + else SendEntry(fields); } - /* Do not do this, ownership changes to other thread - * for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - delete fields[i]; - } - delete [] fields; - */ - - } - - - //file->clear(); // remove end of file evil bits - //file->seekg(0, ios::beg); // and seek to start. - - if ( mode != STREAM ) { + if ( mode != STREAM ) EndCurrentSend(); - } - + return true; -} + } bool Ascii::DoHeartbeat(double network_time, double current_time) { @@ -500,12 +515,13 @@ bool Ascii::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - Update(); // call update and not DoUpdate, because update actually checks disabled. + Update(); // call update and not DoUpdate, because update + // checks disabled. break; default: assert(false); } return true; -} + } diff --git a/src/input/readers/Benchmark.cc b/src/input/readers/Benchmark.cc index deff2b038d..29f0070fec 100644 --- a/src/input/readers/Benchmark.cc +++ b/src/input/readers/Benchmark.cc @@ -22,7 +22,7 @@ using threading::Field; Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) -{ + { multiplication_factor = double(BifConst::InputBenchmark::factor); autospread = double(BifConst::InputBenchmark::autospread); spread = int(BifConst::InputBenchmark::spread); @@ -32,19 +32,19 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend) timedspread = double(BifConst::InputBenchmark::timedspread); heart_beat_interval = double(BifConst::Threading::heart_beat_interval); -} + } Benchmark::~Benchmark() -{ + { DoClose(); -} + } void Benchmark::DoClose() -{ -} + { + } bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) -{ + { mode = arg_mode; num_fields = arg_num_fields; @@ -54,18 +54,20 @@ bool Benchmark::DoInit(string path, int arg_mode, int arg_num_fields, const Fiel if ( autospread != 0.0 ) autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + { Error(Fmt("Unsupported read mode %d for source %s", mode, path.c_str())); return false; - } + } heartbeatstarttime = CurrTime(); DoUpdate(); return true; -} + } -string Benchmark::RandomString(const int len) { +string Benchmark::RandomString(const int len) + { string s(len, ' '); static const char values[] = @@ -73,65 +75,65 @@ string Benchmark::RandomString(const int len) { "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz"; - for (int i = 0; i < len; ++i) { + for (int i = 0; i < len; ++i) s[i] = values[rand() / (RAND_MAX / sizeof(values))]; - } return s; -} + } -double Benchmark::CurrTime() { +double Benchmark::CurrTime() + { struct timeval tv; assert ( gettimeofday(&tv, 0) >= 0 ); return double(tv.tv_sec) + double(tv.tv_usec) / 1e6; -} + } // read the entire file and send appropriate thingies back to InputMgr -bool Benchmark::DoUpdate() { +bool Benchmark::DoUpdate() + { int linestosend = num_lines * heart_beat_interval; - for ( int i = 0; i < linestosend; i++ ) { + for ( int i = 0; i < linestosend; i++ ) + { Value** field = new Value*[num_fields]; - for (unsigned int j = 0; j < num_fields; j++ ) { + for (unsigned int j = 0; j < num_fields; j++ ) field[j] = EntryToVal(fields[j]->type, fields[j]->subtype); - } - if ( mode == STREAM ) { + if ( mode == STREAM ) // do not do tracking, spread out elements over the second that we have... Put(field); - } else { + else SendEntry(field); - } - if ( stopspreadat == 0 || num_lines < stopspreadat ) { + if ( stopspreadat == 0 || num_lines < stopspreadat ) + { if ( spread != 0 ) usleep(spread); if ( autospread_time != 0 ) usleep( autospread_time ); - } + } - if ( timedspread != 0.0 ) { + if ( timedspread != 0.0 ) + { double diff; - do { + do diff = CurrTime() - heartbeatstarttime; - //printf("%d %f\n", i, diff); - //} while ( diff < i/threading::Manager::HEART_BEAT_INTERVAL*(num_lines + (num_lines * timedspread) ) ); - } while ( diff/heart_beat_interval < i/(linestosend + (linestosend * timedspread) ) ); - //} while ( diff < 0.8); - } + while ( diff/heart_beat_interval < i/(linestosend + + (linestosend * timedspread) ) ); + } } - if ( mode != STREAM ) { + if ( mode != STREAM ) EndCurrentSend(); - } return true; } -threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { +threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) + { Value* val = new Value(type, true); // basically construct something random from the fields that we want. @@ -170,7 +172,8 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { val->val.port_val.proto = TRANSPORT_UNKNOWN; break; - case TYPE_SUBNET: { + case TYPE_SUBNET: + { val->val.subnet_val.prefix = StringToAddr("192.168.17.1"); val->val.subnet_val.length = 16; } @@ -192,28 +195,32 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { Value** lvals = new Value* [length]; - if ( type == TYPE_TABLE ) { + if ( type == TYPE_TABLE ) + { val->val.set_val.vals = lvals; val->val.set_val.size = length; - } else if ( type == TYPE_VECTOR ) { + } + else if ( type == TYPE_VECTOR ) + { val->val.vector_val.vals = lvals; val->val.vector_val.size = length; - } else { + } + else assert(false); - } if ( length == 0 ) break; //empty - for ( unsigned int pos = 0; pos < length; pos++ ) { - + for ( unsigned int pos = 0; pos < length; pos++ ) + { Value* newval = EntryToVal(subtype, TYPE_ENUM); - if ( newval == 0 ) { + if ( newval == 0 ) + { Error("Error while reading set"); return 0; - } + } lvals[pos] = newval; - } + } break; } @@ -226,20 +233,11 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype) { return val; -} + } bool Benchmark::DoHeartbeat(double network_time, double current_time) { - /* - * This does not work the way I envisioned it, because the queueing is the problem. - printf("%f\n", CurrTime() - current_time); - if ( CurrTime() - current_time > 0.25 ) { - // event has hung for a time. refuse. - SendEvent("EndBenchmark", 0, 0); - return true; - } */ - ReaderBackend::DoHeartbeat(network_time, current_time); num_lines = (int) ( (double) num_lines*multiplication_factor); num_lines += add; @@ -251,7 +249,8 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - if ( multiplication_factor != 1 || add != 0 ) { + if ( multiplication_factor != 1 || add != 0 ) + { // we have to document at what time we changed the factor to what value. Value** v = new Value*[2]; v[0] = new Value(TYPE_COUNT, true); @@ -260,12 +259,11 @@ bool Benchmark::DoHeartbeat(double network_time, double current_time) v[1]->val.double_val = CurrTime(); SendEvent("lines_changed", 2, v); - } + } - if ( autospread != 0.0 ) { - autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); + if ( autospread != 0.0 ) // because executing this in every loop is apparently too expensive. - } + autospread_time = (int) ( (double) 1000000 / (autospread * (double) num_lines) ); Update(); // call update and not DoUpdate, because update actually checks disabled. diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index f656be769c..43c782de29 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -24,79 +24,86 @@ using threading::Value; using threading::Field; Raw::Raw(ReaderFrontend *frontend) : ReaderBackend(frontend) -{ + { file = 0; in = 0; - //keyMap = new map(); - separator.assign( (const char*) BifConst::InputRaw::record_separator->Bytes(), BifConst::InputRaw::record_separator->Len()); - if ( separator.size() != 1 ) { + if ( separator.size() != 1 ) Error("separator length has to be 1. Separator will be truncated."); } -} - Raw::~Raw() -{ + { DoClose(); -} + } void Raw::DoClose() -{ - if ( file != 0 ) { + { + if ( file != 0 ) + { Close(); + } } -} bool Raw::Open() -{ - if ( execute ) { + { + if ( execute ) + { file = popen(fname.c_str(), "r"); - if ( file == NULL ) { + if ( file == NULL ) + { Error(Fmt("Could not execute command %s", fname.c_str())); return false; + } } - } else { + else + { file = fopen(fname.c_str(), "r"); - if ( file == NULL ) { + if ( file == NULL ) + { Error(Fmt("Init: cannot open %s", fname.c_str())); return false; + } } - } in = new boost::fdistream(fileno(file)); - if ( execute && mode == STREAM ) { + if ( execute && mode == STREAM ) + { fcntl(fileno(file), F_SETFL, O_NONBLOCK); - } + } return true; -} + } bool Raw::Close() -{ - if ( file == NULL ) { + { + if ( file == NULL ) + { InternalError(Fmt("Trying to close closed file for stream %s", fname.c_str())); return false; - } + } - if ( execute ) { + if ( execute ) + { delete(in); pclose(file); - } else { + } + else + { delete(in); fclose(file); - } + } in = NULL; file = NULL; return true; -} + } bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* const* arg_fields) -{ + { fname = path; mode = arg_mode; mtime = 0; @@ -107,24 +114,30 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con num_fields = arg_num_fields; fields = arg_fields; - if ( path.length() == 0 ) { + if ( path.length() == 0 ) + { Error("No source path provided"); return false; - } + } - if ( arg_num_fields != 1 ) { - Error("Filter for raw reader contains more than one field. Filters for the raw reader may only contain exactly one string field. Filter ignored."); + if ( arg_num_fields != 1 ) + { + Error("Filter for raw reader contains more than one field. " + "Filters for the raw reader may only contain exactly one string field. " + "Filter ignored."); return false; - } + } - if ( fields[0]->type != TYPE_STRING ) { + if ( fields[0]->type != TYPE_STRING ) + { Error("Filter for raw reader contains a field that is not of type string."); return false; - } + } // do Initialization char last = path[path.length()-1]; - if ( last == '|' ) { + if ( last == '|' ) + { execute = true; fname = path.substr(0, fname.length() - 1); @@ -137,19 +150,17 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con } else { execute = false; - if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) { + if ( ( mode != MANUAL ) && (mode != REREAD) && ( mode != STREAM ) ) + { Error(Fmt("Unsupported read mode %d for source %s", mode, fname.c_str())); return false; - } + } result = Open(); + } - } - - if ( result == false ) { + if ( result == false ) return result; - } - #ifdef DEBUG Debug(DBG_INPUT, "Raw reader created, will perform first update"); @@ -162,62 +173,68 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con Debug(DBG_INPUT, "First update went through"); #endif return true; -} - - -bool Raw::GetLine(string& str) { - while ( getline(*in, str, separator[0]) ) { - return true; } + +bool Raw::GetLine(string& str) + { + while ( getline(*in, str, separator[0]) ) + return true; + return false; -} + } // read the entire file and send appropriate thingies back to InputMgr -bool Raw::DoUpdate() { - if ( firstrun ) { +bool Raw::DoUpdate() + { + if ( firstrun ) firstrun = false; - } else { + else + { switch ( mode ) { case REREAD: + { // check if the file has changed struct stat sb; - if ( stat(fname.c_str(), &sb) == -1 ) { + if ( stat(fname.c_str(), &sb) == -1 ) + { Error(Fmt("Could not get stat for %s", fname.c_str())); return false; - } + } - if ( sb.st_mtime <= mtime ) { + if ( sb.st_mtime <= mtime ) // no change return true; - } mtime = sb.st_mtime; // file changed. reread. // fallthrough + } case MANUAL: case STREAM: - if ( mode == STREAM && file != NULL && in != NULL ) { + if ( mode == STREAM && file != NULL && in != NULL ) + { //fpurge(file); in->clear(); // remove end of file evil bits break; - } + } Close(); - if ( !Open() ) { + if ( !Open() ) return false; - } + break; default: assert(false); } - } + } string line; - while ( GetLine(line) ) { + while ( GetLine(line) ) + { assert (num_fields == 1); Value** fields = new Value*[1]; @@ -228,14 +245,14 @@ bool Raw::DoUpdate() { fields[0] = val; Put(fields); - } + } return true; -} + } bool Raw::DoHeartbeat(double network_time, double current_time) -{ + { ReaderBackend::DoHeartbeat(network_time, current_time); switch ( mode ) { @@ -244,12 +261,12 @@ bool Raw::DoHeartbeat(double network_time, double current_time) break; case REREAD: case STREAM: - Update(); // call update and not DoUpdate, because update actually checks disabled. + Update(); // call update and not DoUpdate, because update + // checks disabled. break; default: assert(false); } return true; -} - + } From 2e452dc29ff39c557d7de5e6b21af93ce2e80690 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 10:49:17 -0700 Subject: [PATCH 120/123] remove last remnants of autostart, which has been removed for quite a while. --- scripts/base/frameworks/input/main.bro | 6 ------ src/input/Manager.cc | 1 - src/input/ReaderBackend.h | 1 - 3 files changed, 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e06dfae005..a52cd97b4b 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -24,9 +24,6 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; - ## Automatically start the input stream after the first filter has been added - autostart: bool &default=T; - ## Descriptive name. Used to remove a filter at a later time name: string; @@ -68,9 +65,6 @@ export { ## Read mode to use for this stream mode: Mode &default=default_mode; - ## Automatically start the input stream after the first filter has been added - autostart: bool &default=T; - ## Descriptive name. Used to remove a filter at a later time name: string; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index 0fde16b87d..3f7fcea078 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -287,7 +287,6 @@ bool Manager::CreateStream(Stream* info, RecordVal* description) } EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); - Val *autostart = description->LookupWithDefault(rtype->FieldOffset("autostart")); ReaderFrontend* reader_obj = new ReaderFrontend(reader->InternalInt()); assert(reader_obj); diff --git a/src/input/ReaderBackend.h b/src/input/ReaderBackend.h index 5b230ca652..ca54d8a204 100644 --- a/src/input/ReaderBackend.h +++ b/src/input/ReaderBackend.h @@ -227,7 +227,6 @@ private: // For implementing Fmt(). char* buf; unsigned int buf_len; - bool autostart; unsigned int num_fields; const threading::Field* const * fields; // raw mapping From 96a7e068f085291c3ee7db0bca39cb1df18055ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 11:29:57 -0700 Subject: [PATCH 121/123] baselines for the autostart removal. --- .../scripts.base.frameworks.input.event/out | 14 ++++---- .../out | 18 +++++------ .../scripts.base.frameworks.input.raw/out | 16 +++++----- .../scripts.base.frameworks.input.reread/out | 16 +++++----- .../out | 32 +++++++++---------- .../out | 16 +++++----- .../out | 14 ++++---- .../out | 6 ++-- 8 files changed, 66 insertions(+), 66 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out index 59070cd88e..bb3b6d0a9e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.event/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -8,7 +8,7 @@ print A::b; Input::EVENT_NEW 1 T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -18,7 +18,7 @@ print A::b; Input::EVENT_NEW 2 T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -28,7 +28,7 @@ print A::b; Input::EVENT_NEW 3 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -38,7 +38,7 @@ print A::b; Input::EVENT_NEW 4 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -48,7 +48,7 @@ print A::b; Input::EVENT_NEW 5 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -58,7 +58,7 @@ print A::b; Input::EVENT_NEW 6 F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out index 06e28de441..bb69da3267 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executestreamraw/out @@ -1,4 +1,4 @@ -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -14,7 +14,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -30,7 +30,7 @@ Input::remove(input); }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -46,7 +46,7 @@ Input::remove(input); }] Input::EVENT_NEW q3r3057fdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -62,7 +62,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfs\d -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -78,7 +78,7 @@ Input::remove(input); }] Input::EVENT_NEW -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -94,7 +94,7 @@ Input::remove(input); }] Input::EVENT_NEW dfsdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -110,7 +110,7 @@ Input::remove(input); }] Input::EVENT_NEW sdf -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -126,7 +126,7 @@ Input::remove(input); }] Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. -[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=tail -f ../input.log |, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out index 34a5599dc9..55e7610e1e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.raw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.raw/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -6,7 +6,7 @@ print A::s; }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -14,7 +14,7 @@ print A::s; }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -22,7 +22,7 @@ print A::s; }] Input::EVENT_NEW q3r3057fdf -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -30,7 +30,7 @@ print A::s; }] Input::EVENT_NEW sdfs\d -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -38,7 +38,7 @@ print A::s; }] Input::EVENT_NEW -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -46,7 +46,7 @@ print A::s; }] Input::EVENT_NEW dfsdf -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -54,7 +54,7 @@ print A::s; }] Input::EVENT_NEW sdf -[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out index 46a30f387f..5cce15f6c7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out @@ -15,7 +15,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -96,7 +96,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -201,7 +201,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -366,7 +366,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -489,7 +489,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -612,7 +612,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -735,7 +735,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -858,7 +858,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out index d85c8f2e83..9d62fdbef4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.rereadraw/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -6,7 +6,7 @@ print A::s; }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -14,7 +14,7 @@ print A::s; }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -22,7 +22,7 @@ print A::s; }] Input::EVENT_NEW q3r3057fdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -30,7 +30,7 @@ print A::s; }] Input::EVENT_NEW sdfs\d -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -38,7 +38,7 @@ print A::s; }] Input::EVENT_NEW -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -46,7 +46,7 @@ print A::s; }] Input::EVENT_NEW dfsdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -54,7 +54,7 @@ print A::s; }] Input::EVENT_NEW sdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -62,7 +62,7 @@ print A::s; }] Input::EVENT_NEW 3rw43wRRERLlL#RWERERERE. -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -70,7 +70,7 @@ print A::s; }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -78,7 +78,7 @@ print A::s; }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -86,7 +86,7 @@ print A::s; }] Input::EVENT_NEW q3r3057fdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -94,7 +94,7 @@ print A::s; }] Input::EVENT_NEW sdfs\d -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -102,7 +102,7 @@ print A::s; }] Input::EVENT_NEW -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -110,7 +110,7 @@ print A::s; }] Input::EVENT_NEW dfsdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; @@ -118,7 +118,7 @@ print A::s; }] Input::EVENT_NEW sdf -[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, autostart=T, name=input, fields=, want_record=F, ev=line +[source=input.log, reader=Input::READER_RAW, mode=Input::REREAD, name=input, fields=, want_record=F, ev=line { print A::description; print A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out index 937acf428e..07a3ffdba5 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.streamraw/out @@ -1,4 +1,4 @@ -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -13,7 +13,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -28,7 +28,7 @@ Input::remove(input); }] Input::EVENT_NEW DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -43,7 +43,7 @@ Input::remove(input); }] Input::EVENT_NEW q3r3057fdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -58,7 +58,7 @@ Input::remove(input); }] Input::EVENT_NEW sdfs\d -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -73,7 +73,7 @@ Input::remove(input); }] Input::EVENT_NEW -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -88,7 +88,7 @@ Input::remove(input); }] Input::EVENT_NEW dfsdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; @@ -103,7 +103,7 @@ Input::remove(input); }] Input::EVENT_NEW sdf -[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, autostart=T, name=input, fields=, want_record=F, ev=line +[source=../input.log, reader=Input::READER_RAW, mode=Input::STREAM, name=input, fields=, want_record=F, ev=line { print A::outfile, A::description; print A::outfile, A::tpe; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index 56b36a1a0e..a1bbb9bbe4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,4 +1,4 @@ -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -16,7 +16,7 @@ print right; Input::EVENT_NEW [i=1] T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -34,7 +34,7 @@ print right; Input::EVENT_NEW [i=2] T -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -52,7 +52,7 @@ print right; Input::EVENT_NEW [i=3] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -70,7 +70,7 @@ print right; Input::EVENT_NEW [i=4] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -88,7 +88,7 @@ print right; Input::EVENT_NEW [i=5] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, @@ -106,7 +106,7 @@ print right; Input::EVENT_NEW [i=6] F -[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, autostart=T, name=input, destination={ +[source=input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ [2] = T, [4] = F, [6] = F, diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out index a61a4a2993..41d9438da0 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twotables/out @@ -30,7 +30,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -120,7 +120,7 @@ BB } ============EVENT============ Description -[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh2, destination={ +[source=../input2.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh2, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, @@ -240,7 +240,7 @@ BB }, vc=[10, 20, 30], ve=[]] ============EVENT============ Description -[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, autostart=T, name=ssh, destination={ +[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={ [-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ 2, 4, From 4de6d76488e0d85f7085aa53528bee93b7d7b8b7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 11:30:18 -0700 Subject: [PATCH 122/123] fix up the executeraw test - now it works for the first time and does not always fail --- src/input/readers/Raw.cc | 10 ++++++++++ .../scripts.base.frameworks.input.executeraw/out | 9 +++++++++ .../scripts/base/frameworks/input/executeraw.bro | 16 ++++++++++------ 3 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 43c782de29..ce0b4f8a5f 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -178,6 +178,12 @@ bool Raw::DoInit(string path, int arg_mode, int arg_num_fields, const Field* con bool Raw::GetLine(string& str) { + if ( in->peek() == std::iostream::traits_type::eof() ) + return false; + + if ( in->eofbit == true || in->failbit == true ) + return false; + while ( getline(*in, str, separator[0]) ) return true; @@ -247,6 +253,10 @@ bool Raw::DoUpdate() Put(fields); } +#ifdef DEBUG + Debug(DBG_INPUT, "DoUpdate finished successfully"); +#endif + return true; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out new file mode 100644 index 0000000000..8611b35dd3 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.executeraw/out @@ -0,0 +1,9 @@ +[source=wc -l ../input.log |, reader=Input::READER_RAW, mode=Input::MANUAL, name=input, fields=, want_record=F, ev=line +{ +print outfile, description; +print outfile, tpe; +print outfile, s; +close(outfile); +}] +Input::EVENT_NEW + 8 ../input.log diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 6fceebf885..6d07a9bf29 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,5 +1,6 @@ # -# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait -k 1 # @TEST-EXEC: btest-diff out @TEST-START-FILE input.log @@ -13,21 +14,24 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen -module A; +global outfile: file; type Val: record { s: string; }; event line(description: Input::EventDescription, tpe: Input::Event, s: string) { - print description; - print tpe; - print s; + print outfile, description; + print outfile, tpe; + print outfile, s; + close(outfile); } event bro_init() { - Input::add_event([$source="wc input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); + outfile = open ("../out"); + Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line]); Input::remove("input"); } From 24173807ea8b1c895bc9ee75cb9924ba198901ea Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 25 May 2012 11:35:56 -0700 Subject: [PATCH 123/123] reactivate network_time check in threading manager. previously this line made all input framework tests fail - it works now. Some of the other recent changes of the threading manager must have fixed that problem. This was easy :) --- src/threading/Manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 491b8379e8..4a05fb8d41 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -125,7 +125,7 @@ void Manager::Process() if ( msg->Process() ) { - //if ( network_time ) // FIXME: ask robin again if he needs this. makes input interface not work in bro_init. + if ( network_time ) did_process = true; }