From 6e6073ff4c4e01ab3171d3724dd6fe5f44df463e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 17 Oct 2011 14:19:17 -0700 Subject: [PATCH 01/58] it compiles (but doesn't do anything useful) --- src/CMakeLists.txt | 4 ++ src/Func.cc | 3 ++ src/InputMgr.cc | 90 +++++++++++++++++++++++++++++++++++++++++ src/InputMgr.h | 24 +++++++++++ src/InputReader.cc | 12 ++++++ src/InputReader.h | 29 +++++++++++++ src/InputReaderAscii.cc | 12 ++++++ src/InputReaderAscii.h | 20 +++++++++ src/NetVar.cc | 2 + src/input.bif | 16 ++++++++ src/main.cc | 4 ++ src/types.bif | 7 ++++ 12 files changed, 223 insertions(+) create mode 100644 src/InputMgr.cc create mode 100644 src/InputMgr.h create mode 100644 src/InputReader.cc create mode 100644 src/InputReader.h create mode 100644 src/InputReaderAscii.cc create mode 100644 src/InputReaderAscii.h create mode 100644 src/input.bif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b4779e1557..1693bad4eb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -141,6 +141,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif logging.bif + input.bif event.bif const.bif types.bif @@ -332,6 +333,9 @@ set(bro_SRCS IRC.cc List.cc Reporter.cc + InputMgr.cc + InputReader.cc + InputReaderAscii.cc LogMgr.cc LogWriter.cc LogWriterAscii.cc diff --git a/src/Func.cc b/src/Func.cc index 65cb22b09d..829bc89238 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -523,11 +523,13 @@ void builtin_error(const char* msg, BroObj* arg) #include "bro.bif.func_h" #include "logging.bif.func_h" +#include "input.bif.func_h" #include "reporter.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" #include "logging.bif.func_def" +#include "input.bif.func_def" #include "reporter.bif.func_def" #include "strings.bif.func_def" @@ -542,6 +544,7 @@ void init_builtin_funcs() #include "bro.bif.func_init" #include "logging.bif.func_init" +#include "input.bif.func_init" #include "reporter.bif.func_init" #include "strings.bif.func_init" diff --git a/src/InputMgr.cc b/src/InputMgr.cc new file mode 100644 index 0000000000..cacc512bcf --- /dev/null +++ b/src/InputMgr.cc @@ -0,0 +1,90 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include + +#include "InputMgr.h" +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" + + +#include "InputReader.h" + +#include "InputReaderAscii.h" + + +struct InputReaderDefinition { + bro_int_t type; // the type + const char *name; // descriptive name for error messages + bool (*init)(); // optional one-time inifializing function + InputReader* (*factory)(); // factory function for creating instances +}; + +InputReaderDefinition input_readers[] = { + { BifEnum::Input::READER_ASCII, "Ascii", 0, InputReaderAscii::Instantiate }, + + // End marker + { BifEnum::Input::READER_DEFAULT, "None", 0, (InputReader* (*)())0 } +}; + +InputMgr::InputMgr() +{ + DBG_LOG(DBG_LOGGING, "this has to happen"); +} + + +// create a new input reader object to be used at whomevers leisure lateron. +InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +{ + InputReaderDefinition* ir = input_readers; + exit(12); + + while ( true ) { + if ( ir->type == BifEnum::Input::READER_DEFAULT ) + { + DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); + reporter->Error("unknown reader when creating reader"); + return 0; + } + + if ( ir->type != reader->AsEnum() ) { + // no, didn't find the right one... + ++ir; + continue; + } + + + // call init function of writer if presnt + if ( ir->init ) + { + if ( (*ir->init)() ) + { + //clear it to be not called again + ir->init = 0; + } else { + // ohok. init failed, kill factory for all eternity + ir->factory = 0; + DBG_LOG(DBG_LOGGING, "failed to init input class %s", ir->name); + return 0; + } + + } + + if ( !ir->factory ) + // no factory? + return 0; + + // all done. break. + break; + } + + assert(ir->factory); + InputReader* reader_obj = (*ir->factory)(); + assert(reader_obj); + + return reader_obj; + +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h new file mode 100644 index 0000000000..978481afba --- /dev/null +++ b/src/InputMgr.h @@ -0,0 +1,24 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef INPUTMGR_H +#define INPUTMGR_H + +#include "InputReader.h" +#include "BroString.h" + +#include "Val.h" +#include "EventHandler.h" +#include "RemoteSerializer.h" + + +class InputMgr { +public: + InputMgr(); + + InputReader* CreateReader(EnumVal* reader, string source); +}; + +extern InputMgr* input_mgr; + + +#endif /* INPUTMGR_H */ diff --git a/src/InputReader.cc b/src/InputReader.cc new file mode 100644 index 0000000000..f2b3b05801 --- /dev/null +++ b/src/InputReader.cc @@ -0,0 +1,12 @@ + +#include "InputReader.h" + +InputReader::InputReader() +{ + +} + +InputReader::~InputReader() +{ + +} \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h new file mode 100644 index 0000000000..ce8303383a --- /dev/null +++ b/src/InputReader.h @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Same notes about thread safety as in LogWriter.h apply. + + +#ifndef INPUTREADER_H +#define INPUTREADER_H + +class InputReader { +public: + InputReader(); + virtual ~InputReader(); + +protected: + // Methods that have to be overwritten by the individual readers + +private: + friend class InputMgr; + + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. + + bool disabled; + + bool Disabled() { return disabled; } +}; + + +#endif /* INPUTREADER_H */ diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc new file mode 100644 index 0000000000..97933c2a6e --- /dev/null +++ b/src/InputReaderAscii.cc @@ -0,0 +1,12 @@ + +#include "InputReaderAscii.h" +#include "DebugLogger.h" + +InputReaderAscii::InputReaderAscii() +{ + DBG_LOG(DBG_LOGGING, "input reader initialized"); +} + +InputReaderAscii::~InputReaderAscii() +{ +} \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h new file mode 100644 index 0000000000..c471639fd4 --- /dev/null +++ b/src/InputReaderAscii.h @@ -0,0 +1,20 @@ + +#ifndef INPUTREADERASCII_H +#define INPUTREADERASCII_H + +#include "InputReader.h" + +class InputReaderAscii : public InputReader { +public: + InputReaderAscii(); + ~InputReaderAscii(); + + static InputReader* Instantiate() { return new InputReaderAscii; } + +protected: + +private: +}; + + +#endif /* INPUTREADERASCII_H */ diff --git a/src/NetVar.cc b/src/NetVar.cc index 25e4f7a0bc..5a6ac96fba 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -255,6 +255,7 @@ StringVal* cmd_line_bpf_filter; #include "types.bif.netvar_def" #include "event.bif.netvar_def" #include "logging.bif.netvar_def" +#include "input.bif.netvar_def" #include "reporter.bif.netvar_def" void init_event_handlers() @@ -315,6 +316,7 @@ void init_net_var() #include "const.bif.netvar_init" #include "types.bif.netvar_init" #include "logging.bif.netvar_init" +#include "input.bif.netvar_init" #include "reporter.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); diff --git a/src/input.bif b/src/input.bif new file mode 100644 index 0000000000..edb6b4e9bb --- /dev/null +++ b/src/input.bif @@ -0,0 +1,16 @@ +# functions and types for the input framework + +module Input; + +%%{ +#include "InputMgr.h" +#include "NetVar.h" +%%} + +function Input::__create_reader%(reader: Input::Reader, source: string%) : bool + %{ + exit(5); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + return new Val( the_reader != 0, TYPE_BOOL ); + %} + diff --git a/src/main.cc b/src/main.cc index dfa46c3050..b3f2512b40 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "File.h" #include "Reporter.h" #include "LogMgr.h" +#include "InputMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -72,6 +73,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; LogMgr* log_mgr; +InputMgr* input_mgr; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -724,6 +726,8 @@ int main(int argc, char** argv) remote_serializer = new RemoteSerializer(); event_registry = new EventRegistry(); log_mgr = new LogMgr(); + + input_mgr = new InputMgr(); if ( events_file ) event_player = new EventPlayer(events_file); diff --git a/src/types.bif b/src/types.bif index da6bd6e031..ee43207ddd 100644 --- a/src/types.bif +++ b/src/types.bif @@ -167,4 +167,11 @@ enum ID %{ Unknown, %} +module Input; + +enum Reader %{ + READER_DEFAULT, + READER_ASCII, +%} + module GLOBAL; From 0eafeb03693e3134bb47219df43066a390a42d55 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 13:16:09 -0700 Subject: [PATCH 02/58] works (thanks to robin) --- scripts/base/init-bare.bro | 2 ++ src/NetVar.h | 1 + src/input.bif | 1 - 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 859a69f2dc..8d1d2a312f 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1508,3 +1508,5 @@ const parse_udp_tunnels = F &redef; # Load the logging framework here because it uses fairly deep integration with # BiFs and script-land defined types. @load base/frameworks/logging + +@load base/input.bif diff --git a/src/NetVar.h b/src/NetVar.h index f8def230c0..957a86aeb3 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,6 +264,7 @@ extern void init_net_var(); #include "types.bif.netvar_h" #include "event.bif.netvar_h" #include "logging.bif.netvar_h" +#include "input.bif.netvar_h" #include "reporter.bif.netvar_h" #endif diff --git a/src/input.bif b/src/input.bif index edb6b4e9bb..3ff5284c63 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,7 +9,6 @@ module Input; function Input::__create_reader%(reader: Input::Reader, source: string%) : bool %{ - exit(5); InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); return new Val( the_reader != 0, TYPE_BOOL ); %} From f8be3519c7b868cd23829141b9588396df50e6ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 15:41:07 -0700 Subject: [PATCH 03/58] well, it compiles. and perhaps it sends an event. billiant. --- src/InputMgr.cc | 20 +++++++++++++++++--- src/InputMgr.h | 10 +++++++++- src/InputReader.cc | 19 +++++++++++++++++++ src/InputReader.h | 15 ++++++++++++++- src/input.bif | 6 ++++-- 5 files changed, 63 insertions(+), 7 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index cacc512bcf..5ab602f4f3 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,17 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) { InputReaderDefinition* ir = input_readers; - exit(12); - + + RecordType* rtype = eventDescription->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + { + reporter->Error("eventDescription argument not of right type"); + return 0; + } + while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { @@ -86,5 +92,13 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source) return reader_obj; } + +void InputMgr::Error(InputReader* reader, const char* msg) +{ + reporter->Error(fmt("error with input reader for %s: %s", + reader->Source().c_str(), msg)); +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 978481afba..3d3d0f2d3f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -10,12 +10,20 @@ #include "EventHandler.h" #include "RemoteSerializer.h" +class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source); + InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + +protected: + friend class InputReader; + + // Reports an error for the given reader. + void Error(InputReader* reader, const char* msg); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index f2b3b05801..ef47bb1e10 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,5 +1,7 @@ #include "InputReader.h" +#include "EventRegistry.h" +#include "Event.h" InputReader::InputReader() { @@ -9,4 +11,21 @@ InputReader::InputReader() InputReader::~InputReader() { +} + +void InputReader::Error(const char *msg) +{ + input_mgr->Error(this, msg); +} + +bool InputReader::Init(string source, string eventName) { + EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + if ( handler == 0 ) { + reporter->Error("Event %s not found", eventName.c_str()); + return false; + } + + mgr.Dispatch(new Event(handler, 0)); + return true; } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index ce8303383a..58a56e8221 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -6,16 +6,29 @@ #ifndef INPUTREADER_H #define INPUTREADER_H +#include "InputMgr.h" +#include "BroString.h" + class InputReader { public: InputReader(); virtual ~InputReader(); + + bool Init(string source, string eventName); protected: // Methods that have to be overwritten by the individual readers - + + // Reports an error to the user. + void Error(const char *msg); + + // The following methods return the information as passed to Init(). + const string Source() const { return source; } + private: friend class InputMgr; + + string source; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/input.bif b/src/input.bif index 3ff5284c63..e2fc2ae91e 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,9 +7,11 @@ module Input; #include "NetVar.h" %%} -function Input::__create_reader%(reader: Input::Reader, source: string%) : bool +type Event: record; + +function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 9c8b0dec3b8622a592aa1c787d9bde7d53f45f5f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 19 Oct 2011 16:35:34 -0700 Subject: [PATCH 04/58] event from c++ to script works (at last...) --- scripts/base/frameworks/input/__load__.bro | 1 + scripts/base/frameworks/input/main.bro | 11 +++++++++++ scripts/base/init-bare.bro | 3 ++- src/InputMgr.cc | 7 ++++--- src/InputMgr.h | 2 +- src/InputReader.cc | 19 +++++++++++-------- src/LogMgr.cc | 2 +- src/input.bif | 4 ++-- 8 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 scripts/base/frameworks/input/__load__.bro create mode 100644 scripts/base/frameworks/input/main.bro diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/input/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro new file mode 100644 index 0000000000..cb071872ac --- /dev/null +++ b/scripts/base/frameworks/input/main.bro @@ -0,0 +1,11 @@ + +module Input; + +export { + type Event: record { + name: string; + columns: any; + }; +} + +@load base/input.bif diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8d1d2a312f..ade92bfd6e 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1509,4 +1509,5 @@ const parse_udp_tunnels = F &redef; # BiFs and script-land defined types. @load base/frameworks/logging -@load base/input.bif +@load base/frameworks/input + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5ab602f4f3..f3fb2ff935 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,11 +35,11 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription) +InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = eventDescription->Type()->AsRecordType(); + RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) { reporter->Error("eventDescription argument not of right type"); @@ -49,7 +49,6 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) { - DBG_LOG(DBG_LOGGING, "unknown reader when creating reader"); reporter->Error("unknown reader when creating reader"); return 0; } @@ -89,6 +88,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, string event InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); + reader_obj->Init(source, eventName); + return reader_obj; } diff --git a/src/InputMgr.h b/src/InputMgr.h index 3d3d0f2d3f..e98f3d77b7 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -16,7 +16,7 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, string eventName, RecordVal* eventDescription); + InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); protected: friend class InputReader; diff --git a/src/InputReader.cc b/src/InputReader.cc index ef47bb1e10..fc9be7f2b6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,7 +1,7 @@ #include "InputReader.h" -#include "EventRegistry.h" -#include "Event.h" +// #include "EventRegistry.h" +// #include "Event.h" InputReader::InputReader() { @@ -19,13 +19,16 @@ void InputReader::Error(const char *msg) } bool InputReader::Init(string source, string eventName) { - EventHandler* handler = event_registry->Lookup(eventName.c_str()); + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} - if ( handler == 0 ) { - reporter->Error("Event %s not found", eventName.c_str()); - return false; - } + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); - mgr.Dispatch(new Event(handler, 0)); + //mgr.Dispatch(new Event(handler, vl)); return true; } \ No newline at end of file diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9e320f8810..fc4b89b0ed 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) +bool LogMgr::(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index e2fc2ae91e..88d4e32129 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,9 @@ module Input; type Event: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventName: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventName->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From 365406024627c8c2b0b3d41263840a7fbaf029ed Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 21 Oct 2011 14:01:18 -0700 Subject: [PATCH 05/58] compiles. sill doesn't do much. --- scripts/base/frameworks/input/main.bro | 8 ++++-- src/InputMgr.cc | 35 ++++++++++++++++++++---- src/InputMgr.h | 8 +++++- src/InputReader.cc | 29 ++++++++++---------- src/InputReader.h | 10 ++++++- src/InputReaderAscii.cc | 38 +++++++++++++++++++++++++- src/InputReaderAscii.h | 10 +++++++ src/LogMgr.cc | 2 +- src/input.bif | 6 ++-- 9 files changed, 117 insertions(+), 29 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index cb071872ac..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,9 +2,11 @@ module Input; export { - type Event: record { - name: string; - columns: any; + type ReaderDescription: record { + source: string; + idx: any; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f3fb2ff935..500a325249 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -35,14 +35,14 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* event) +InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) { InputReaderDefinition* ir = input_readers; - RecordType* rtype = InputReaderDefinition->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Event, 0) ) + RecordType* rtype = description->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) { - reporter->Error("eventDescription argument not of right type"); + reporter->Error("readerDescription argument not of right type"); return 0; } @@ -88,7 +88,11 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, string source, RecordVal* e InputReader* reader_obj = (*ir->factory)(); assert(reader_obj); - reader_obj->Init(source, eventName); + // get the source... + const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); + string source((const char*) bsource->Bytes(), bsource->Len()); + + reader_obj->Init(source, 0, NULL); return reader_obj; @@ -100,6 +104,27 @@ void InputMgr::Error(InputReader* reader, const char* msg) reader->Source().c_str(), msg)); } +/* + TODO: + +void InputMgr::SendEvent(string name) { + //EventHandler* handler = event_registry->Lookup(eventName.c_str()); + + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", eventName.c_str()); + // return false; + //} + + //val_list* vl = new val_list; + //vl->append(new Val(12, TYPE_COUNT)); + + //mgr.Dispatch(new Event(handler, vl)); + + +} + +*/ + diff --git a/src/InputMgr.h b/src/InputMgr.h index e98f3d77b7..255d61fe5e 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -9,20 +9,26 @@ #include "Val.h" #include "EventHandler.h" #include "RemoteSerializer.h" +#include "LogMgr.h" // for the LogVal and LogType data types class InputReader; + class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, string source, RecordVal* event); + InputReader* CreateReader(EnumVal* reader, RecordVal* description); protected: friend class InputReader; // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + +private: + // required functionality + // InputValsToRecord to convert received inputvals back to bro records / tables / whatever }; diff --git a/src/InputReader.cc b/src/InputReader.cc index fc9be7f2b6..4d29040e60 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,7 +5,7 @@ InputReader::InputReader() { - + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -18,17 +18,18 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } -bool InputReader::Init(string source, string eventName) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); - return true; +bool InputReader::Init(string arg_source, int arg_num_fields, + const LogField* const * arg_fields) +{ + source = arg_source; + num_fields = arg_num_fields; + fields = arg_fields; + + // disable if DoInit returns error. + disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + return !disabled; +} + +void InputReader::Finish() { + DoFinish(); } \ No newline at end of file diff --git a/src/InputReader.h b/src/InputReader.h index 58a56e8221..f3638c7246 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -8,16 +8,22 @@ #include "InputMgr.h" #include "BroString.h" +#include "LogMgr.h" class InputReader { public: InputReader(); virtual ~InputReader(); - bool Init(string source, string eventName); + bool Init(string arg_source, int num_fields, const LogField* const* fields); + void Finish(); + protected: // Methods that have to be overwritten by the individual readers + virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + + virtual void DoFinish() = 0; // Reports an error to the user. void Error(const char *msg); @@ -29,6 +35,8 @@ private: friend class InputMgr; string source; + int num_fields; + const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the // writer as disabled. diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 97933c2a6e..b9aab16815 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,11 +2,47 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include + InputReaderAscii::InputReaderAscii() { - DBG_LOG(DBG_LOGGING, "input reader initialized"); + //DBG_LOG(DBG_LOGGING, "input reader initialized"); + file = 0; } InputReaderAscii::~InputReaderAscii() { +} + +void InputReaderAscii::DoFinish() +{ +} + +bool InputReaderAscii::DoInit(string path, int num_fields, + const LogField* const * fields) +{ + fname = path; + + file = new ifstream(path.c_str()); + if ( !file->is_open() ) { + return false; + } + + // try to read the header line... + string line; + if ( !getline(*file, line) ) + return false; + + // split on tabs... + istringstream ss(line); + while ( ss ) { + string s; + if ( !getline(ss, s, '\t')) + break; + + + } + + + return false; } \ No newline at end of file diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c471639fd4..0d2008ed7f 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -3,6 +3,9 @@ #define INPUTREADERASCII_H #include "InputReader.h" +#include +#include + class InputReaderAscii : public InputReader { public: @@ -12,8 +15,15 @@ public: static InputReader* Instantiate() { return new InputReaderAscii; } protected: + + virtual bool DoInit(string path, int num_fields, + const LogField* const * fields); + virtual void DoFinish(); private: + + ifstream* file; + string fname; }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index fc4b89b0ed..9e320f8810 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -475,7 +475,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) stream->writers.erase(*j); } -bool LogMgr::(EnumVal* id, RecordVal* sval) +bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) { RecordType* rtype = sval->Type()->AsRecordType(); diff --git a/src/input.bif b/src/input.bif index 88d4e32129..3da869ea08 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,11 +7,11 @@ module Input; #include "NetVar.h" %%} -type Event: record; +type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, source: string, eventDescription: Input::Event%) : bool +function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), source->AsString()->CheckString(), eventDescription->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} From d7a3b85fcda42a1485ef7a53c5419100e4b5ea95 Mon Sep 17 00:00:00 2001 From: amannb Date: Tue, 25 Oct 2011 11:47:23 -0700 Subject: [PATCH 06/58] many helper functions --- src/InputMgr.cc | 71 +++++++++++++++------ src/InputMgr.h | 4 +- src/InputReader.cc | 33 +++++++++- src/InputReader.h | 12 ++++ src/InputReaderAscii.cc | 134 +++++++++++++++++++++++++++++++++++++--- src/InputReaderAscii.h | 22 +++++++ 6 files changed, 245 insertions(+), 31 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 500a325249..ec46c55813 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -100,31 +100,62 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) void InputMgr::Error(InputReader* reader, const char* msg) { - reporter->Error(fmt("error with input reader for %s: %s", - reader->Source().c_str(), msg)); + reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* - TODO: -void InputMgr::SendEvent(string name) { - //EventHandler* handler = event_registry->Lookup(eventName.c_str()); - - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", eventName.c_str()); - // return false; - //} - - //val_list* vl = new val_list; - //vl->append(new Val(12, TYPE_COUNT)); - - //mgr.Dispatch(new Event(handler, vl)); +void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + val_list* vl = new val_list; + for ( int i = 0; i < num_vals; i++) { + vl->append(LogValToVal(vals[i])); + } + + mgr.Dispatch(new Event(handler, vl)); +} + +Val* InputMgr::LogValToVal(const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + return new Val(val->val.int_val, val->type); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + return new Val(val->val.uint_val, val->type); + break; + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return new Val(val->val.double_val, val->type); + break; + + case TYPE_STRING: + { + BroString *s = new BroString(*(val->val.string_val)); + return new StringVal(s); + break; + } + + case TYPE_PORT: + return new PortVal(val->val.uint_val); + break; + + default: + reporter->InternalError("unsupported type for input_read"); + } + + + reporter->InternalError("Impossible error"); + return NULL; } - -*/ - - diff --git a/src/InputMgr.h b/src/InputMgr.h index 255d61fe5e..79f76a1e6f 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -13,7 +13,6 @@ class InputReader; - class InputMgr { public: InputMgr(); @@ -29,6 +28,9 @@ protected: private: // required functionality // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + Val* LogValToVal(const LogVal* val); + + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 4d29040e60..d812349733 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -32,4 +32,35 @@ bool InputReader::Init(string arg_source, int arg_num_fields, void InputReader::Finish() { DoFinish(); -} \ No newline at end of file +} + +bool InputReader::Update() { + return DoUpdate(); +} + +// stolen from logwriter +const char* InputReader::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + + diff --git a/src/InputReader.h b/src/InputReader.h index f3638c7246..9d776276fa 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -18,12 +18,17 @@ public: bool Init(string arg_source, int num_fields, const LogField* const* fields); void Finish(); + + bool Update(); protected: // Methods that have to be overwritten by the individual readers virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; + + // update file contents to logmgr + virtual bool DoUpdate() = 0; // Reports an error to the user. void Error(const char *msg); @@ -31,6 +36,9 @@ protected: // The following methods return the information as passed to Init(). const string Source() const { return source; } + // A thread-safe version of fmt(). (stolen from logwriter) + const char* Fmt(const char* format, ...); + private: friend class InputMgr; @@ -44,6 +52,10 @@ private: bool disabled; bool Disabled() { return disabled; } + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; }; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index b9aab16815..4bc4b81cda 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -1,9 +1,22 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReaderAscii.h" #include "DebugLogger.h" #include +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position) + : name(arg_name), type(arg_type) +{ + position = arg_position; +} + +FieldMapping::FieldMapping(const FieldMapping& arg) + : name(arg.name), type(arg.type) +{ + position = arg.position; +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -18,31 +31,134 @@ void InputReaderAscii::DoFinish() { } -bool InputReaderAscii::DoInit(string path, int num_fields, - const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) { fname = path; file = new ifstream(path.c_str()); if ( !file->is_open() ) { + Error(Fmt("cannot open %s", path.c_str())); return false; } // try to read the header line... string line; - if ( !getline(*file, line) ) + if ( !getline(*file, line) ) { + Error("could not read first line"); return false; + } // split on tabs... - istringstream ss(line); - while ( ss ) { + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { string s; - if ( !getline(ss, s, '\t')) + if ( !getline(splitstream, s, '\t')) break; - + // current found heading in s... compare if we want it + for ( int i = 0; i < num_fields; i++ ) { + const LogField* field = fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, i); + columnMap.push_back(f); + wantFields++; + break; // done with searching + } + } + + // look if we did push something... + if ( columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + columnMap.push_back(empty); + } + + // done + currTab++; } + + if ( wantFields != num_fields ) { + // we did not find all fields? + // :( + Error("wantFields != num_fields"); + return false; + } + + this->num_fields = num_fields; - return false; -} \ No newline at end of file + // well, that seems to have worked... + return true; +} + +// read the entire file and send appropriate thingies back to InputMgr +bool InputReaderAscii::DoUpdate() { + // TODO: all the stuff we need for a second reading. + // *cough* + // + + + string line; + while ( getline(*file, line ) ) { + // split on tabs + + istringstream splitstream(line); + string s; + + LogVal fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) + break; + + + if ( currTab >= columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal val(currMapping.type, true); + + switch ( currMapping.type ) { + case TYPE_STRING: + val.val.string_val = new string(s); + + default: + Error(Fmt("unsupported field format %d for %s", currMapping.type, + currMapping.name.c_str())); + return false; + } + + currField++; + } + + if ( currField != num_fields ) { + Error("curr_field != num_fields in DoUpdate"); + return false; + } + + // ok, now we have built our line. send it back to... whomever. + + } + + return true; +} diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 0d2008ed7f..551a08b02e 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -1,3 +1,4 @@ +// See the file "COPYING" in the main distribution directory for copyright. #ifndef INPUTREADERASCII_H #define INPUTREADERASCII_H @@ -5,6 +6,19 @@ #include "InputReader.h" #include #include +#include + +// Description for input field mapping +struct FieldMapping { + string name; + TypeTag type; + int position; + + FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const FieldMapping& arg); + FieldMapping() { position = -1; } + bool IsEmpty() { return position == -1; } +}; class InputReaderAscii : public InputReader { @@ -19,11 +33,19 @@ protected: virtual bool DoInit(string path, int num_fields, const LogField* const * fields); virtual void DoFinish(); + + virtual bool DoUpdate(); private: ifstream* file; string fname; + + unsigned int num_fields; + + // map columns in the file to columns to send back to the manager + vector columnMap; + }; From 5b0c307f87a7213951e586f6e881501dc658f03d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 25 Oct 2011 14:11:21 -0700 Subject: [PATCH 07/58] very basic input to event working... --- scripts/base/frameworks/input/main.bro | 4 ++-- src/InputMgr.cc | 18 ++++++++++++++++-- src/InputReader.cc | 11 +++++++++++ src/InputReader.h | 3 +++ src/InputReaderAscii.cc | 14 ++++++++++---- 5 files changed, 42 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bea9d73d2 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any; - destination: any; + val: any &optional; + destination: any &optional; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ec46c55813..5cd82b613f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -91,8 +91,22 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) // get the source... const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - - reader_obj->Init(source, 0, NULL); + + RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + LogField** fields = new LogField*[idx->NumFields()]; + for ( int i = 0; i < idx->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = idx->FieldName(i); + field->type = idx->FieldType(i)->Tag(); + fields[i] = field; + } + + + reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Update(); return reader_obj; diff --git a/src/InputReader.cc b/src/InputReader.cc index d812349733..d512bc1699 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -5,6 +5,8 @@ InputReader::InputReader() { + buf = 0; + buf_len = 1024; disabled = true; // disabled will be set correcty in init. } @@ -18,6 +20,11 @@ void InputReader::Error(const char *msg) input_mgr->Error(this, msg); } +void InputReader::Error(const string &msg) + { + input_mgr->Error(this, msg.c_str()); + } + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { @@ -38,6 +45,10 @@ bool InputReader::Update() { return DoUpdate(); } +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { + input_mgr->SendEvent(name, num_vals, vals); +} + // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) { diff --git a/src/InputReader.h b/src/InputReader.h index 9d776276fa..3bfb0adf91 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -31,6 +31,7 @@ protected: virtual bool DoUpdate() = 0; // Reports an error to the user. + void Error(const string &msg); void Error(const char *msg); // The following methods return the information as passed to Init(). @@ -39,6 +40,8 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); + void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4bc4b81cda..5d6b23416a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -37,7 +37,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const file = new ifstream(path.c_str()); if ( !file->is_open() ) { - Error(Fmt("cannot open %s", path.c_str())); + Error(Fmt("cannot open %s", fname.c_str())); return false; } @@ -108,7 +108,7 @@ bool InputReaderAscii::DoUpdate() { istringstream splitstream(line); string s; - LogVal fields[num_fields]; + LogVal** fields = new LogVal*[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; @@ -136,11 +136,12 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal val(currMapping.type, true); + LogVal* val = new LogVal(currMapping.type, true); switch ( currMapping.type ) { case TYPE_STRING: - val.val.string_val = new string(s); + val->val.string_val = new string(s); + break; default: Error(Fmt("unsupported field format %d for %s", currMapping.type, @@ -148,6 +149,8 @@ bool InputReaderAscii::DoUpdate() { return false; } + fields[currField] = val; + currField++; } @@ -157,6 +160,9 @@ bool InputReaderAscii::DoUpdate() { } // ok, now we have built our line. send it back to... whomever. + // for testing purposes: fixed event. + + SendEvent("inputEvent", num_fields, fields); } From b245d4168a6c9d9ce9243a34f7fcc704d526a7d9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:02:57 -0700 Subject: [PATCH 08/58] yay, basic table assignment. --- scripts/base/frameworks/input/main.bro | 4 +- src/InputMgr.cc | 60 ++++++++++++++++++++++++-- src/InputMgr.h | 8 +++- src/InputReader.cc | 5 +++ src/InputReader.h | 2 + src/InputReaderAscii.cc | 16 +++++++ 6 files changed, 87 insertions(+), 8 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bea9d73d2..b5a05af3b6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -5,8 +5,8 @@ export { type ReaderDescription: record { source: string; idx: any; - val: any &optional; - destination: any &optional; + val: any; + destination: any; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5cd82b613f..5c30922863 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,16 @@ #include "InputReaderAscii.h" +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + unsigned int num_idx_fields; + unsigned int num_val_fields; + + TableVal* tab; + + }; struct InputReaderDefinition { bro_int_t type; // the type @@ -30,7 +40,7 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - DBG_LOG(DBG_LOGGING, "this has to happen"); + //DBG_LOG(DBG_LOGGING, "this has to happen"); } @@ -93,8 +103,10 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) string source((const char*) bsource->Bytes(), bsource->Len()); RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields()]; + LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; for ( int i = 0; i < idx->NumFields(); i++ ) { // FIXME: do type checking... @@ -103,15 +115,43 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) field->type = idx->FieldType(i)->Tag(); fields[i] = field; } + for ( int i = 0; i < val->NumFields(); i++ ) + { + // FIXME: do type checking... + LogField* field = new LogField(); + field->name = val->FieldName(i); + field->type = val->FieldType(i)->Tag(); + fields[idx->NumFields() + i] = field; + } + + ReaderInfo* info = new ReaderInfo; + info->reader = reader_obj; + info->type = reader; + info->num_idx_fields = idx->NumFields(); + info->num_val_fields = val->NumFields(); + info->tab = dst; + readers.push_back(info); - reader_obj->Init(source, idx->NumFields(), fields); + reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); reader_obj->Update(); return reader_obj; } +void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); + reporter->Error("assigned"); +} + + void InputMgr::Error(InputReader* reader, const char* msg) { reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); @@ -172,4 +212,16 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return NULL; } - +InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->reader == reader ) + { + return *s; + } + } + + return 0; + } + diff --git a/src/InputMgr.h b/src/InputMgr.h index 79f76a1e6f..136be2d608 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,14 +24,18 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); + + void Put(const InputReader* reader, const LogVal* const *vals); private: - // required functionality - // InputValsToRecord to convert received inputvals back to bro records / tables / whatever + struct ReaderInfo; + Val* LogValToVal(const LogVal* val); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); + vector readers; }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index d512bc1699..6502fdb421 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -25,6 +25,11 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } +void InputReader::Put(const LogVal* const *val) +{ + input_mgr->Put(this, val); +} + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3bfb0adf91..3725c3d461 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -42,6 +42,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + void Put(const LogVal* const *val); + private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 5d6b23416a..d0d4a3014c 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -143,6 +143,21 @@ bool InputReaderAscii::DoUpdate() { val->val.string_val = new string(s); break; + case TYPE_BOOL: + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + val->val.uint_val = atoi(s.c_str()); + break; + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); @@ -163,6 +178,7 @@ bool InputReaderAscii::DoUpdate() { // for testing purposes: fixed event. SendEvent("inputEvent", num_fields, fields); + Put(fields); } From 86730c13dda7df06fa96b151c17ec437aa742b4f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 26 Oct 2011 17:46:43 -0700 Subject: [PATCH 09/58] more complex types... --- scripts/base/frameworks/input/main.bro | 3 + src/InputMgr.cc | 314 +++++++++++++++++++++++-- src/InputMgr.h | 17 +- src/InputReader.cc | 11 + src/InputReader.h | 2 + src/InputReaderAscii.cc | 35 ++- src/input.bif | 17 +- src/types.bif | 4 + 8 files changed, 372 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index b5a05af3b6..4bb7129d03 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -2,11 +2,14 @@ module Input; export { + const default_reader = READER_ASCII &redef; + type ReaderDescription: record { source: string; idx: any; val: any; destination: any; + reader: Reader &default=default_reader; }; } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5c30922863..648a933a22 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -21,6 +21,8 @@ struct InputMgr::ReaderInfo { unsigned int num_val_fields; TableVal* tab; + RecordType* rtype; + RecordType* itype; }; @@ -45,7 +47,7 @@ InputMgr::InputMgr() // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) +InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; @@ -55,6 +57,8 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) reporter->Error("readerDescription argument not of right type"); return 0; } + + EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -106,38 +110,191 @@ InputReader* InputMgr::CreateReader(EnumVal* reader, RecordVal* description) RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - LogField** fields = new LogField*[idx->NumFields() + val->NumFields()]; - for ( int i = 0; i < idx->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = idx->FieldName(i); - field->type = idx->FieldType(i)->Tag(); - fields[i] = field; + + vector fieldsV; // vector, because we don't know the length beforehands + + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return 0; } - for ( int i = 0; i < val->NumFields(); i++ ) - { - // FIXME: do type checking... - LogField* field = new LogField(); - field->name = val->FieldName(i); - field->type = val->FieldType(i)->Tag(); - fields[idx->NumFields() + i] = field; + + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader; - info->num_idx_fields = idx->NumFields(); - info->num_val_fields = val->NumFields(); + Ref(reader); + info->num_idx_fields = idxfields; + info->num_val_fields = valfields; info->tab = dst; + Ref(dst); + info->rtype = val; + Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. + info->id = id; + Ref(id); // ditto... + info->itype = idx; + Ref(idx); readers.push_back(info); - reader_obj->Init(source, idx->NumFields() + val->NumFields(), fields); + reader_obj->Init(source, fieldsV.size(), fields); reader_obj->Update(); return reader_obj; +} +bool InputMgr::IsCompatibleType(BroType* t) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_RECORD: + // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. + // ^ recursive checking is done in UnrollRecordType. + return true; + + case TYPE_FILE: + case TYPE_FUNC: + return false; + + + case TYPE_TABLE: + return false; + + case TYPE_VECTOR: + { + return IsCompatibleType(t->AsVectorType()->YieldType()); + } + + default: + return false; + } + + return false; + } + + +bool InputMgr::RemoveReader(EnumVal* id) { + ReaderInfo *i = 0; + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } + } + + if ( i == 0 ) { + return false; // not found + } + + Unref(i->type); + Unref(i->tab); + Unref(i->itype); + Unref(i->rtype); + Unref(i->id); + + delete(i->reader); + delete(i); + + return true; +} + +bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { + for ( int i = 0; i < rec->NumFields(); i++ ) + { + + if ( !IsCompatibleType(rec->FieldType(i)) ) { + reporter->Error("Incompatible type \"%s\" in table definition for InputReader", type_name(rec->FieldType(i)->Tag())); + return false; + } + + if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) + { + + string prep = nameprepend + rec->FieldName(i) + "."; + + if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) + { + return false; + } + + } else { + LogField* field = new LogField(); + field->name = nameprepend + rec->FieldName(i); + field->type = rec->FieldType(i)->Tag(); + + fields->push_back(field); + } + } + + return true; + +} + +bool InputMgr::ForceUpdate(EnumVal* id) +{ + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + i->reader->Update(); + return true; +} + +Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { + Val* idxval; + int position = 0; + + + if ( num_fields == 1 ) { + idxval = LogValToVal(vals[0]); + } else { + ListVal *l = new ListVal(TYPE_ANY); + for ( int j = 0 ; j < type->NumFields(); j++ ) { + if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { + l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); + } else { + l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + position++; + } + } + idxval = l; + } + + assert ( position == num_fields ); + + return idxval; + } void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { @@ -147,10 +304,65 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { return; } - i->tab->Assign(LogValToVal(vals[0]), LogValToVal(vals[1])); - reporter->Error("assigned"); + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + i->tab->Assign(idxval, valval); } +void InputMgr::Clear(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + i->tab->RemoveAll(); +} + +bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return false; + } + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + + return ( i->tab->Delete(idxval) != 0 ); +} void InputMgr::Error(InputReader* reader, const char* msg) { @@ -174,7 +386,46 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } -Val* InputMgr::LogValToVal(const LogVal* val) { + +Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + if ( position == 0 ) { + reporter->InternalError("Need position"); + return 0; + } + + /* + if ( request_type->Tag() != TYPE_RECORD ) { + reporter->InternalError("I only work with records"); + return 0; + } */ + + + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + for ( int i = 0; i < request_type->NumFields(); i++ ) { + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; + +} + +Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { + + if ( request_type != TYPE_ANY && request_type != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + return 0; + } + + switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: @@ -203,6 +454,10 @@ Val* InputMgr::LogValToVal(const LogVal* val) { return new PortVal(val->val.uint_val); break; + case TYPE_ADDR: + return new AddrVal(val->val.addr_val); + break; + default: reporter->InternalError("unsupported type for input_read"); } @@ -225,3 +480,18 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) return 0; } + +InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) + { + for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) + { + if ( (*s)->id == id ) + { + return *s; + } + } + + return 0; + } + + diff --git a/src/InputMgr.h b/src/InputMgr.h index 136be2d608..d5f732935c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -11,13 +11,17 @@ #include "RemoteSerializer.h" #include "LogMgr.h" // for the LogVal and LogType data types +#include + class InputReader; class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* reader, RecordVal* description); + InputReader* CreateReader(EnumVal* id, RecordVal* description); + bool ForceUpdate(EnumVal* id); + bool RemoveReader(EnumVal* id); protected: friend class InputReader; @@ -26,14 +30,23 @@ protected: void Error(InputReader* reader, const char* msg); void Put(const InputReader* reader, const LogVal* const *vals); + void Clear(const InputReader* reader); + bool Delete(const InputReader* reader, const LogVal* const *vals); private: struct ReaderInfo; - Val* LogValToVal(const LogVal* val); + bool IsCompatibleType(BroType* t); + + bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); + Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); + ReaderInfo* FindReader(const EnumVal* id); vector readers; }; diff --git a/src/InputReader.cc b/src/InputReader.cc index 6502fdb421..1facc57c7f 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -30,6 +30,17 @@ void InputReader::Put(const LogVal* const *val) input_mgr->Put(this, val); } +void InputReader::Clear() +{ + input_mgr->Clear(this); +} + +void InputReader::Delete(const LogVal* const *val) +{ + input_mgr->Delete(this, val); +} + + bool InputReader::Init(string arg_source, int arg_num_fields, const LogField* const * arg_fields) { diff --git a/src/InputReader.h b/src/InputReader.h index 3725c3d461..0e93344e1a 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -43,6 +43,8 @@ protected: void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); + void Clear(); + void Delete(const LogVal* const *val); private: friend class InputMgr; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index d0d4a3014c..e4d581c0d8 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -144,6 +144,13 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else { + val->val.int_val = 0; + } + break; + case TYPE_INT: val->val.int_val = atoi(s.c_str()); break; @@ -155,16 +162,37 @@ bool InputReaderAscii::DoUpdate() { break; case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // fallthrough + } + case TYPE_ADDR: { + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + default: Error(Fmt("unsupported field format %d for %s", currMapping.type, currMapping.name.c_str())); return false; } - fields[currField] = val; + fields[currMapping.position] = val; currField++; } @@ -174,10 +202,7 @@ bool InputReaderAscii::DoUpdate() { return false; } - // ok, now we have built our line. send it back to... whomever. - // for testing purposes: fixed event. - - SendEvent("inputEvent", num_fields, fields); + // ok, now we have built our line. send it back to the input manager Put(fields); } diff --git a/src/input.bif b/src/input.bif index 3da869ea08..4e2cbf07b5 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,9 +9,22 @@ module Input; type ReaderDescription: record; -function Input::__create_reader%(reader: Input::Reader, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(reader->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} + +function Input::__force_update%(id: ID%) : bool + %{ + bool res = input_mgr->ForceUpdate(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_reader%(id: ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + diff --git a/src/types.bif b/src/types.bif index ee43207ddd..7b81a7f631 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,4 +174,8 @@ enum Reader %{ READER_ASCII, %} +enum ID %{ + Unknown, +%} + module GLOBAL; From f20125d22de20a7147026602f83fa1ed710700ac Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 28 Oct 2011 13:11:47 -0700 Subject: [PATCH 10/58] little snag with hashing functionality... --- src/InputMgr.cc | 232 +++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 11 +- src/InputReader.cc | 13 ++- src/InputReader.h | 9 +- src/InputReaderAscii.cc | 80 ++++++++++++-- src/InputReaderAscii.h | 5 +- src/Val.h | 5 +- 7 files changed, 333 insertions(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 648a933a22..5270c83d5a 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -13,6 +13,18 @@ #include "InputReaderAscii.h" +#include "CompHash.h" + + +class InputHash { +public: + HashKey* valhash; + HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. +}; + +declare(PDict, InputHash); + + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -24,6 +36,9 @@ struct InputMgr::ReaderInfo { RecordType* rtype; RecordType* itype; + PDict(InputHash)* currDict; + PDict(InputHash)* lastDict; + }; struct InputReaderDefinition { @@ -147,10 +162,20 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx; Ref(idx); readers.push_back(info); + info->currDict = new PDict(InputHash); + info->lastDict = new PDict(InputHash); - reader_obj->Init(source, fieldsV.size(), fields); - reader_obj->Update(); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + if ( success == false ) { + RemoveReader(id); + return 0; + } + success = reader_obj->Update(); + if ( success == false ) { + RemoveReader(id); + return 0; + } return reader_obj; @@ -198,7 +223,6 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } - bool InputMgr::RemoveReader(EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) @@ -215,6 +239,9 @@ bool InputMgr::RemoveReader(EnumVal* id) { return false; // not found } + i->reader->Finish(); + + Unref(i->type); Unref(i->tab); Unref(i->itype); @@ -267,8 +294,7 @@ bool InputMgr::ForceUpdate(EnumVal* id) return false; } - i->reader->Update(); - return true; + return i->reader->Update(); } Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { @@ -297,6 +323,105 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } + +void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + + InputHash *h = i->lastDict->Lookup(idxhash); + if ( h != 0 ) { + // seen before + if ( h->valhash->Hash() == valhash->Hash() ) { + // ok, double. + i->lastDict->Remove(idxhash); + i->currDict->Insert(idxhash, h); + return; + } else { + // updated + i->lastDict->Remove(idxhash); + delete(h); + } + } + + + Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* valval; + + int position = i->num_idx_fields; + if ( i->num_val_fields == 1 ) { + valval = LogValToVal(vals[i->num_idx_fields]); + } else { + RecordVal * r = new RecordVal(i->rtype); + + /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { + reporter->InternalError("Type mismatch"); + return; + } */ + + for ( int j = 0; j < i->rtype->NumFields(); j++) { + + Val* val = 0; + if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + position++; + } + + if ( val == 0 ) { + reporter->InternalError("conversion error"); + return; + } + + r->Assign(j,val); + + } + valval = r; + } + + //i->tab->Assign(idxval, valval); + HashKey* k = i->tab->ComputeHash(idxval); + if ( !k ) { + reporter->InternalError("could not hash"); + return; + } + + i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + ih->idxkey = k; + ih->valhash = valhash; + + i->currDict->Insert(idxhash, ih); + +} + +void InputMgr::EndCurrentSend(const InputReader* reader) { + ReaderInfo *i = FindReader(reader); + if ( i == 0 ) { + reporter->InternalError("Unknown reader"); + return; + } + + // lastdict contains all deleted entries + IterCookie *c = i->lastDict->InitForIteration(); + InputHash* ih; + while ( ( ih = i->lastDict->NextEntry(c )) ) { + i->tab->Delete(ih->idxkey); + } + + i->lastDict->Clear(); + delete(i->lastDict); + + i->lastDict = i->currDict; + i->currDict = new PDict(InputHash); +} + void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -418,6 +543,95 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } +HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { + int length = 0; + + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals"); + } + + } + + int position = 0; + char *data = (char*) malloc(length); + for ( int i = 0; i < num_elements; i++ ) { + const LogVal* val = vals[i]; + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + *(data+position) = val->val.int_val; + position += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + *(data+position) = val->val.uint_val; + position += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + *(data+position) = val->val.double_val; + position += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + { + memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); + position += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + position += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + default: + reporter->InternalError("unsupported type for hashlogvals2"); + } + + + } + + assert(position == length); + return new HashKey(data, length); + + +} + Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { if ( request_type != TYPE_ANY && request_type != val->type ) { @@ -495,3 +709,11 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) } +string InputMgr::Hash(const string &input) { + unsigned char digest[16]; + hash_md5(input.length(), (const unsigned char*) input.c_str(), digest); + string out((const char*) digest, 16); + return out; +} + + diff --git a/src/InputMgr.h b/src/InputMgr.h index d5f732935c..d147fa262a 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -15,9 +15,10 @@ class InputReader; + class InputMgr { public: - InputMgr(); + InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); bool ForceUpdate(EnumVal* id); @@ -32,6 +33,9 @@ protected: void Put(const InputReader* reader, const LogVal* const *vals); void Clear(const InputReader* reader); bool Delete(const InputReader* reader, const LogVal* const *vals); + + void SendEntry(const InputReader* reader, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader); private: struct ReaderInfo; @@ -40,6 +44,8 @@ private: bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); @@ -49,6 +55,9 @@ private: ReaderInfo* FindReader(const EnumVal* id); vector readers; + + string Hash(const string &input); + }; extern InputMgr* input_mgr; diff --git a/src/InputReader.cc b/src/InputReader.cc index 1facc57c7f..494df3fb81 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -41,20 +41,22 @@ void InputReader::Delete(const LogVal* const *val) } -bool InputReader::Init(string arg_source, int arg_num_fields, +bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * arg_fields) { source = arg_source; num_fields = arg_num_fields; + index_fields = arg_idx_fields; fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_fields); + disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); return !disabled; } void InputReader::Finish() { DoFinish(); + disabled = true; } bool InputReader::Update() { @@ -91,3 +93,10 @@ const char* InputReader::Fmt(const char* format, ...) } +void InputReader::SendEntry(const LogVal* const *vals) { + input_mgr->SendEntry(this, vals); +} + +void InputReader::EndCurrentSend() { + input_mgr->EndCurrentSend(this); +} diff --git a/src/InputReader.h b/src/InputReader.h index 0e93344e1a..b547d29506 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,7 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int num_fields, const LogField* const* fields); + bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); void Finish(); @@ -23,7 +23,7 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int num_fields, const LogField* const * fields) = 0; + virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; virtual void DoFinish() = 0; @@ -46,11 +46,16 @@ protected: void Clear(); void Delete(const LogVal* const *val); + void SendEntry(const LogVal* const *vals); + void EndCurrentSend(); + + private: friend class InputMgr; string source; int num_fields; + int index_fields; const LogField* const * fields; // When an error occurs, this method is called to set a flag marking the diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4d581c0d8..e434f7e750 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -21,17 +21,26 @@ InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; + + //keyMap = new map(); } InputReaderAscii::~InputReaderAscii() { + DoFinish(); } void InputReaderAscii::DoFinish() { + columnMap.empty(); + if ( file != 0 ) { + file->close(); + delete(file); + file = 0; + } } -bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) { fname = path; @@ -47,6 +56,9 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const Error("could not read first line"); return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; // split on tabs... istringstream splitstream(line); @@ -83,12 +95,10 @@ bool InputReaderAscii::DoInit(string path, int num_fields, const LogField* const if ( wantFields != num_fields ) { // we did not find all fields? // :( - Error("wantFields != num_fields"); + Error("One of the requested fields could not be found in the input data file"); return false; } - - this->num_fields = num_fields; // well, that seems to have worked... return true; @@ -101,6 +111,9 @@ bool InputReaderAscii::DoUpdate() { // + // new keymap + //map *newKeyMap = new map(); + string line; while ( getline(*file, line ) ) { // split on tabs @@ -109,10 +122,12 @@ bool InputReaderAscii::DoUpdate() { string s; LogVal** fields = new LogVal*[num_fields]; + //string string_fields[num_fields]; unsigned int currTab = 0; unsigned int currField = 0; while ( splitstream ) { + if ( !getline(splitstream, s, '\t') ) break; @@ -146,8 +161,11 @@ bool InputReaderAscii::DoUpdate() { case TYPE_BOOL: if ( s == "T" ) { val->val.int_val = 1; - } else { + } else if ( s == "F" ) { val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; } break; @@ -173,9 +191,15 @@ bool InputReaderAscii::DoUpdate() { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // fallthrough + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + } case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 copy_addr(t, val->val.addr_val); @@ -193,19 +217,57 @@ bool InputReaderAscii::DoUpdate() { } fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; currField++; } if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate"); + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); return false; } - // ok, now we have built our line. send it back to the input manager - Put(fields); + + SendEntry(fields); + + /* + string indexstring = ""; + string valstring = ""; + for ( unsigned int i = 0; i < idx_fields; i++ ) { + indexstring.append(string_fields[i]); + } + + for ( unsigned int i = idx_fields; i < num_fields; i++ ) { + valstring.append(string_fields[i]); + } + + string valhash = Hash(valstring); + string indexhash = Hash(indexstring); + + if ( keyMap->find(indexhash) == keyMap->end() ) { + // new key + Put(fields); + } else if ( (*keyMap)[indexhash] != valhash ) { + // changed key + Put(fields); + keyMap->erase(indexhash); + } else { + // field not changed + keyMap->erase(indexhash); + } + + + (*newKeyMap)[indexhash] = valhash; + */ + + for ( unsigned int i = 0; i < num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } + + EndCurrentSend(); return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 551a08b02e..c26a139dcd 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -30,7 +30,7 @@ public: protected: - virtual bool DoInit(string path, int num_fields, + virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, const LogField* const * fields); virtual void DoFinish(); @@ -42,9 +42,12 @@ private: string fname; unsigned int num_fields; + unsigned int idx_fields; // map columns in the file to columns to send back to the manager vector columnMap; + + //map *keyMap; }; diff --git a/src/Val.h b/src/Val.h index d851be311b..3ae0bc3334 100644 --- a/src/Val.h +++ b/src/Val.h @@ -841,6 +841,9 @@ public: timer = 0; } + HashKey* ComputeHash(const Val* index) const + { return table_hash->ComputeHash(index, 1); } + protected: friend class Val; friend class StateAccess; @@ -851,8 +854,6 @@ protected: void CheckExpireAttr(attr_tag at); int ExpandCompoundAndInit(val_list* vl, int k, Val* new_val); int CheckAndAssign(Val* index, Val* new_val, Opcode op = OP_ASSIGN); - HashKey* ComputeHash(const Val* index) const - { return table_hash->ComputeHash(index, 1); } bool AddProperties(Properties arg_state); bool RemoveProperties(Properties arg_state); From 638976791efb3b7d9d491266d83bb30b73d4bd53 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:00:19 -0700 Subject: [PATCH 11/58] hashing seems to work _correctly_ now... --- src/InputMgr.cc | 44 ++++++++++++++++++++++++++++++----------- src/InputReaderAscii.cc | 41 +++++++++++++++++++++++++++++++++----- src/InputReaderAscii.h | 3 +++ 3 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5270c83d5a..e3acfb9505 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -290,10 +290,10 @@ bool InputMgr::ForceUpdate(EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->Error("Reader not found"); + reporter->InternalError("Reader not found"); return false; } - + return i->reader->Update(); } @@ -302,8 +302,9 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo int position = 0; - if ( num_fields == 1 ) { + if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { idxval = LogValToVal(vals[0]); + position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); for ( int j = 0 ; j < type->NumFields(); j++ ) { @@ -317,6 +318,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo idxval = l; } + //reporter->Error("Position: %d, num_fields: %d", position, num_fields); assert ( position == num_fields ); return idxval; @@ -331,8 +333,15 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + + reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + + //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); InputHash *h = i->lastDict->Lookup(idxhash); if ( h != 0 ) { @@ -393,9 +402,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } i->tab->Assign(idxval, k, valval); + InputHash* ih = new InputHash(); + k = i->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; + //i->tab->Delete(k); i->currDict->Insert(idxhash, ih); @@ -407,11 +419,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { reporter->InternalError("Unknown reader"); return; } - - // lastdict contains all deleted entries + // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c )) ) { + reporter->Error("ending"); + while ( ( ih = i->lastDict->NextEntry(c) ) ) { + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -582,28 +595,37 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } + //reporter->Error("Length: %d", length); + int position = 0; char *data = (char*) malloc(length); + if ( data == 0 ) { + reporter->InternalError("Could not malloc?"); + } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; switch ( val->type ) { case TYPE_BOOL: case TYPE_INT: - *(data+position) = val->val.int_val; + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); + memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+position) = val->val.int_val; position += sizeof(val->val.int_val); break; case TYPE_COUNT: case TYPE_COUNTER: case TYPE_PORT: - *(data+position) = val->val.uint_val; + //*(data+position) = val->val.uint_val; + memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); position += sizeof(val->val.uint_val); break; case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - *(data+position) = val->val.double_val; + //*(data+position) = val->val.double_val; + memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); position += sizeof(val->val.double_val); break; @@ -685,7 +707,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const InputReader* reader) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->reader == reader ) + if ( (*s)->reader && (*s)->reader == reader ) { return *s; } @@ -699,7 +721,7 @@ InputMgr::ReaderInfo* InputMgr::FindReader(const EnumVal* id) { for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) + if ( (*s)->id && (*s)->id->AsEnum() == id->AsEnum() ) { return *s; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e434f7e750..4755b2b74f 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -50,6 +50,16 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + + this->num_fields = num_fields; + this->idx_fields = idx_fields; + this->fields = fields; + + return true; +} + + +bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; if ( !getline(*file, line) ) { @@ -57,9 +67,6 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } - this->num_fields = num_fields; - this->idx_fields = idx_fields; - // split on tabs... istringstream splitstream(line); unsigned int currTab = 0; @@ -70,7 +77,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const break; // current found heading in s... compare if we want it - for ( int i = 0; i < num_fields; i++ ) { + for ( unsigned int i = 0; i < num_fields; i++ ) { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position @@ -92,7 +99,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const currTab++; } - if ( wantFields != num_fields ) { + if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( Error("One of the requested fields could not be found in the input data file"); @@ -106,9 +113,30 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { + + + // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) + if ( file && file->is_open() ) { + file->close(); + } + file = new ifstream(fname.c_str()); + if ( !file->is_open() ) { + Error(Fmt("cannot open %s", fname.c_str())); + return false; + } + // + + // file->seekg(0, ios::beg); // do not forget clear. + + + if ( ReadHeader() == false ) { + return false; + } + // TODO: all the stuff we need for a second reading. // *cough* // + // // new keymap @@ -152,6 +180,7 @@ bool InputReaderAscii::DoUpdate() { } LogVal* val = new LogVal(currMapping.type, true); + //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { case TYPE_STRING: @@ -267,6 +296,8 @@ bool InputReaderAscii::DoUpdate() { } + //file->clear(); // remove end of file evil bits + //file->seekg(0, ios::beg); // and seek to start. EndCurrentSend(); return true; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c26a139dcd..673a2cdae2 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,6 +37,8 @@ protected: virtual bool DoUpdate(); private: + + bool ReadHeader(); ifstream* file; string fname; @@ -46,6 +48,7 @@ private: // map columns in the file to columns to send back to the manager vector columnMap; + const LogField* const * fields; // raw mapping //map *keyMap; From b5a77aa77baf69b788adb7b40f4246f0692939a0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 2 Nov 2011 14:29:58 -0700 Subject: [PATCH 12/58] reading seems to work with all atomic types + records... --- src/InputMgr.cc | 25 ++++++++++++++++++++----- src/InputReaderAscii.cc | 2 +- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e3acfb9505..bbfd354189 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -334,12 +334,12 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } - reporter->Error("Hashing %d index fields", i->num_idx_fields); + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); - reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); - reporter->Error("Hashing %d val fields", i->num_val_fields); + //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); + //reporter->Error("Hashing %d val fields", i->num_val_fields); HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); - reporter->Error("Result: %d", (uint64_t) valhash->Hash()); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -422,7 +422,6 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; - reporter->Error("ending"); while ( ( ih = i->lastDict->NextEntry(c) ) ) { reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); @@ -589,6 +588,11 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals length += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals"); } @@ -641,6 +645,13 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals position += NUM_ADDR_WORDS*sizeof(uint32_t); break; + case TYPE_SUBNET: + memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + position += sizeof(val->val.subnet_val.width); + memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + position += sizeof(val->val.subnet_val.net); + break; + default: reporter->InternalError("unsupported type for hashlogvals2"); } @@ -694,6 +705,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new AddrVal(val->val.addr_val); break; + case TYPE_SUBNET: + return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); + break; + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4755b2b74f..39635de407 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -216,7 +216,7 @@ bool InputReaderAscii::DoUpdate() { case TYPE_SUBNET: { int pos = s.find("/"); - string width = s.substr(pos); + string width = s.substr(pos+1); val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; From 4845c3a9a61d95dd0246f3beb6fcaa6e74c88f8c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 3 Nov 2011 14:04:13 -0700 Subject: [PATCH 13/58] send events when input entries change --- src/InputMgr.cc | 97 +++++++++++++++++++++++++++++++++++++++++++++- src/InputMgr.h | 8 +++- src/InputReader.cc | 24 +++++++----- src/input.bif | 11 ++++++ src/types.bif | 7 ++++ 5 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index bbfd354189..e987dda81e 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -38,6 +38,8 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; + + list* events; // events we fire when "something" happens }; @@ -165,6 +167,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->events = new list(); int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { @@ -223,7 +226,7 @@ bool InputMgr::IsCompatibleType(BroType* t) return false; } -bool InputMgr::RemoveReader(EnumVal* id) { +bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -254,6 +257,43 @@ bool InputMgr::RemoveReader(EnumVal* id) { return true; } +bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + i->events->push_back(eventName); + + return true; +} + +bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + bool erased = false; + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) + { + if ( *it == eventName ) { + it = i->events->erase(it); + erased = true; + } + else + ++it; + } + + + return erased; +} + + bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -286,7 +326,7 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } -bool InputMgr::ForceUpdate(EnumVal* id) +bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -333,6 +373,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { return; } + bool updated = false; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); @@ -355,6 +397,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { // updated i->lastDict->Remove(idxhash); delete(h); + updated = true; } } @@ -411,8 +454,25 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + EnumVal* ev; + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + + Ref(idxval); + Ref(valval); + SendEvent(*it, ev, idxval, valval); + ++it; + } + } + void InputMgr::EndCurrentSend(const InputReader* reader) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { @@ -423,6 +483,23 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events->size() > 0 ) { + ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + Val *val = i->tab->Lookup(idx); + assert(val != 0); + + std::list::iterator it = i->events->begin(); + while ( it != i->events->end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } @@ -523,6 +600,22 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } +void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +{ + EventHandler* handler = event_registry->Lookup(name.c_str()); + if ( handler == 0 ) { + reporter->Error("Event %s not found", name.c_str()); + return; + } + + val_list* vl = new val_list; + vl->append(event); + vl->append(left); + vl->append(right); + + mgr.Dispatch(new Event(handler, vl)); +} + Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d147fa262a..378838c2cf 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -21,8 +21,11 @@ public: InputMgr(); InputReader* CreateReader(EnumVal* id, RecordVal* description); - bool ForceUpdate(EnumVal* id); - bool RemoveReader(EnumVal* id); + bool ForceUpdate(const EnumVal* id); + bool RemoveReader(const EnumVal* id); + bool RegisterEvent(const EnumVal* id, string eventName); + bool UnregisterEvent(const EnumVal* id, string eventName); + protected: friend class InputReader; @@ -43,6 +46,7 @@ private: bool IsCompatibleType(BroType* t); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); + void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 494df3fb81..994f8b9b97 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -1,13 +1,12 @@ +// See the file "COPYING" in the main distribution directory for copyright. #include "InputReader.h" -// #include "EventRegistry.h" -// #include "Event.h" InputReader::InputReader() { buf = 0; buf_len = 1024; - disabled = true; // disabled will be set correcty in init. + disabled = true; // disabled will be set correcty in init. } InputReader::~InputReader() @@ -21,9 +20,9 @@ void InputReader::Error(const char *msg) } void InputReader::Error(const string &msg) - { +{ input_mgr->Error(this, msg.c_str()); - } +} void InputReader::Put(const LogVal* const *val) { @@ -54,16 +53,19 @@ bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields return !disabled; } -void InputReader::Finish() { +void InputReader::Finish() +{ DoFinish(); disabled = true; } -bool InputReader::Update() { +bool InputReader::Update() +{ return DoUpdate(); } -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { +void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +{ input_mgr->SendEvent(name, num_vals, vals); } @@ -93,10 +95,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) { +void InputReader::SendEntry(const LogVal* const *vals) +{ input_mgr->SendEntry(this, vals); } -void InputReader::EndCurrentSend() { +void InputReader::EndCurrentSend() +{ input_mgr->EndCurrentSend(this); } diff --git a/src/input.bif b/src/input.bif index 4e2cbf07b5..90dd2386b1 100644 --- a/src/input.bif +++ b/src/input.bif @@ -21,6 +21,17 @@ function Input::__force_update%(id: ID%) : bool return new Val( res, TYPE_BOOL ); %} +function Input::__add_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_event%(id: ID, name: string%) : bool + %{ + bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL ); + %} function Input::__remove_reader%(id: ID%) : bool %{ diff --git a/src/types.bif b/src/types.bif index 7b81a7f631..f90a954224 100644 --- a/src/types.bif +++ b/src/types.bif @@ -174,6 +174,13 @@ enum Reader %{ READER_ASCII, %} +enum Event %{ + EVENT_NEW, + EVENT_CHANGED, + EVENT_REMOVED, +%} + + enum ID %{ Unknown, %} From 2e3874331d0540724a47e91c8b0146b7b3fa6846 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 12:41:10 -0700 Subject: [PATCH 14/58] support for filters and little event fix --- scripts/base/frameworks/input/main.bro | 10 ++ src/InputMgr.cc | 204 +++++++++++++++++++++---- src/InputMgr.h | 5 + src/input.bif | 22 ++- 4 files changed, 210 insertions(+), 31 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4bb7129d03..35ca6dfa4e 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -11,6 +11,16 @@ export { destination: any; reader: Reader &default=default_reader; }; + + type Filter: record { + name: string; + ## descriptive name. for later removal + + + pred: function(typ: Input::Event, left: any, right: any): bool &optional; + ## decision function, that decides if an inserton, update or removal should really be executed + }; + } @load base/input.bif diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e987dda81e..0b87a561ad 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -24,6 +24,11 @@ public: declare(PDict, InputHash); +struct InputMgr::Filter { + EnumVal* id; + string name; + Func* pred; +}; struct InputMgr::ReaderInfo { EnumVal* id; @@ -39,10 +44,17 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - list* events; // events we fire when "something" happens + list events; // events we fire when "something" happens + list filters; // events we fire when "something" happens + +// ~ReaderInfo(); }; +//void InputMgr::~ReaderInfo() { +// +//} + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -167,8 +179,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); - info->events = new list(); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { RemoveReader(id); @@ -264,7 +274,7 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return false; } - i->events->push_back(eventName); + i->events.push_back(eventName); return true; } @@ -276,21 +286,23 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - bool erased = false; + //bool erased = false; - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { if ( *it == eventName ) { - it = i->events->erase(it); - erased = true; + it = i->events.erase(it); + return true; + // erased = true; } else ++it; } - return erased; + return false; + //return erased; } @@ -337,6 +349,59 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + + i->filters.push_back(filter); + + return true; +} + +bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->InternalError("Reader not found"); + return false; + } + + + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) + { + if ( (*it).name == name ) { + it = i->filters.erase(it); + return true; + break; + } + else + ++it; + } + + return false;; +} + + + Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -398,6 +463,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->lastDict->Remove(idxhash); delete(h); updated = true; + } } @@ -437,6 +503,48 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { valval = r; } + + Val* oldval = 0; + if ( updated == true ) { + // in that case, we need the old value to send the event (if we send an event). + oldval = i->tab->Lookup(idxval); + } + + + // call filters first do determine if we really add / change the entry + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev; + Ref(idxval); + Ref(valval); + + if ( updated ) { + ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + } else { + ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + } + + val_list vl(3); + vl.append(ev); + vl.append(idxval); + vl.append(valval); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit. + return; + } + + ++it; + } + + //i->tab->Assign(idxval, valval); HashKey* k = i->tab->ComputeHash(idxval); if ( !k ) { @@ -454,21 +562,28 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { i->currDict->Insert(idxhash, ih); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { + // send events now that we are kind of finished. + std::list::iterator filter_iterator = i->events.begin(); + while ( filter_iterator != i->events.end() ) { EnumVal* ev; - if ( updated ) { + Ref(idxval); + + if ( updated ) { // in case of update send back the old value. ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); + assert ( oldval != 0 ); + Ref(oldval); + SendEvent(*filter_iterator, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + Ref(valval); + SendEvent(*filter_iterator, ev, idxval, valval); } - Ref(idxval); - Ref(valval); - SendEvent(*it, ev, idxval, valval); - ++it; + ++filter_iterator; } + + } @@ -483,24 +598,61 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { IterCookie *c = i->lastDict->InitForIteration(); InputHash* ih; while ( ( ih = i->lastDict->NextEntry(c) ) ) { + + if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event + { - if ( i->events->size() > 0 ) { ListVal *idx = i->tab->RecoverIndex(ih->idxkey); assert(idx != 0); Val *val = i->tab->Lookup(idx); assert(val != 0); - std::list::iterator it = i->events->begin(); - while ( it != i->events->end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; + + { + // ask filter, if we want to expire this element... + std::list::iterator it = i->filters.begin(); + while ( it != i->filters.end() ) { + if (! (*it).pred ) { + continue; + } + + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); + + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = (*it).pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // throw away. Hence - we quit and simply go to the next entry of lastDict + continue; + } + + ++it; + } } + + // + + { + std::list::iterator it = i->events.begin(); + while ( it != i->events.end() ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(*it, ev, idx, val); + ++it; + } + } + } - reporter->Error("Expiring element"); + //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 378838c2cf..74914e65ad 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -26,6 +26,9 @@ public: bool RegisterEvent(const EnumVal* id, string eventName); bool UnregisterEvent(const EnumVal* id, string eventName); + bool AddFilter(EnumVal *id, RecordVal* filter); + bool RemoveFilter(EnumVal* id, const string &name); + protected: friend class InputReader; @@ -62,6 +65,8 @@ private: string Hash(const string &input); + struct Filter; + }; extern InputMgr* input_mgr; diff --git a/src/input.bif b/src/input.bif index 90dd2386b1..2301482506 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,34 +8,46 @@ module Input; %%} type ReaderDescription: record; +type Filter: record; -function Input::__create_reader%(id: ID, description: Input::ReaderDescription%) : bool +function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__force_update%(id: ID%) : bool +function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: ID, name: string%) : bool +function Input::__add_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_event%(id: ID, name: string%) : bool +function Input::__remove_event%(id: Log::ID, name: string%) : bool %{ bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: ID%) : bool +function Input::__remove_reader%(id: Log::ID%) : bool %{ bool res = input_mgr->RemoveReader(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} +function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool + %{ + bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_filter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} From 5f37040c962a9f3d524ddcccfaeba982acb0d500 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 13:59:43 -0700 Subject: [PATCH 15/58] filters really working as intented (though probably still memleaky) --- src/InputMgr.cc | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0b87a561ad..7ca1e1b485 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -537,8 +537,14 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Unref(v); if ( result == false ) { - // throw away. Hence - we quit. - return; + if ( !updated ) { + // throw away. Hence - we quit. + return; + } else { + // keep old one + i->currDict->Insert(idxhash, h); + return; + } } ++it; @@ -596,8 +602,11 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } // lastdict contains all deleted entries and should be empty apart from that IterCookie *c = i->lastDict->InitForIteration(); + i->lastDict->MakeRobustCookie(c); InputHash* ih; - while ( ( ih = i->lastDict->NextEntry(c) ) ) { + HashKey *lastDictIdxKey; + //while ( ( ih = i->lastDict->NextEntry(c) ) ) { + while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event { @@ -609,6 +618,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { { + bool doBreak = false; // ask filter, if we want to expire this element... std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -627,13 +637,21 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { Val* v = (*it).pred->Call(&vl); bool result = v->AsBool(); Unref(v); + + ++it; if ( result == false ) { - // throw away. Hence - we quit and simply go to the next entry of lastDict + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); + doBreak = true; continue; } - ++it; + } + + if ( doBreak ) { + continue; } } @@ -654,6 +672,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); + i->lastDict->Remove(lastDictIdxKey); + delete(ih); } i->lastDict->Clear(); From 2aa0f6da5795aa90c83c9fa1872776737f1e7416 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 14:33:34 -0700 Subject: [PATCH 16/58] beautify script calls, track filters --- scripts/base/frameworks/input/main.bro | 63 +++++++++++++++++++++++++- src/input.bif | 12 ++--- 2 files changed, 68 insertions(+), 7 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 35ca6dfa4e..e0c41f19be 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -20,7 +20,68 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; - + + const no_filter: Filter = [$name=""]; # Sentinel. + + global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_reader: function(id: Log::ID) : bool; + global force_update: function(id: Log::ID) : bool; + global add_event: function(id: Log::ID, name: string) : bool; + global remove_event: function(id: Log::ID, name: string) : bool; + global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; + global remove_filter: function(id: Log::ID, name: string) : bool; + global get_filter: function(id: ID, name: string) : Filter; + } @load base/input.bif + + +module Input; + +global filters: table[ID, string] of Filter; + +function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool + { + return __create_reader(id, description); + } + +function remove_reader(id: Log::ID) : bool + { + return __remove_reader(id); + } + +function force_update(id: Log::ID) : bool + { + return __force_update(id); + } + +function add_event(id: Log::ID, name: string) : bool + { + return __add_event(id, name); + } + +function remove_event(id: Log::ID, name: string) : bool + { + return __remove_event(id, name); + } + +function add_filter(id: Log::ID, filter: Input::Filter) : bool + { + filters[id, filter$name] = filter; + return __add_filter(id, filter); + } + +function remove_filter(id: Log::ID, name: string) : bool + { + delete filters[id, name]; + return __remove_filter(id, name); + } + +function get_filter(id: ID, name: string) : Filter + { + if ( [id, name] in filters ) + return filters[id, name]; + + return no_filter; + } diff --git a/src/input.bif b/src/input.bif index 2301482506..c4bf14e3ed 100644 --- a/src/input.bif +++ b/src/input.bif @@ -16,6 +16,12 @@ function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescript return new Val( the_reader != 0, TYPE_BOOL ); %} +function Input::__remove_reader%(id: Log::ID%) : bool + %{ + bool res = input_mgr->RemoveReader(id->AsEnumVal()); + return new Val( res, TYPE_BOOL ); + %} + function Input::__force_update%(id: Log::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); @@ -34,12 +40,6 @@ function Input::__remove_event%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool - %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From 1d39eaf32dff51ef779188f3123e3d63a28f71f8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 4 Nov 2011 15:03:40 -0700 Subject: [PATCH 17/58] small fixes, less leakiness --- scripts/base/frameworks/input/main.bro | 1 - src/InputMgr.cc | 98 +++++++++++--------------- 2 files changed, 41 insertions(+), 58 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index e0c41f19be..6e19452f7a 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -16,7 +16,6 @@ export { name: string; ## descriptive name. for later removal - pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## decision function, that decides if an inserton, update or removal should really be executed }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ca1e1b485..18b17c7576 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,8 +28,14 @@ struct InputMgr::Filter { EnumVal* id; string name; Func* pred; + + ~Filter(); }; +InputMgr::Filter::~Filter() { + Unref(id); +} + struct InputMgr::ReaderInfo { EnumVal* id; EnumVal* type; @@ -45,15 +51,20 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* lastDict; list events; // events we fire when "something" happens - list filters; // events we fire when "something" happens - -// ~ReaderInfo(); + list filters; // filters that can prevent our actions + ~ReaderInfo(); }; -//void InputMgr::~ReaderInfo() { -// -//} +InputMgr::ReaderInfo::~ReaderInfo() { + Unref(type); + Unref(tab); + Unref(itype); + Unref(rtype); + Unref(id); + + delete(reader); +} struct InputReaderDefinition { bro_int_t type; // the type @@ -71,10 +82,8 @@ InputReaderDefinition input_readers[] = { InputMgr::InputMgr() { - //DBG_LOG(DBG_LOGGING, "this has to happen"); } - // create a new input reader object to be used at whomevers leisure lateron. InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) { @@ -163,30 +172,26 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader; - Ref(reader); + info->type = reader->Ref()->AsEnumVal(); info->num_idx_fields = idxfields; info->num_val_fields = valfields; - info->tab = dst; - Ref(dst); - info->rtype = val; - Ref(val); // we save a pointer of it... I really hope that this wasn't already done anywhere. - info->id = id; - Ref(id); // ditto... - info->itype = idx; - Ref(idx); - readers.push_back(info); + info->tab = dst->Ref()->AsTableVal(); + info->rtype = val->Ref()->AsRecordType(); + info->id = id->Ref()->AsEnumVal(); + info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + readers.push_back(info); + int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); if ( success == false ) { - RemoveReader(id); + assert( RemoveReader(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - RemoveReader(id); + assert ( RemoveReader(id) ); return 0; } @@ -240,12 +245,12 @@ bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { - if ( (*s)->id == id ) - { - i = (*s); - readers.erase(s); // remove from vector - break; - } + if ( (*s)->id == id ) + { + i = (*s); + readers.erase(s); // remove from vector + break; + } } if ( i == 0 ) { @@ -254,14 +259,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { i->reader->Finish(); - - Unref(i->type); - Unref(i->tab); - Unref(i->itype); - Unref(i->rtype); - Unref(i->id); - - delete(i->reader); delete(i); return true; @@ -279,6 +276,8 @@ bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { return true; } +// remove first event with name eventName +// (though there shouldn't really be several events with the same name... bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -286,23 +285,18 @@ bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { return false; } - //bool erased = false; - std::list::iterator it = i->events.begin(); while ( it != i->events.end() ) { if ( *it == eventName ) { it = i->events.erase(it); return true; - // erased = true; } else ++it; } - return false; - //return erased; } @@ -335,14 +329,13 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec } return true; - } bool InputMgr::ForceUpdate(const EnumVal* id) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -352,7 +345,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -380,7 +373,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { - reporter->InternalError("Reader not found"); + reporter->Error("Reader not found"); return false; } @@ -427,7 +420,6 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo assert ( position == num_fields ); return idxval; - } @@ -538,7 +530,8 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( result == false ) { if ( !updated ) { - // throw away. Hence - we quit. + // throw away. Hence - we quit. And remove the entry from the current dictionary... + delete(i->currDict->RemoveEntry(idxhash)); return; } else { // keep old one @@ -588,9 +581,6 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { ++filter_iterator; } - - - } @@ -670,13 +660,12 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { } - //reporter->Error("Expiring element"); i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); + i->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); + i->lastDict->Clear(); // should be empty... but... well... who knows... delete(i->lastDict); i->lastDict = i->currDict; @@ -699,11 +688,6 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { } else { RecordVal * r = new RecordVal(i->rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - for ( int j = 0; j < i->rtype->NumFields(); j++) { Val* val = 0; From 5983d44d950d2b93d91c36846668b35fb24d437a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 8 Nov 2011 15:33:32 -0800 Subject: [PATCH 18/58] read header line in bro logfile format --- src/InputReaderAscii.cc | 19 +++++++++++++++++-- src/InputReaderAscii.h | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 39635de407..e4770421c6 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -62,7 +62,7 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const bool InputReaderAscii::ReadHeader() { // try to read the header line... string line; - if ( !getline(*file, line) ) { + if ( !GetLine(line) ) { Error("could not read first line"); return false; } @@ -111,6 +111,21 @@ bool InputReaderAscii::ReadHeader() { return true; } +bool InputReaderAscii::GetLine(string& str) { + while ( getline(*file, str) ) { + if ( str[0] != '#' ) { + return true; + } + + if ( str.compare(0,8, "#fields\t") == 0 ) { + str = str.substr(8); + return true; + } + } + + return false; +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -143,7 +158,7 @@ bool InputReaderAscii::DoUpdate() { //map *newKeyMap = new map(); string line; - while ( getline(*file, line ) ) { + while ( GetLine(line ) ) { // split on tabs istringstream splitstream(line); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 673a2cdae2..f86cfd0062 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -37,8 +37,9 @@ protected: virtual bool DoUpdate(); private: - bool ReadHeader(); + + bool GetLine(string& str); ifstream* file; string fname; From 1a642f3568c3c8c445f76a5b7438f385b880bbe7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 14 Nov 2011 17:18:28 -0800 Subject: [PATCH 19/58] tried enum support - doesn't yet work due to internal bro interface problems... --- src/InputMgr.cc | 6 ++++++ src/InputReaderAscii.cc | 2 +- src/input.bif | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 18b17c7576..8db5a70c66 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -828,6 +828,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { length += val->val.string_val->size(); break; @@ -883,6 +884,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals break; case TYPE_STRING: + case TYPE_ENUM: { memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); position += val->val.string_val->size(); @@ -958,6 +960,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_ENUM: + reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + + default: reporter->InternalError("unsupported type for input_read"); } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index e4770421c6..f91c55b666 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -128,7 +128,6 @@ bool InputReaderAscii::GetLine(string& str) { // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { - // dirty, fix me. (well, apparently after trying seeking, etc - this is not that bad) if ( file && file->is_open() ) { @@ -198,6 +197,7 @@ bool InputReaderAscii::DoUpdate() { //bzero(val, sizeof(LogVal)); switch ( currMapping.type ) { + case TYPE_ENUM: case TYPE_STRING: val->val.string_val = new string(s); break; diff --git a/src/input.bif b/src/input.bif index c4bf14e3ed..7b051fba16 100644 --- a/src/input.bif +++ b/src/input.bif @@ -51,3 +51,4 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} + From cde8153c1868f6bd3984a2c35aeb645b1023968d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 08:36:03 -0800 Subject: [PATCH 20/58] switch to set if record or simple value is desired. --- scripts/base/frameworks/input/main.bro | 1 + src/InputMgr.cc | 14 ++++++++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 6e19452f7a..2b87ac980c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,6 +9,7 @@ export { idx: any; val: any; destination: any; + want_record: bool &default=T; reader: Reader &default=default_reader; }; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8db5a70c66..5f99583c10 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -42,6 +42,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; + bool want_record; TableVal* tab; RecordType* rtype; @@ -147,6 +148,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -181,6 +183,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->itype = idx->Ref()->AsRecordType(); info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); + info->want_record = ( want_record->InternalInt() == 1 ); + + if ( valfields > 1 ) { + assert(info->want_record); + } readers.push_back(info); @@ -311,7 +318,6 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec if ( rec->FieldType(i)->Tag() == TYPE_RECORD ) { - string prep = nameprepend + rec->FieldName(i) + "."; if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) ) @@ -464,7 +470,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -683,7 +689,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { Val* valval; int position = i->num_idx_fields; - if ( i->num_val_fields == 1 ) { + if ( i->num_val_fields == 1 && !i->want_record ) { valval = LogValToVal(vals[i->num_idx_fields]); } else { RecordVal * r = new RecordVal(i->rtype); @@ -961,7 +967,7 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; case TYPE_ENUM: - reporter->InternalError("Sorry, Enums reading does not yet work, missing internal inferface"); + reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); default: From 4a3c9923253c8c26973a24c537a3ca231961db5e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 10:57:45 -0800 Subject: [PATCH 21/58] InputReader can read Sets. --- src/InputMgr.cc | 244 +++++++++++++++++++++++++--------------- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 211 ++++++++++++++++++++++------------ src/InputReaderAscii.h | 5 + src/LogMgr.cc | 6 +- src/LogMgr.h | 4 +- 6 files changed, 310 insertions(+), 162 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 5f99583c10..ba17d8448f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -234,11 +234,14 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_TABLE: - return false; + { + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } case TYPE_VECTOR: { - return IsCompatibleType(t->AsVectorType()->YieldType()); + return false; // do me... + //return IsCompatibleType(t->AsVectorType()->YieldType()); } default: @@ -329,6 +332,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec LogField* field = new LogField(); field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); + if ( field->type == TYPE_TABLE ) { + field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } fields->push_back(field); } @@ -810,49 +816,133 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ } + +int InputMgr::GetLogValLength(const LogVal* val) { + int length = 0; + + switch (val->type) { + case TYPE_BOOL: + case TYPE_INT: + length += sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + length += sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + length += sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + length += val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + length += NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: + length += sizeof(val->val.subnet_val.width); + length += sizeof(val->val.subnet_val.net); + break; + + case TYPE_TABLE: { + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += GetLogValLength(val->val.set_val.vals[i]); + } + break; + } + + default: + reporter->InternalError("unsupported type %d for GetLogValLength", val->type); + } + + return length; + +} + +int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { + switch ( val->type ) { + case TYPE_BOOL: + case TYPE_INT: + //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, startpos); + memcpy(data+startpos, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); + //*(data+startpos) = val->val.int_val; + return sizeof(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + //*(data+startpos) = val->val.uint_val; + memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); + return sizeof(val->val.uint_val); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + //*(data+startpos) = val->val.double_val; + memcpy(data+startpos, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); + return sizeof(val->val.double_val); + break; + + case TYPE_STRING: + case TYPE_ENUM: + { + memcpy(data+startpos, val->val.string_val->c_str(), val->val.string_val->length()); + return val->val.string_val->size(); + break; + } + + case TYPE_ADDR: + memcpy(data+startpos, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); + return NUM_ADDR_WORDS*sizeof(uint32_t); + break; + + case TYPE_SUBNET: { + int length = 0; + memcpy(data+startpos,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); + length += sizeof(val->val.subnet_val.width); + memcpy(data+startpos, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); + length += sizeof(val->val.subnet_val.net); + return length; + break; + } + + case TYPE_TABLE: { + int length = 0; + for ( int i = 0; i < val->val.set_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.set_val.vals[i]); + } + return length; + break; + } + + default: + reporter->InternalError("unsupported type %d for CopyLogVal", val->type); + return 0; + } + + reporter->InternalError("internal error"); + return 0; + +} + HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals) { int length = 0; for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch (val->type) { - case TYPE_BOOL: - case TYPE_INT: - length += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - length += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - length += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - length += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - length += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - length += sizeof(val->val.subnet_val.width); - length += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals"); - } - + length += GetLogValLength(val); } //reporter->Error("Length: %d", length); @@ -864,56 +954,7 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } for ( int i = 0; i < num_elements; i++ ) { const LogVal* val = vals[i]; - switch ( val->type ) { - case TYPE_BOOL: - case TYPE_INT: - //reporter->Error("Adding field content to pos %d: %lld", val->val.int_val, position); - memcpy(data+position, (const void*) &(val->val.int_val), sizeof(val->val.int_val)); - //*(data+position) = val->val.int_val; - position += sizeof(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - //*(data+position) = val->val.uint_val; - memcpy(data+position, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); - position += sizeof(val->val.uint_val); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - //*(data+position) = val->val.double_val; - memcpy(data+position, (const void*) &(val->val.double_val), sizeof(val->val.double_val)); - position += sizeof(val->val.double_val); - break; - - case TYPE_STRING: - case TYPE_ENUM: - { - memcpy(data+position, val->val.string_val->c_str(), val->val.string_val->length()); - position += val->val.string_val->size(); - break; - } - - case TYPE_ADDR: - memcpy(data+position, val->val.addr_val, NUM_ADDR_WORDS*sizeof(uint32_t)); - position += NUM_ADDR_WORDS*sizeof(uint32_t); - break; - - case TYPE_SUBNET: - memcpy(data+position,(const char*) &(val->val.subnet_val.width), sizeof(val->val.subnet_val.width) ); - position += sizeof(val->val.subnet_val.width); - memcpy(data+position, (const char*) &(val->val.subnet_val.net), sizeof(val->val.subnet_val.net) ); - position += sizeof(val->val.subnet_val.net); - break; - - default: - reporter->InternalError("unsupported type for hashlogvals2"); - } - - + position += CopyLogVal(data, position, val); } assert(position == length); @@ -966,6 +1007,29 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new SubNetVal(val->val.subnet_val.net, val->val.subnet_val.width); break; + case TYPE_TABLE: { + if ( val->val.set_val.size == 0 ) { + // empty table + TypeList* set_index = new TypeList(base_type(TYPE_ANY)); + // iim quite sure this does not work... we probably need the internal set type for this... + reporter->InternalError("Implement me."); + return new TableVal(new SetType(set_index, 0)); + } else { + // all entries have to have the same type... + TypeTag type = val->val.set_val.vals[0]->type; + TypeList* set_index = new TypeList(base_type(type)); + set_index->Append(base_type(type)); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + assert( val->val.set_val.vals[i]->type == type); + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; + } + break; + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 74914e65ad..d4bfa5c355 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -52,6 +52,8 @@ private: void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); + int GetLogValLength(const LogVal* val); + int CopyLogVal(char *data, const int startpos, const LogVal* val); Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index f91c55b666..60a8c5685a 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,12 +11,22 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) + : name(arg_name), type(arg_type), set_type(arg_set_type) +{ + position = arg_position; +} + FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type) + : name(arg.name), type(arg.type), set_type(arg.set_type) { position = arg.position; } +FieldMapping FieldMapping::setType() { + return FieldMapping(name, set_type, position); +} + InputReaderAscii::InputReaderAscii() { //DBG_LOG(DBG_LOGGING, "input reader initialized"); @@ -81,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, i); + FieldMapping f(field->name, field->type, field->set_type, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -126,6 +136,132 @@ bool InputReaderAscii::GetLine(string& str) { return false; } + +LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { + + LogVal* val = new LogVal(field.type, true); + //bzero(val, sizeof(LogVal)); + + switch ( field.type ) { + case TYPE_ENUM: + case TYPE_STRING: + val->val.string_val = new string(s); + break; + + case TYPE_BOOL: + if ( s == "T" ) { + val->val.int_val = 1; + } else if ( s == "F" ) { + val->val.int_val = 0; + } else { + Error(Fmt("Invalid value for boolean: %s", s.c_str())); + return false; + } + break; + + case TYPE_INT: + val->val.int_val = atoi(s.c_str()); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + val->val.double_val = atof(s.c_str()); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + val->val.uint_val = atoi(s.c_str()); + break; + + case TYPE_SUBNET: { + int pos = s.find("/"); + string width = s.substr(pos+1); + val->val.subnet_val.width = atoi(width.c_str()); + string addr = s.substr(0, pos); + s = addr; + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + val->val.subnet_val.net = dotted_to_addr(s.c_str()); + break; + + } + case TYPE_ADDR: { + // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // Solve this some other time.... + addr_type t = dotted_to_addr(s.c_str()); +#ifdef BROv6 + copy_addr(t, val->val.addr_val); +#else + copy_addr(&t, val->val.addr_val); +#endif + break; + } + + case TYPE_TABLE: { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } + + // how many entries do we have... + unsigned int length = 1; + for ( unsigned int i = 0; i < s.size(); i++ ) + if ( s[i] == ',') length++; + + unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + + istringstream splitstream(s); + while ( splitstream ) { + string element; + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); + break; + } + + if ( !getline(splitstream, element, ',') ) + break; + + + LogVal* newval = EntryToVal(element, field.setType()); + if ( newval == 0 ) { + Error("Error while reading set"); + return 0; + } + lvals[pos] = newval; + + pos++; + + } + + if ( pos != length ) { + Error("Internal error while parsing set: did not find all elements"); + return 0; + } + + break; + } + + + default: + Error(Fmt("unsupported field format %d for %s", field.type, + field.name.c_str())); + return 0; + } + + return val; + +} + // read the entire file and send appropriate thingies back to InputMgr bool InputReaderAscii::DoUpdate() { @@ -161,7 +297,6 @@ bool InputReaderAscii::DoUpdate() { // split on tabs istringstream splitstream(line); - string s; LogVal** fields = new LogVal*[num_fields]; //string string_fields[num_fields]; @@ -170,6 +305,7 @@ bool InputReaderAscii::DoUpdate() { unsigned int currField = 0; while ( splitstream ) { + string s; if ( !getline(splitstream, s, '\t') ) break; @@ -193,73 +329,10 @@ bool InputReaderAscii::DoUpdate() { return false; } - LogVal* val = new LogVal(currMapping.type, true); - //bzero(val, sizeof(LogVal)); - - switch ( currMapping.type ) { - case TYPE_ENUM: - case TYPE_STRING: - val->val.string_val = new string(s); - break; - - case TYPE_BOOL: - if ( s == "T" ) { - val->val.int_val = 1; - } else if ( s == "F" ) { - val->val.int_val = 0; - } else { - Error(Fmt("Invalid value for boolean: %s", s.c_str())); - return false; - } - break; - - case TYPE_INT: - val->val.int_val = atoi(s.c_str()); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - val->val.double_val = atof(s.c_str()); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - val->val.uint_val = atoi(s.c_str()); - break; - - case TYPE_SUBNET: { - int pos = s.find("/"); - string width = s.substr(pos+1); - val->val.subnet_val.width = atoi(width.c_str()); - string addr = s.substr(0, pos); - s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - val->val.subnet_val.net = dotted_to_addr(s.c_str()); - break; - - } - case TYPE_ADDR: { - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. - // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); -#ifdef BROv6 - copy_addr(t, val->val.addr_val); -#else - copy_addr(&t, val->val.addr_val); -#endif - break; - } - - - default: - Error(Fmt("unsupported field format %d for %s", currMapping.type, - currMapping.name.c_str())); + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { return false; - } - + } fields[currMapping.position] = val; //string_fields[currMapping.position] = s; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index f86cfd0062..ab2b89339c 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,11 +12,15 @@ struct FieldMapping { string name; TypeTag type; + TypeTag set_type; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } + + FieldMapping setType(); bool IsEmpty() { return position == -1; } }; @@ -38,6 +42,7 @@ protected: private: bool ReadHeader(); + LogVal* EntryToVal(string s, FieldMapping type); bool GetLine(string& str); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f78a2a19e0..9818d9cdfb 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -81,16 +81,18 @@ struct LogMgr::Stream { bool LogField::Read(SerializationFormat* fmt) { int t; + int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); type = (TypeTag) t; + set_type = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..40dab8677b 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,10 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; + // needed by input framework. otherwise it cannot determine the inner type of a set. + TypeTag set_type; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type) { } + : name(other.name), type(other.type), set_type(other.set_type) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From 3d0162bcdc2e2a3e4923ac22ec5597cb4d167304 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 11:18:48 -0800 Subject: [PATCH 22/58] isCompatibleType works correctly for tables. --- src/InputMgr.cc | 27 +++++++++++++++++---------- src/InputMgr.h | 2 +- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index ba17d8448f..4506501c94 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -205,7 +205,8 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return reader_obj; } -bool InputMgr::IsCompatibleType(BroType* t) + +bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) return false; @@ -223,24 +224,29 @@ bool InputMgr::IsCompatibleType(BroType* t) case TYPE_INTERVAL: case TYPE_ENUM: case TYPE_STRING: - case TYPE_RECORD: - // for record: check, if all elements are compatible? But... LogMgr also doesn't do this. - // ^ recursive checking is done in UnrollRecordType. return true; - - case TYPE_FILE: - case TYPE_FUNC: - return false; + case TYPE_RECORD: + return ! atomic_only; case TYPE_TABLE: { - return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); } case TYPE_VECTOR: { return false; // do me... + + //if ( atomic_only ) + // return false; + // //return IsCompatibleType(t->AsVectorType()->YieldType()); } @@ -249,7 +255,8 @@ bool InputMgr::IsCompatibleType(BroType* t) } return false; - } +} + bool InputMgr::RemoveReader(const EnumVal* id) { ReaderInfo *i = 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index d4bfa5c355..93c6447467 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -46,7 +46,7 @@ protected: private: struct ReaderInfo; - bool IsCompatibleType(BroType* t); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); From fb5f26e7fcf5d3bda5c25472e2933e7ed975ddb1 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 15:23:46 -0800 Subject: [PATCH 23/58] make default values work (thanks to robin) --- src/InputMgr.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4506501c94..a0a9d6a35c 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -97,7 +97,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) return 0; } - EnumVal* reader = description->Lookup(rtype->FieldOffset("reader"))->AsEnumVal(); + EnumVal* reader = description->LookupWithDefault(rtype->FieldOffset("reader"))->AsEnumVal(); while ( true ) { if ( ir->type == BifEnum::Input::READER_DEFAULT ) @@ -148,7 +148,6 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - Val *want_record = description->Lookup(rtype->FieldOffset("want_record")); vector fieldsV; // vector, because we don't know the length beforehands @@ -163,9 +162,11 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) if ( status ) { reporter->Error("Problem unrolling"); + Unref(reader); return 0; } + Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { @@ -174,7 +175,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; - info->type = reader->Ref()->AsEnumVal(); + info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault info->num_idx_fields = idxfields; info->num_val_fields = valfields; info->tab = dst->Ref()->AsTableVal(); @@ -184,6 +185,7 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) info->currDict = new PDict(InputHash); info->lastDict = new PDict(InputHash); info->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { assert(info->want_record); From 821878835a800edb794c25c32a59ef67958a87da Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 15 Nov 2011 16:32:35 -0800 Subject: [PATCH 24/58] read vector. still missing: enums, empty fields for optional parameters. --- src/InputMgr.cc | 45 +++++++++++++++++++++++++++------ src/InputReaderAscii.cc | 56 ++++++++++++++++++++++++++--------------- src/InputReaderAscii.h | 6 ++--- src/LogMgr.cc | 6 ++--- src/LogMgr.h | 6 ++--- 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index a0a9d6a35c..4c21268c84 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -244,12 +244,10 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) case TYPE_VECTOR: { - return false; // do me... - - //if ( atomic_only ) - // return false; - // - //return IsCompatibleType(t->AsVectorType()->YieldType()); + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType(), true); } default: @@ -342,7 +340,9 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->name = nameprepend + rec->FieldName(i); field->type = rec->FieldType(i)->Tag(); if ( field->type == TYPE_TABLE ) { - field->set_type = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); + } else if ( field->type == TYPE_VECTOR ) { + field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); } fields->push_back(field); @@ -870,6 +870,13 @@ int InputMgr::GetLogValLength(const LogVal* val) { break; } + case TYPE_VECTOR: { + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += GetLogValLength(val->val.vector_val.vals[i]); + } + break; + } + default: reporter->InternalError("unsupported type %d for GetLogValLength", val->type); } @@ -936,6 +943,15 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { break; } + case TYPE_VECTOR: { + int length = 0; + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); + } + return length; + break; + } + default: reporter->InternalError("unsupported type %d for CopyLogVal", val->type); return 0; @@ -1039,6 +1055,21 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { break; } + case TYPE_VECTOR: { + assert ( val->val.vector_val.size > 1 ); // implement empty vector... + + // all entries have to have the same type... + TypeTag type = val->val.vector_val.vals[0]->type; + VectorType* vt = new VectorType(base_type(type)); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + assert( val->val.vector_val.vals[i]->type == type); + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; + + } + case TYPE_ENUM: reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 60a8c5685a..3b4409e652 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -11,20 +11,20 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int position = arg_position; } -FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position) - : name(arg_name), type(arg_type), set_type(arg_set_type) +FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) + : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; } FieldMapping::FieldMapping(const FieldMapping& arg) - : name(arg.name), type(arg.type), set_type(arg.set_type) + : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; } -FieldMapping FieldMapping::setType() { - return FieldMapping(name, set_type, position); +FieldMapping FieldMapping::subType() { + return FieldMapping(name, subtype, position); } InputReaderAscii::InputReaderAscii() @@ -91,7 +91,7 @@ bool InputReaderAscii::ReadHeader() { const LogField* field = fields[i]; if ( field->name == s ) { // cool, found field. note position - FieldMapping f(field->name, field->type, field->set_type, i); + FieldMapping f(field->name, field->type, field->subtype, i); columnMap.push_back(f); wantFields++; break; // done with searching @@ -112,7 +112,7 @@ bool InputReaderAscii::ReadHeader() { if ( wantFields != (int) num_fields ) { // we did not find all fields? // :( - Error("One of the requested fields could not be found in the input data file"); + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); return false; } @@ -199,25 +199,40 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - case TYPE_TABLE: { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } - + case TYPE_TABLE: + case TYPE_VECTOR: + // First - common initialization + // Then - initialization for table. + // Then - initialization for vector. + // Then - common stuff + { // how many entries do we have... unsigned int length = 1; for ( unsigned int i = 0; i < s.size(); i++ ) if ( s[i] == ',') length++; unsigned int pos = 0; + LogVal** lvals = new LogVal* [length]; - val->val.set_val.vals = lvals; - val->val.set_val.size = length; + + if ( field.type == TYPE_TABLE ) { + // construct a table from entry... + // for the moment assume, that entries are split by ",". + + /* Fix support for emtyp tables if ( s == "-" ) { + // empty + val->val.set_val.size = 0; + break; + } */ + + val->val.set_val.vals = lvals; + val->val.set_val.size = length; + } else if ( field.type == TYPE_VECTOR ) { + val->val.vector_val.vals = lvals; + val->val.vector_val.size = length; + } else { + assert(false); + } istringstream splitstream(s); while ( splitstream ) { @@ -232,7 +247,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; - LogVal* newval = EntryToVal(element, field.setType()); + LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { Error("Error while reading set"); return 0; @@ -243,6 +258,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { } + if ( pos != length ) { Error("Internal error while parsing set: did not find all elements"); return 0; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index ab2b89339c..56c1001acb 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,15 +12,15 @@ struct FieldMapping { string name; TypeTag type; - TypeTag set_type; + TypeTag subtype; int position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); - FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_set_type, int arg_position); + FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); FieldMapping() { position = -1; } - FieldMapping setType(); + FieldMapping subType(); bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 9818d9cdfb..6eaace3893 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,16 +83,16 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "set_type") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; - set_type = (TypeTag) it; + subtype = (TypeTag) it; return success; } bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)set_type, "set_type")); + return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() diff --git a/src/LogMgr.h b/src/LogMgr.h index 40dab8677b..b8530d29ab 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -15,12 +15,12 @@ class SerializationFormat; struct LogField { string name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set. - TypeTag set_type; + // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), set_type(other.set_type) { } + : name(other.name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); From ab68d8400789156a23b1e6bbaf57828e7eac7c83 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:13:36 -0800 Subject: [PATCH 25/58] reading of enum types (thanks, Seth) --- src/InputMgr.cc | 50 +++++++++++++++++++++++++++------------------- src/InputMgr.h | 4 ++-- src/InputReader.cc | 3 ++- src/InputReader.h | 2 +- 4 files changed, 34 insertions(+), 25 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 4c21268c84..64f3e2eb9b 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -422,7 +422,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( num_fields == 1 && type->FieldType(0)->Tag() != TYPE_RECORD ) { - idxval = LogValToVal(vals[0]); + idxval = LogValToVal(vals[0], type->FieldType(0)); position = 1; } else { ListVal *l = new ListVal(TYPE_ANY); @@ -430,7 +430,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo if ( type->FieldType(j)->Tag() == TYPE_RECORD ) { l->Append(LogValToRecordVal(vals, type->FieldType(j)->AsRecordType(), &position)); } else { - l->Append(LogValToVal(vals[position], type->FieldType(j)->Tag())); + l->Append(LogValToVal(vals[position], type->FieldType(j))); position++; } } @@ -486,7 +486,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -501,7 +501,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -705,7 +705,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { int position = i->num_idx_fields; if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields]); + valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); } else { RecordVal * r = new RecordVal(i->rtype); @@ -715,7 +715,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)->Tag()); + val = LogValToVal(vals[position], i->rtype->FieldType(j)); position++; } @@ -760,7 +760,7 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } - +/* Does not work atm, because LogValToVal needs BroType void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); @@ -775,7 +775,7 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c } mgr.Dispatch(new Event(handler, vl)); -} +} */ void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) { @@ -814,7 +814,7 @@ Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_ if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { fieldVal = LogValToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position); } else { - fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)->Tag()); + fieldVal = LogValToVal(vals[*position], request_type->FieldType(i)); (*position)++; } @@ -988,10 +988,10 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } -Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { +Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { - if ( request_type != TYPE_ANY && request_type != val->type ) { - reporter->InternalError("Typetags don't match: %d vs %d", request_type, val->type); + if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { + reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } @@ -1041,13 +1041,12 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { return new TableVal(new SetType(set_index, 0)); } else { // all entries have to have the same type... - TypeTag type = val->val.set_val.vals[0]->type; - TypeList* set_index = new TypeList(base_type(type)); - set_index->Append(base_type(type)); + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); SetType* s = new SetType(set_index, 0); TableVal* t = new TableVal(s); for ( int i = 0; i < val->val.set_val.size; i++ ) { - assert( val->val.set_val.vals[i]->type == type); t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); } return t; @@ -1059,19 +1058,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, TypeTag request_type) { assert ( val->val.vector_val.size > 1 ); // implement empty vector... // all entries have to have the same type... - TypeTag type = val->val.vector_val.vals[0]->type; - VectorType* vt = new VectorType(base_type(type)); + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) { - assert( val->val.vector_val.vals[i]->type == type); v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); } return v; } - case TYPE_ENUM: - reporter->InternalError("Sorry, Enum reading does not yet work, missing internal inferface"); + case TYPE_ENUM: { + // well, this is kind of stupid, because EnumType just mangles the module name and the var name together again... + // but well + string module = extract_module_name(val->val.string_val->c_str()); + string var = extract_var_name(val->val.string_val->c_str()); + bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); + if ( index == -1 ) { + reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s", module.c_str(), var.c_str()); + } + return new EnumVal(index, request_type->Ref()->AsEnumType() ); + break; + } default: diff --git a/src/InputMgr.h b/src/InputMgr.h index 93c6447467..17b7e2e804 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -55,11 +55,11 @@ private: int GetLogValLength(const LogVal* val); int CopyLogVal(char *data, const int startpos, const LogVal* val); - Val* LogValToVal(const LogVal* val, TypeTag request_type = TYPE_ANY); + Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 994f8b9b97..7403d1f989 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -64,10 +64,11 @@ bool InputReader::Update() return DoUpdate(); } +/* void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { input_mgr->SendEvent(name, num_vals, vals); -} +} */ // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index b547d29506..41000e4c0c 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -40,7 +40,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); void Put(const LogVal* const *val); void Clear(); From 4fef1e3f8c5fe7005f23f91d6001cd1d79ffcdd2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 22:47:28 -0800 Subject: [PATCH 26/58] set & entry separator configuration (with the restriction that they have to be exactly one character long) --- .../base/frameworks/input/readers/ascii.bro | 19 +++++++++ src/InputReaderAscii.cc | 40 +++++++++++++++++-- src/InputReaderAscii.h | 15 ++++++- src/input.bif | 9 +++++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 scripts/base/frameworks/input/readers/ascii.bro diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro new file mode 100644 index 0000000000..9f630975a2 --- /dev/null +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -0,0 +1,19 @@ +##! Interface for the ascii input reader. + +module InputAscii; + +export { + ## Separator between fields. + ## Please note that the separator has to be exactly one character long + const separator = "\t" &redef; + + ## Separator between set elements. + ## Please note that the separator has to be exactly one character long + const set_separator = "," &redef; + + ## String to use for empty fields. + const empty_field = "-" &redef; + + ## String to use for an unset &optional field. + const unset_field = "-" &redef; +} diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 3b4409e652..22cdcfdcf0 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -2,6 +2,7 @@ #include "InputReaderAscii.h" #include "DebugLogger.h" +#include "NetVar.h" #include @@ -29,15 +30,46 @@ FieldMapping FieldMapping::subType() { InputReaderAscii::InputReaderAscii() { - //DBG_LOG(DBG_LOGGING, "input reader initialized"); file = 0; //keyMap = new map(); + + separator_len = BifConst::LogAscii::separator->Len(); + separator = new char[separator_len]; + memcpy(separator, BifConst::LogAscii::separator->Bytes(), + separator_len); + if ( separator_len != 1 ) { + Error("separator length has to be 1. Separator will be truncated."); + } + + set_separator_len = BifConst::LogAscii::set_separator->Len(); + set_separator = new char[set_separator_len]; + memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), + set_separator_len); + if ( set_separator_len != 1 ) { + Error("set_separator length has to be 1. Separator will be truncated."); + } + + empty_field_len = BifConst::LogAscii::empty_field->Len(); + empty_field = new char[empty_field_len]; + memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), + empty_field_len); + + unset_field_len = BifConst::LogAscii::unset_field->Len(); + unset_field = new char[unset_field_len]; + memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), + unset_field_len); + } InputReaderAscii::~InputReaderAscii() { DoFinish(); + + delete [] separator; + delete [] set_separator; + delete [] empty_field; + delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -83,7 +115,7 @@ bool InputReaderAscii::ReadHeader() { int wantFields = 0; while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t')) + if ( !getline(splitstream, s, separator[0])) break; // current found heading in s... compare if we want it @@ -243,7 +275,7 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { break; } - if ( !getline(splitstream, element, ',') ) + if ( !getline(splitstream, element, set_separator[0]) ) break; @@ -322,7 +354,7 @@ bool InputReaderAscii::DoUpdate() { while ( splitstream ) { string s; - if ( !getline(splitstream, s, '\t') ) + if ( !getline(splitstream, s, separator[0]) ) break; diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 56c1001acb..d69b8c04bc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -57,7 +57,20 @@ private: const LogField* const * fields; // raw mapping //map *keyMap; - + // + // Options set from the script-level. + char* separator; + int separator_len; + + char* set_separator; + int set_separator_len; + + char* empty_field; + int empty_field_len; + + char* unset_field; + int unset_field_len; + }; diff --git a/src/input.bif b/src/input.bif index 7b051fba16..aaef25dcc3 100644 --- a/src/input.bif +++ b/src/input.bif @@ -52,3 +52,12 @@ function Input::__remove_filter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +# Options for Ascii Reader + +module InputAscii; + +const separator: string; +const set_separator: string; +const empty_field: string; +const unset_field: string; + From 4dd95fcf3cf1efc83ec4d2c53fee765d2a800674 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 16 Nov 2011 23:51:51 -0800 Subject: [PATCH 27/58] support for uninitialized fields & empty sets and tables. The only snag is... with the default output format of the log-file writer, the input reader cannot tell if a table or set is empty or uninitialized (both cases use the same character by default). In this case, by default it is assumed that the field/vector is uninitalized. --- scripts/base/frameworks/input/__load__.bro | 2 + src/InputMgr.cc | 61 ++++++++++------------ src/InputReaderAscii.cc | 61 ++++++++-------------- src/InputReaderAscii.h | 12 ++--- 4 files changed, 57 insertions(+), 79 deletions(-) diff --git a/scripts/base/frameworks/input/__load__.bro b/scripts/base/frameworks/input/__load__.bro index a10fe855df..a3315186d5 100644 --- a/scripts/base/frameworks/input/__load__.bro +++ b/scripts/base/frameworks/input/__load__.bro @@ -1 +1,3 @@ @load ./main +@load ./readers/ascii + diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 64f3e2eb9b..9c0e9c12b5 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -505,10 +505,10 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { position++; } - if ( val == 0 ) { + /* if ( val == 0 ) { reporter->InternalError("conversion error"); return; - } + } */ r->Assign(j,val); @@ -871,7 +871,9 @@ int InputMgr::GetLogValLength(const LogVal* val) { } case TYPE_VECTOR: { - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { + reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; @@ -945,7 +947,8 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_VECTOR: { int length = 0; - for ( int i = 0; i < val->val.vector_val.size; i++ ) { + int j = val->val.vector_val.size; + for ( int i = 0; i < j; i++ ) { length += CopyLogVal(data, startpos+length, val->val.vector_val.vals[i]); } return length; @@ -994,6 +997,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { reporter->InternalError("Typetags don't match: %d vs %d", request_type->Tag(), val->type); return 0; } + + if ( !val->present ) { + return 0; // unset field + } switch ( val->type ) { @@ -1033,38 +1040,28 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { break; case TYPE_TABLE: { - if ( val->val.set_val.size == 0 ) { - // empty table - TypeList* set_index = new TypeList(base_type(TYPE_ANY)); - // iim quite sure this does not work... we probably need the internal set type for this... - reporter->InternalError("Implement me."); - return new TableVal(new SetType(set_index, 0)); - } else { - // all entries have to have the same type... - BroType* type = request_type->AsTableType()->Indices()->PureType(); - TypeList* set_index = new TypeList(type->Ref()); - set_index->Append(type->Ref()); - SetType* s = new SetType(set_index, 0); - TableVal* t = new TableVal(s); - for ( int i = 0; i < val->val.set_val.size; i++ ) { - t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return t; - } + // all entries have to have the same type... + BroType* type = request_type->AsTableType()->Indices()->PureType(); + TypeList* set_index = new TypeList(type->Ref()); + set_index->Append(type->Ref()); + SetType* s = new SetType(set_index, 0); + TableVal* t = new TableVal(s); + for ( int i = 0; i < val->val.set_val.size; i++ ) { + t->Assign(LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return t; break; } case TYPE_VECTOR: { - assert ( val->val.vector_val.size > 1 ); // implement empty vector... - - // all entries have to have the same type... - BroType* type = request_type->AsVectorType()->YieldType(); - VectorType* vt = new VectorType(type->Ref()); - VectorVal* v = new VectorVal(vt); - for ( int i = 0; i < val->val.vector_val.size; i++ ) { - v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); - } - return v; + // all entries have to have the same type... + BroType* type = request_type->AsVectorType()->YieldType(); + VectorType* vt = new VectorType(type->Ref()); + VectorVal* v = new VectorVal(vt); + for ( int i = 0; i < val->val.vector_val.size; i++ ) { + v->Assign(i, LogValToVal( val->val.set_val.vals[i], type ), 0); + } + return v; } diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 22cdcfdcf0..4a0d4157bc 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -34,31 +34,19 @@ InputReaderAscii::InputReaderAscii() //keyMap = new map(); - separator_len = BifConst::LogAscii::separator->Len(); - separator = new char[separator_len]; - memcpy(separator, BifConst::LogAscii::separator->Bytes(), - separator_len); - if ( separator_len != 1 ) { + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); + if ( separator.size() != 1 ) { Error("separator length has to be 1. Separator will be truncated."); } - set_separator_len = BifConst::LogAscii::set_separator->Len(); - set_separator = new char[set_separator_len]; - memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), - set_separator_len); - if ( set_separator_len != 1 ) { + set_separator.assign( (const char*) BifConst::InputAscii::set_separator->Bytes(), BifConst::InputAscii::set_separator->Len()); + if ( set_separator.size() != 1 ) { Error("set_separator length has to be 1. Separator will be truncated."); } - empty_field_len = BifConst::LogAscii::empty_field->Len(); - empty_field = new char[empty_field_len]; - memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), - empty_field_len); - - unset_field_len = BifConst::LogAscii::unset_field->Len(); - unset_field = new char[unset_field_len]; - memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), - unset_field_len); + empty_field.assign( (const char*) BifConst::InputAscii::empty_field->Bytes(), BifConst::InputAscii::empty_field->Len()); + + unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); } @@ -66,10 +54,6 @@ InputReaderAscii::~InputReaderAscii() { DoFinish(); - delete [] separator; - delete [] set_separator; - delete [] empty_field; - delete [] unset_field; } void InputReaderAscii::DoFinish() @@ -172,7 +156,10 @@ bool InputReaderAscii::GetLine(string& str) { LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { LogVal* val = new LogVal(field.type, true); - //bzero(val, sizeof(LogVal)); + + if ( s.compare(unset_field) == 0 ) { // field is not set... + return new LogVal(field.type, false); + } switch ( field.type ) { case TYPE_ENUM: @@ -244,19 +231,13 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { if ( s[i] == ',') length++; unsigned int pos = 0; + + if ( s.compare(empty_field) == 0 ) + length = 0; LogVal** lvals = new LogVal* [length]; if ( field.type == TYPE_TABLE ) { - // construct a table from entry... - // for the moment assume, that entries are split by ",". - - /* Fix support for emtyp tables if ( s == "-" ) { - // empty - val->val.set_val.size = 0; - break; - } */ - val->val.set_val.vals = lvals; val->val.set_val.size = length; } else if ( field.type == TYPE_VECTOR ) { @@ -266,18 +247,20 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { assert(false); } + if ( length == 0 ) + break; //empty + istringstream splitstream(s); while ( splitstream ) { string element; - if ( pos >= length ) { - Error(Fmt("Internal error while parsing set. pos %d > length %d", pos, length)); - break; - } - if ( !getline(splitstream, element, set_separator[0]) ) break; - + + if ( pos >= length ) { + Error(Fmt("Internal error while parsing set. pos %d >= length %d. Element: %s", pos, length, element.c_str())); + break; + } LogVal* newval = EntryToVal(element, field.subType()); if ( newval == 0 ) { diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index d69b8c04bc..c848c17110 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -59,17 +59,13 @@ private: //map *keyMap; // // Options set from the script-level. - char* separator; - int separator_len; + string separator; - char* set_separator; - int set_separator_len; + string set_separator; - char* empty_field; - int empty_field_len; + string empty_field; - char* unset_field; - int unset_field_len; + string unset_field; }; From e2c521fc4e6ef9eb66eb6cd431f9186468b14b70 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 18 Nov 2011 10:49:20 -0800 Subject: [PATCH 28/58] start reworking input framework... does not compile at the moment, but there are a few uncommitted changes that will be reverted in the next commit. --- scripts/base/frameworks/input/main.bro | 46 ++--- src/InputMgr.cc | 263 ++++++++++++------------- src/InputMgr.h | 18 +- src/InputReader.cc | 39 ++-- src/InputReader.h | 37 ++-- src/input.bif | 22 +-- 6 files changed, 208 insertions(+), 217 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 2b87ac980c..d9c0812498 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,30 +4,36 @@ module Input; export { const default_reader = READER_ASCII &redef; - type ReaderDescription: record { + type StreamDescription: record { source: string; - idx: any; - val: any; - destination: any; - want_record: bool &default=T; reader: Reader &default=default_reader; }; type Filter: record { - name: string; ## descriptive name. for later removal + name: string; + ## for tables + idx: any &optional; + val: any &optional; + destination: any &optional; + want_record: bool &default=T; + table_ev: any &optional; # event containing idx, val as values. + + ## decision function, that decides if an insertion, update or removal should really be executed. + ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; - ## decision function, that decides if an inserton, update or removal should really be executed + + ## for "normalized" events + ev: any &optional; + ev_description: any &optional; }; const no_filter: Filter = [$name=""]; # Sentinel. - global create_reader: function(id: Log::ID, description: Input::ReaderDescription) : bool; - global remove_reader: function(id: Log::ID) : bool; + global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_event: function(id: Log::ID, name: string) : bool; - global remove_event: function(id: Log::ID, name: string) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; global remove_filter: function(id: Log::ID, name: string) : bool; global get_filter: function(id: ID, name: string) : Filter; @@ -41,14 +47,14 @@ module Input; global filters: table[ID, string] of Filter; -function create_reader(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool { - return __create_reader(id, description); + return __create_stream(id, description); } -function remove_reader(id: Log::ID) : bool +function remove_stream(id: Log::ID) : bool { - return __remove_reader(id); + return __remove_stream(id); } function force_update(id: Log::ID) : bool @@ -56,16 +62,6 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_event(id: Log::ID, name: string) : bool - { - return __add_event(id, name); - } - -function remove_event(id: Log::ID, name: string) : bool - { - return __remove_event(id, name); - } - function add_filter(id: Log::ID, filter: Input::Filter) : bool { filters[id, filter$name] = filter; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 9c0e9c12b5..f9250f6f0f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -27,22 +27,11 @@ declare(PDict, InputHash); struct InputMgr::Filter { EnumVal* id; string name; - Func* pred; - ~Filter(); -}; - -InputMgr::Filter::~Filter() { - Unref(id); -} - -struct InputMgr::ReaderInfo { - EnumVal* id; - EnumVal* type; - InputReader* reader; unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; + EventHandlerPtr table_event; TableVal* tab; RecordType* rtype; @@ -50,18 +39,42 @@ struct InputMgr::ReaderInfo { PDict(InputHash)* currDict; PDict(InputHash)* lastDict; - - list events; // events we fire when "something" happens - list filters; // filters that can prevent our actions + + Func* pred; + + EventHandlerPtr event; + RecordType* event_type; + + ~Filter(); +}; + +InputMgr::Filter::~Filter() { + Unref(id); + if ( tab ) + Unref(tab); + if ( itype ) + Unref(itype); + if ( rtype ) + Unref(rtype); + if ( event_type) + Unref(event_type); +} + +struct InputMgr::ReaderInfo { + EnumVal* id; + EnumVal* type; + InputReader* reader; + + //list events; // events we fire when "something" happens + map filters; // filters that can prevent our actions ~ReaderInfo(); }; InputMgr::ReaderInfo::~ReaderInfo() { + // all the contents of filters should delete themselves automatically... + Unref(type); - Unref(tab); - Unref(itype); - Unref(rtype); Unref(id); delete(reader); @@ -86,14 +99,14 @@ InputMgr::InputMgr() } // create a new input reader object to be used at whomevers leisure lateron. -InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) +InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) { InputReaderDefinition* ir = input_readers; RecordType* rtype = description->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::ReaderDescription, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::StreamDescription, 0) ) { - reporter->Error("readerDescription argument not of right type"); + reporter->Error("Streamdescription argument not of right type"); return 0; } @@ -145,55 +158,15 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - RecordType *idx = description->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = description->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); - TableVal *dst = description->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); - - - vector fieldsV; // vector, because we don't know the length beforehands - - - bool status = !UnrollRecordType(&fieldsV, idx, ""); - - int idxfields = fieldsV.size(); - - status = status || !UnrollRecordType(&fieldsV, val, ""); - int valfields = fieldsV.size() - idxfields; - - if ( status ) { - reporter->Error("Problem unrolling"); - Unref(reader); - return 0; - } - - Val *want_record = description->LookupWithDefault(rtype->FieldOffset("want_record")); - - LogField** fields = new LogField*[fieldsV.size()]; - for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { - fields[i] = fieldsV[i]; - } ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault - info->num_idx_fields = idxfields; - info->num_val_fields = valfields; - info->tab = dst->Ref()->AsTableVal(); - info->rtype = val->Ref()->AsRecordType(); info->id = id->Ref()->AsEnumVal(); - info->itype = idx->Ref()->AsRecordType(); - info->currDict = new PDict(InputHash); - info->lastDict = new PDict(InputHash); - info->want_record = ( want_record->InternalInt() == 1 ); - Unref(want_record); // ref'd by lookupwithdefault - - if ( valfields > 1 ) { - assert(info->want_record); - } readers.push_back(info); - int success = reader_obj->Init(source, fieldsV.size(), idxfields, fields); + int success = reader_obj->Init(source); if ( success == false ) { assert( RemoveReader(id) ); return 0; @@ -208,6 +181,86 @@ InputReader* InputMgr::CreateReader(EnumVal* id, RecordVal* description) } +bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + + RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + + vector fieldsV; // vector, because we don't know the length beforehands + + bool status = !UnrollRecordType(&fieldsV, idx, ""); + + int idxfields = fieldsV.size(); + + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + LogField** fields = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + fields[i] = fieldsV[i]; + } + + Filter filter; + filter.name = name->AsString()->CheckString(); + filter.id = id->Ref()->AsEnumVal(); + filter.pred = pred ? pred->AsFunc() : 0; + filter.num_idx_fields = idxfields; + filter.num_val_fields = valfields; + filter.tab = dst ? dst->Ref()->AsTableVal() : 0; + filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; + filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.currDict = new PDict(InputHash); + filter.lastDict = new PDict(InputHash); + filter.want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + if ( valfields > 1 ) { + assert(info->want_record); + } + + i->filters[id->InternalInt()] = filter; + + // ok, now we have to alert the reader of our new filter with our funky new fields + // the id is handled in a ... well, to be honest, a little bit sneaky manner. + // the "problem" is, that we can have several filters in the reader for one filter in the log manager. + // this is due to the fact, that a filter can either output it's result as a table, as an event... + // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets + // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want + // to build a union of these values and figure it out later. + // hence -> filter id is multiplicated with 2. + // filterId*2 -> results for table + // filterId*2+1 -> results for event + i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + + return true; +} + + bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) { if ( ! t ) @@ -258,7 +311,7 @@ bool InputMgr::IsCompatibleType(BroType* t, bool atomic_only) } -bool InputMgr::RemoveReader(const EnumVal* id) { +bool InputMgr::RemoveStream(const EnumVal* id) { ReaderInfo *i = 0; for ( vector::iterator s = readers.begin(); s != readers.end(); ++s ) { @@ -281,42 +334,6 @@ bool InputMgr::RemoveReader(const EnumVal* id) { return true; } -bool InputMgr::RegisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - i->events.push_back(eventName); - - return true; -} - -// remove first event with name eventName -// (though there shouldn't really be several events with the same name... -bool InputMgr::UnregisterEvent(const EnumVal* id, string eventName) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->InternalError("Reader not found"); - return false; - } - - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) - { - if ( *it == eventName ) { - it = i->events.erase(it); - return true; - } - else - ++it; - } - - return false; -} - - bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend) { for ( int i = 0; i < rec->NumFields(); i++ ) { @@ -363,34 +380,6 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { - ReaderInfo *i = FindReader(id); - if ( i == 0 ) { - reporter->Error("Reader not found"); - return false; - } - - RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) - { - reporter->Error("filter argument not of right type"); - return false; - } - - - Val* name = fval->Lookup(rtype->FieldOffset("name")); - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); - - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - - i->filters.push_back(filter); - - return true; -} - bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -398,7 +387,7 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } - +/* std::list::iterator it = i->filters.begin(); while ( it != i->filters.end() ) { @@ -410,8 +399,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { else ++it; } + */ - return false;; + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + it->filters.erase(it); + return true; } @@ -444,7 +440,7 @@ Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const Lo } -void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -605,7 +601,7 @@ void InputMgr::SendEntry(const InputReader* reader, const LogVal* const *vals) { } -void InputMgr::EndCurrentSend(const InputReader* reader) { +void InputMgr::EndCurrentSend(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -693,7 +689,7 @@ void InputMgr::EndCurrentSend(const InputReader* reader) { i->currDict = new PDict(InputHash); } -void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { +void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -733,7 +729,7 @@ void InputMgr::Put(const InputReader* reader, const LogVal* const *vals) { i->tab->Assign(idxval, valval); } -void InputMgr::Clear(const InputReader* reader) { +void InputMgr::Clear(const InputReader* reader, int id) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); @@ -873,7 +869,6 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_VECTOR: { int j = val->val.vector_val.size; for ( int i = 0; i < j; i++ ) { - reporter->Error("size is %d", val->val.vector_val.size); length += GetLogValLength(val->val.vector_val.vals[i]); } break; diff --git a/src/InputMgr.h b/src/InputMgr.h index 17b7e2e804..1cacf89143 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -20,11 +20,9 @@ class InputMgr { public: InputMgr(); - InputReader* CreateReader(EnumVal* id, RecordVal* description); + InputReader* CreateStream(EnumVal* id, RecordVal* description); bool ForceUpdate(const EnumVal* id); - bool RemoveReader(const EnumVal* id); - bool RegisterEvent(const EnumVal* id, string eventName); - bool UnregisterEvent(const EnumVal* id, string eventName); + bool RemoveStream(const EnumVal* id); bool AddFilter(EnumVal *id, RecordVal* filter); bool RemoveFilter(EnumVal* id, const string &name); @@ -36,12 +34,14 @@ protected: // Reports an error for the given reader. void Error(InputReader* reader, const char* msg); - void Put(const InputReader* reader, const LogVal* const *vals); - void Clear(const InputReader* reader); - bool Delete(const InputReader* reader, const LogVal* const *vals); + // for readers to write to input stream in direct mode (reporting new/deleted values directly) + void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Clear(const InputReader* reader, int id); + bool Delete(const InputReader* reader, int id, const LogVal* const *vals); - void SendEntry(const InputReader* reader, const LogVal* const *vals); - void EndCurrentSend(const InputReader* reader); + // for readers to write to input stream in indirect mode (manager is monitoring new/deleted values) + void SendEntry(const InputReader* reader, int id, const LogVal* const *vals); + void EndCurrentSend(const InputReader* reader, int id); private: struct ReaderInfo; diff --git a/src/InputReader.cc b/src/InputReader.cc index 7403d1f989..1008cf1b67 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -24,35 +24,42 @@ void InputReader::Error(const string &msg) input_mgr->Error(this, msg.c_str()); } -void InputReader::Put(const LogVal* const *val) +void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, val); + input_mgr->Put(this, int id, val); } -void InputReader::Clear() +void InputReader::Clear(int id) { - input_mgr->Clear(this); + input_mgr->Clear(this, int id); } -void InputReader::Delete(const LogVal* const *val) +void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, val); + input_mgr->Delete(this, int id, val); } -bool InputReader::Init(string arg_source, int arg_num_fields, int arg_idx_fields, - const LogField* const * arg_fields) +bool InputReader::Init(string arg_source) { source = arg_source; - num_fields = arg_num_fields; - index_fields = arg_idx_fields; - fields = arg_fields; // disable if DoInit returns error. - disabled = !DoInit(arg_source, arg_num_fields, arg_idx_fields, arg_fields); + disabled = !DoInit(arg_source); return !disabled; } +bool InputReader::AddFilter(int id, int arg_num_fields, + const LogField* const * arg_fields) +{ + return DoAddFilter(int id, arg_num_fields, arg_fields); +} + +bool InputReader::RemoveFilter(int id) +{ + return DoRemoveFilter(int id); +} + void InputReader::Finish() { DoFinish(); @@ -96,12 +103,12 @@ const char* InputReader::Fmt(const char* format, ...) } -void InputReader::SendEntry(const LogVal* const *vals) +void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, vals); + input_mgr->SendEntry(this, int id, vals); } -void InputReader::EndCurrentSend() +void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this); + input_mgr->EndCurrentSend(this, int id); } diff --git a/src/InputReader.h b/src/InputReader.h index 41000e4c0c..12f0bc9db4 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -15,7 +15,11 @@ public: InputReader(); virtual ~InputReader(); - bool Init(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const* fields); + bool Init(string arg_source); + + bool AddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + bool RemoveFilter ( int id ); void Finish(); @@ -23,8 +27,11 @@ public: protected: // Methods that have to be overwritten by the individual readers - virtual bool DoInit(string arg_source, int arg_num_fields, int arg_idx_fields, const LogField* const * fields) = 0; - + virtual bool DoInit(string arg_sources) = 0; + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; + virtual bool DoRemoveFilter( int id ); + virtual void DoFinish() = 0; // update file contents to logmgr @@ -42,28 +49,26 @@ protected: //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); - void Put(const LogVal* const *val); - void Clear(); - void Delete(const LogVal* const *val); + // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table + void Put(int id, const LogVal* const *val); + void Delete(int id, const LogVal* const *val); + void Clear(int id); - void SendEntry(const LogVal* const *vals); - void EndCurrentSend(); + // Table-functions (tracking mode): Only changed lines are propagated. + void SendEntry(int id, const LogVal* const *vals); + void EndCurrentSend(int id); private: friend class InputMgr; string source; - int num_fields; - int index_fields; - const LogField* const * fields; - // When an error occurs, this method is called to set a flag marking the - // writer as disabled. + // When an error occurs, this method is called to set a flag marking the + // writer as disabled. - bool disabled; - - bool Disabled() { return disabled; } + bool disabled; + bool Disabled() { return disabled; } // For implementing Fmt(). char* buf; diff --git a/src/input.bif b/src/input.bif index aaef25dcc3..ef069316ab 100644 --- a/src/input.bif +++ b/src/input.bif @@ -7,18 +7,18 @@ module Input; #include "NetVar.h" %%} -type ReaderDescription: record; +type StreamDescription: record; type Filter: record; -function Input::__create_reader%(id: Log::ID, description: Input::ReaderDescription%) : bool +function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ - InputReader *the_reader = input_mgr->CreateReader(id->AsEnumVal(), description->AsRecordVal()); + InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_reader%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Log::ID%) : bool %{ - bool res = input_mgr->RemoveReader(id->AsEnumVal()); + bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} @@ -28,18 +28,6 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->RegisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - -function Input::__remove_event%(id: Log::ID, name: string%) : bool - %{ - bool res = input_mgr->UnregisterEvent(id->AsEnumVal(), name->AsString()->CheckString()); - return new Val( res, TYPE_BOOL ); - %} - function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool %{ bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); From b3f01915fbca99ddda434ae791b143150d27fcf7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:07:50 -0800 Subject: [PATCH 29/58] compiles with basic new filter framework - but crashes on use. --- scripts/base/frameworks/input/main.bro | 16 +- src/InputMgr.cc | 255 +++++++++++------------- src/InputMgr.h | 2 +- src/InputReader.cc | 14 +- src/InputReader.h | 3 +- src/InputReaderAscii.cc | 265 ++++++++++++------------- src/InputReaderAscii.h | 30 ++- 7 files changed, 283 insertions(+), 302 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index d9c0812498..9d83d73ec6 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -14,9 +14,9 @@ export { name: string; ## for tables - idx: any &optional; - val: any &optional; - destination: any &optional; + idx: any; + val: any; + destination: any; want_record: bool &default=T; table_ev: any &optional; # event containing idx, val as values. @@ -25,13 +25,13 @@ export { pred: function(typ: Input::Event, left: any, right: any): bool &optional; ## for "normalized" events - ev: any &optional; - ev_description: any &optional; + # ev: any &optional; + # ev_description: any &optional; }; - const no_filter: Filter = [$name=""]; # Sentinel. + const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::ReaderDescription) : bool; + global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; @@ -47,7 +47,7 @@ module Input; global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::ReaderDescription) : bool +function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index f9250f6f0f..403d656140 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -16,18 +16,20 @@ #include "CompHash.h" -class InputHash { -public: +struct InputHash { HashKey* valhash; HashKey* idxkey; // does not need ref or whatever - if it is present here, it is also still present in the TableVal. }; declare(PDict, InputHash); -struct InputMgr::Filter { +class InputMgr::Filter { +public: EnumVal* id; string name; + //int filter_type; // to distinguish between event and table filters + unsigned int num_idx_fields; unsigned int num_val_fields; bool want_record; @@ -68,6 +70,8 @@ struct InputMgr::ReaderInfo { //list events; // events we fire when "something" happens map filters; // filters that can prevent our actions + bool HasFilter(int id); + ~ReaderInfo(); }; @@ -80,6 +84,15 @@ InputMgr::ReaderInfo::~ReaderInfo() { delete(reader); } +bool InputMgr::ReaderInfo::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } + return true; +} + + struct InputReaderDefinition { bro_int_t type; // the type const char *name; // descriptive name for error messages @@ -168,12 +181,12 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) int success = reader_obj->Init(source); if ( success == false ) { - assert( RemoveReader(id) ); + assert( RemoveStream(id) ); return 0; } success = reader_obj->Update(); if ( success == false ) { - assert ( RemoveReader(id) ); + assert ( RemoveStream(id) ); return 0; } @@ -224,6 +237,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } + // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); @@ -231,8 +245,8 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = rtype ? val->Ref()->AsRecordType() : 0; - filter.itype = itype ? idx->Ref()->AsRecordType() : 0; + filter.rtype = val ? val->Ref()->AsRecordType() : 0; + filter.itype = idx ? idx->Ref()->AsRecordType() : 0; // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); @@ -240,22 +254,11 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(info->want_record); + assert(filter.want_record); } i->filters[id->InternalInt()] = filter; - - // ok, now we have to alert the reader of our new filter with our funky new fields - // the id is handled in a ... well, to be honest, a little bit sneaky manner. - // the "problem" is, that we can have several filters in the reader for one filter in the log manager. - // this is due to the fact, that a filter can either output it's result as a table, as an event... - // ...or as an table _and_ an event. And... if we have a table and an event, we actually need two different sets - // of filters in the reader, because the fields for the table and the event may differ and I absolutely do not want - // to build a union of these values and figure it out later. - // hence -> filter id is multiplicated with 2. - // filterId*2 -> results for table - // filterId*2+1 -> results for event - i->AddFilter( id->InternalInt() * 2, fieldsV.size(), idxfields, fields ); + i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); return true; } @@ -387,31 +390,15 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } -/* - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) - { - if ( (*it).name == name ) { - it = i->filters.erase(it); - return true; - break; - } - else - ++it; - } - */ - map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - it->filters.erase(it); + i->filters.erase(it); return true; } - - Val* InputMgr::LogValToIndexVal(int num_fields, const RecordType *type, const LogVal* const *vals) { Val* idxval; int position = 0; @@ -449,27 +436,28 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const bool updated = false; + assert(i->HasFilter(id)); //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->num_idx_fields, vals); + HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->num_val_fields, vals+i->num_idx_fields); + HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->lastDict->Lookup(idxhash); + InputHash *h = i->filters[id].lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->lastDict->Remove(idxhash); - i->currDict->Insert(idxhash, h); + i->filters[id].lastDict->Remove(idxhash); + i->filters[id].currDict->Insert(idxhash, h); return; } else { // updated - i->lastDict->Remove(idxhash); + i->filters[id].lastDict->Remove(idxhash); delete(h); updated = true; @@ -477,27 +465,22 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - /* if ( i->rtype->NumFields() != (int) i->num_val_fields ) { - reporter->InternalError("Type mismatch"); - return; - } */ - - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -516,17 +499,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->tab->Lookup(idxval); + oldval = i->filters[id].tab->Lookup(idxval); } - // call filters first do determine if we really add / change the entry - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } - + // call filter first to determine if we really add / change the entry + if ( i->filters[id].pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -541,44 +519,45 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = (*it).pred->Call(&vl); + Val* v = i->filters[id].pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->currDict->RemoveEntry(idxhash)); + delete(i->filters[id].currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->currDict->Insert(idxhash, h); + i->filters[id].currDict->Insert(idxhash, h); return; } } - ++it; } //i->tab->Assign(idxval, valval); - HashKey* k = i->tab->ComputeHash(idxval); + HashKey* k = i->filters[id].tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - i->tab->Assign(idxval, k, valval); + i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->tab->ComputeHash(idxval); + k = i->filters[id].tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->currDict->Insert(idxhash, ih); + i->filters[id].currDict->Insert(idxhash, ih); // send events now that we are kind of finished. + + /* FIXME: fix me. std::list::iterator filter_iterator = i->events.begin(); while ( filter_iterator != i->events.end() ) { EnumVal* ev; @@ -597,7 +576,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ++filter_iterator; - } + } */ } @@ -607,86 +586,74 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } + + assert(i->HasFilter(id)); + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->lastDict->InitForIteration(); - i->lastDict->MakeRobustCookie(c); + IterCookie *c = i->filters[id].lastDict->InitForIteration(); + i->filters[id].lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - - if ( i->events.size() != 0 || i->filters.size() != 0 ) // we have a filter or an event - { + while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal *idx = i->tab->RecoverIndex(ih->idxkey); + if ( i->filters[id].pred ) { + ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); assert(idx != 0); - Val *val = i->tab->Lookup(idx); + Val *val = i->filters[id].tab->Lookup(idx); assert(val != 0); - { - bool doBreak = false; - // ask filter, if we want to expire this element... - std::list::iterator it = i->filters.begin(); - while ( it != i->filters.end() ) { - if (! (*it).pred ) { - continue; - } + bool doBreak = false; + // ask predicate, if we want to expire this element... - EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); - Ref(val); + EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + Ref(idx); + Ref(val); - val_list vl(3); - vl.append(ev); - vl.append(idx); - vl.append(val); - Val* v = (*it).pred->Call(&vl); - bool result = v->AsBool(); - Unref(v); - - ++it; - - if ( result == false ) { - // Keep it. Hence - we quit and simply go to the next entry of lastDict - // ah well - and we have to add the entry to currDict... - i->currDict->Insert(lastDictIdxKey, i->lastDict->RemoveEntry(lastDictIdxKey)); - doBreak = true; - continue; - } - - } - - if ( doBreak ) { - continue; - } + val_list vl(3); + vl.append(ev); + vl.append(idx); + vl.append(val); + Val* v = i->filters[id].pred->Call(&vl); + bool result = v->AsBool(); + Unref(v); + + if ( result == false ) { + // Keep it. Hence - we quit and simply go to the next entry of lastDict + // ah well - and we have to add the entry to currDict... + i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + continue; } - + + // { - std::list::iterator it = i->events.begin(); - while ( it != i->events.end() ) { + /* FIXME: events + std::list::iterator it = i->filters[id].events.begin(); + while ( it != i->filters[id].events.end() ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); SendEvent(*it, ev, idx, val); ++it; } + */ } } - i->tab->Delete(ih->idxkey); - i->lastDict->Remove(lastDictIdxKey); // deletex in next line + i->filters[id].tab->Delete(ih->idxkey); + i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->lastDict); + i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... + delete(i->filters[id].lastDict); - i->lastDict = i->currDict; - i->currDict = new PDict(InputHash); + i->filters[id].lastDict = i->filters[id].currDict; + i->filters[id].currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -696,22 +663,24 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); Val* valval; - int position = i->num_idx_fields; - if ( i->num_val_fields == 1 && !i->want_record ) { - valval = LogValToVal(vals[i->num_idx_fields], i->rtype->FieldType(i->num_idx_fields)); + int position = i->filters[id].num_idx_fields; + if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { + valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->rtype); + RecordVal * r = new RecordVal(i->filters[id].rtype); - for ( int j = 0; j < i->rtype->NumFields(); j++) { + for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { Val* val = 0; - if ( i->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->rtype->FieldType(j)->AsRecordType(), &position); + if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->rtype->FieldType(j)); + val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); position++; } @@ -726,7 +695,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->tab->Assign(idxval, valval); + i->filters[id].tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -735,20 +704,24 @@ void InputMgr::Clear(const InputReader* reader, int id) { reporter->InternalError("Unknown reader"); return; } - - i->tab->RemoveAll(); + + assert(i->HasFilter(id)); + + i->filters[id].tab->RemoveAll(); } -bool InputMgr::Delete(const InputReader* reader, const LogVal* const *vals) { +bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { ReaderInfo *i = FindReader(reader); if ( i == 0 ) { reporter->InternalError("Unknown reader"); return false; } - - Val* idxval = LogValToIndexVal(i->num_idx_fields, i->itype, vals); - return ( i->tab->Delete(idxval) != 0 ); + assert(i->HasFilter(id)); + + Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + + return ( i->filters[id].tab->Delete(idxval) != 0 ); } void InputMgr::Error(InputReader* reader, const char* msg) diff --git a/src/InputMgr.h b/src/InputMgr.h index 1cacf89143..5d531cd6fc 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -35,7 +35,7 @@ protected: void Error(InputReader* reader, const char* msg); // for readers to write to input stream in direct mode (reporting new/deleted values directly) - void Put(const InputReader* reader, int id. const LogVal* const *vals); + void Put(const InputReader* reader, int id, const LogVal* const *vals); void Clear(const InputReader* reader, int id); bool Delete(const InputReader* reader, int id, const LogVal* const *vals); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1008cf1b67..1c65985fd6 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -26,17 +26,17 @@ void InputReader::Error(const string &msg) void InputReader::Put(int id, const LogVal* const *val) { - input_mgr->Put(this, int id, val); + input_mgr->Put(this, id, val); } void InputReader::Clear(int id) { - input_mgr->Clear(this, int id); + input_mgr->Clear(this, id); } void InputReader::Delete(int id, const LogVal* const *val) { - input_mgr->Delete(this, int id, val); + input_mgr->Delete(this, id, val); } @@ -52,12 +52,12 @@ bool InputReader::Init(string arg_source) bool InputReader::AddFilter(int id, int arg_num_fields, const LogField* const * arg_fields) { - return DoAddFilter(int id, arg_num_fields, arg_fields); + return DoAddFilter(id, arg_num_fields, arg_fields); } bool InputReader::RemoveFilter(int id) { - return DoRemoveFilter(int id); + return DoRemoveFilter(id); } void InputReader::Finish() @@ -105,10 +105,10 @@ const char* InputReader::Fmt(const char* format, ...) void InputReader::SendEntry(int id, const LogVal* const *vals) { - input_mgr->SendEntry(this, int id, vals); + input_mgr->SendEntry(this, id, vals); } void InputReader::EndCurrentSend(int id) { - input_mgr->EndCurrentSend(this, int id); + input_mgr->EndCurrentSend(this, id); } diff --git a/src/InputReader.h b/src/InputReader.h index 12f0bc9db4..6e3d689750 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -30,7 +30,8 @@ protected: virtual bool DoInit(string arg_sources) = 0; virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) = 0; - virtual bool DoRemoveFilter( int id ); + + virtual bool DoRemoveFilter( int id ) = 0; virtual void DoFinish() = 0; diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 4a0d4157bc..84feb74e61 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -28,6 +28,7 @@ FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } + InputReaderAscii::InputReaderAscii() { file = 0; @@ -58,7 +59,7 @@ InputReaderAscii::~InputReaderAscii() void InputReaderAscii::DoFinish() { - columnMap.empty(); + filters.empty(); if ( file != 0 ) { file->close(); delete(file); @@ -66,7 +67,7 @@ void InputReaderAscii::DoFinish() } } -bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const LogField* const * fields) +bool InputReaderAscii::DoInit(string path) { fname = path; @@ -76,11 +77,39 @@ bool InputReaderAscii::DoInit(string path, int num_fields, int idx_fields, const return false; } + return true; +} - this->num_fields = num_fields; - this->idx_fields = idx_fields; - this->fields = fields; +bool InputReaderAscii::DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ) { + if ( HasFilter(id) ) { + return false; // no, we don't want to add this a second time + } + Filter f; + f.num_fields = arg_num_fields; + f.fields = fields; + + filters[id] = f; + + return true; +} + +bool InputReaderAscii::DoRemoveFilter ( int id ) { + if (!HasFilter(id) ) { + return false; + } + + assert ( filters.erase(id) == 1 ); + + return true; +} + + +bool InputReaderAscii::HasFilter(int id) { + map::iterator it = filters.find(id); + if ( it == filters.end() ) { + return false; + } return true; } @@ -93,46 +122,47 @@ bool InputReaderAscii::ReadHeader() { return false; } - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < num_fields; i++ ) { - const LogField* field = fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + // split on tabs... + istringstream splitstream(line); + unsigned int currTab = 0; + int wantFields = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + // current found heading in s... compare if we want it + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + if ( field->name == s ) { + // cool, found field. note position + FieldMapping f(field->name, field->type, field->subtype, i); + (*it).second.columnMap.push_back(f); + wantFields++; + break; // done with searching + } } + + // look if we did push something... + if ( (*it).second.columnMap.size() == currTab ) { + // no, we didn't. note that... + FieldMapping empty; + (*it).second.columnMap.push_back(empty); + } + + // done + currTab++; + } + + if ( wantFields != (int) (*it).second.num_fields ) { + // we did not find all fields? + // :( + Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); + return false; } - - // look if we did push something... - if ( columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d", wantFields, num_fields)); - return false; } - // well, that seems to have worked... return true; } @@ -314,110 +344,77 @@ bool InputReaderAscii::DoUpdate() { return false; } - // TODO: all the stuff we need for a second reading. - // *cough* - // - // - - - // new keymap - //map *newKeyMap = new map(); - string line; while ( GetLine(line ) ) { - // split on tabs + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - istringstream splitstream(line); - - LogVal** fields = new LogVal*[num_fields]; - //string string_fields[num_fields]; - - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { - - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - + // split on tabs - if ( currTab >= columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; - return false; - } - - FieldMapping currMapping = columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); - if ( val == 0 ) { - return false; - } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; - - currField++; - } - - if ( currField != num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; - } - - - SendEntry(fields); - - /* - string indexstring = ""; - string valstring = ""; - for ( unsigned int i = 0; i < idx_fields; i++ ) { - indexstring.append(string_fields[i]); - } - - for ( unsigned int i = idx_fields; i < num_fields; i++ ) { - valstring.append(string_fields[i]); - } - - string valhash = Hash(valstring); - string indexhash = Hash(indexstring); - - if ( keyMap->find(indexhash) == keyMap->end() ) { - // new key - Put(fields); - } else if ( (*keyMap)[indexhash] != valhash ) { - // changed key - Put(fields); - keyMap->erase(indexhash); - } else { - // field not changed - keyMap->erase(indexhash); - } - - - (*newKeyMap)[indexhash] = valhash; - */ + istringstream splitstream(line); - for ( unsigned int i = 0; i < num_fields; i++ ) { - delete fields[i]; + LogVal** fields = new LogVal*[(*it).second.num_fields]; + //string string_fields[num_fields]; + + unsigned int currTab = 0; + unsigned int currField = 0; + while ( splitstream ) { + + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + + if ( currTab >= (*it).second.columnMap.size() ) { + Error("Tabs in heading do not match tabs in data?"); + //disabled = true; + return false; + } + + FieldMapping currMapping = (*it).second.columnMap[currTab]; + currTab++; + + if ( currMapping.IsEmpty() ) { + // well, that was easy + continue; + } + + if ( currField >= (*it).second.num_fields ) { + Error("internal error - fieldnum greater as possible"); + return false; + } + + LogVal* val = EntryToVal(s, currMapping); + if ( val == 0 ) { + return false; + } + fields[currMapping.position] = val; + //string_fields[currMapping.position] = s; + + currField++; + } + + if ( currField != (*it).second.num_fields ) { + Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); + return false; + } + + + SendEntry((*it).first, fields); + + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + delete fields[i]; + } + delete [] fields; } - delete [] fields; } //file->clear(); // remove end of file evil bits //file->seekg(0, ios::beg); // and seek to start. - EndCurrentSend(); + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { + EndCurrentSend((*it).first); + } return true; } diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c848c17110..01169a3cfc 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -34,13 +34,30 @@ public: protected: - virtual bool DoInit(string path, int arg_num_fields, int arg_idx_fields, - const LogField* const * fields); + virtual bool DoInit(string path); + + virtual bool DoAddFilter( int id, int arg_num_fields, const LogField* const* fields ); + + virtual bool DoRemoveFilter ( int id ); + virtual void DoFinish(); virtual bool DoUpdate(); private: + + struct Filter { + unsigned int num_fields; + + const LogField* const * fields; // raw mapping + + // map columns in the file to columns to send back to the manager + vector columnMap; + + }; + + bool HasFilter(int id); + bool ReadHeader(); LogVal* EntryToVal(string s, FieldMapping type); @@ -49,15 +66,8 @@ private: ifstream* file; string fname; - unsigned int num_fields; - unsigned int idx_fields; + map filters; - // map columns in the file to columns to send back to the manager - vector columnMap; - const LogField* const * fields; // raw mapping - - //map *keyMap; - // // Options set from the script-level. string separator; From 7eb4d9934168fa0d37e01e3dbdd6feb60a88c485 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 12:27:34 -0800 Subject: [PATCH 30/58] very basic functionality kind of works again --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 403d656140..b51cd6fd27 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -47,10 +47,50 @@ public: EventHandlerPtr event; RecordType* event_type; - ~Filter(); + // ~Filter(); + // Filter(); + // Filter(const InputMgr::Filter& filter); + + void DoCleanup(); }; -InputMgr::Filter::~Filter() { +/* +InputMgr::Filter::Filter() { + tab = 0; + itype = 0; + rtype = 0; + event_type = 0; +} + +InputMgr::Filter::Filter(const InputMgr::Filter& f) { + id = f.id; + id->Ref(); + + tab = f.tab; + if ( tab ) + tab->Ref(); + + itype = f.itype; + if ( itype ) + itype->Ref(); + + rtype = f.rtype; + if ( rtype ) + Ref(rtype); + + event_type = f.event_type; + if ( event_type ) + Ref(event_type); + + name = f.name; + num_idx_fields = f.num_idx_fields; + num_val_fields = f.num_val_fields; + want_record = f.want_record; + + +} */ + +void InputMgr::Filter::DoCleanup() { Unref(id); if ( tab ) Unref(tab); @@ -60,7 +100,10 @@ InputMgr::Filter::~Filter() { Unref(rtype); if ( event_type) Unref(event_type); -} + + delete currDict; + delete lastDict; +} struct InputMgr::ReaderInfo { EnumVal* id; @@ -395,6 +438,8 @@ bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { return false; } + i->filters[id->InternalInt()].DoCleanup(); + i->filters.erase(it); return true; } @@ -545,6 +590,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + reporter->Error("assigning"); i->filters[id].tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); From 029871e48c4c63add07d63a52127bb7b50f47189 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 20 Nov 2011 13:42:02 -0800 Subject: [PATCH 31/58] first test. --- .../scripts.base.frameworks.input.basic/out | 14 +++++ .../scripts/base/frameworks/input/basic.bro | 52 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.basic/out create mode 100644 testing/btest/scripts/base/frameworks/input/basic.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.basic/out b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out new file mode 100644 index 0000000000..ebac1866b6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.basic/out @@ -0,0 +1,14 @@ +{ +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={ +2, +4, +1, +3 +}, ss={ +CC, +AA, +BB +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro new file mode 100644 index 0000000000..5e0c7be12e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -0,0 +1,52 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string table table table vector vector func +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From f0e5303330d9842a7454001276ef5857c252bba9 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:09:00 -0800 Subject: [PATCH 32/58] make want_record field for tablefilter work... --- src/InputMgr.cc | 2 +- .../out | 3 ++ .../out | 3 ++ .../frameworks/input/onecolumn-norecord.bro | 38 +++++++++++++++++++ .../frameworks/input/onecolumn-record.bro | 38 +++++++++++++++++++ 5 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro create mode 100644 testing/btest/scripts/base/frameworks/input/onecolumn-record.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index b51cd6fd27..c4180f4f8d 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -515,7 +515,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const int position = i->filters[id].num_idx_fields; if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(i->filters[id].rtype); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out new file mode 100644 index 0000000000..bbce48f4f6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-norecord/out @@ -0,0 +1,3 @@ +{ +[-42] = T +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out new file mode 100644 index 0000000000..3f9af35c59 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.onecolumn-record/out @@ -0,0 +1,3 @@ +{ +[-42] = [b=T] +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro new file mode 100644 index 0000000000..74fd477e28 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::force_update(A::LOG); + print servers; +} diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro new file mode 100644 index 0000000000..3cc7090462 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i +#types bool int +T -42 +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::force_update(A::LOG); + print servers; +} From 18591b53d422e5b97dc8595a5171e50a64feec78 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:20:52 -0800 Subject: [PATCH 33/58] rename filter to tablefilter in preparation of event filters... --- scripts/base/frameworks/input/main.bro | 38 +++++++++---------- src/InputMgr.cc | 6 +-- src/InputMgr.h | 4 +- src/input.bif | 10 ++--- testing/btest/btest.cfg | 2 +- .../scripts/base/frameworks/input/basic.bro | 2 +- .../frameworks/input/onecolumn-norecord.bro | 2 +- .../frameworks/input/onecolumn-record.bro | 2 +- 8 files changed, 33 insertions(+), 33 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 9d83d73ec6..4560421ecc 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -9,7 +9,7 @@ export { reader: Reader &default=default_reader; }; - type Filter: record { + type TableFilter: record { ## descriptive name. for later removal name: string; @@ -29,14 +29,14 @@ export { # ev_description: any &optional; }; - const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. + #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; global remove_stream: function(id: Log::ID) : bool; global force_update: function(id: Log::ID) : bool; - global add_filter: function(id: Log::ID, filter: Input::Filter) : bool; - global remove_filter: function(id: Log::ID, name: string) : bool; - global get_filter: function(id: ID, name: string) : Filter; + global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; + global remove_tablefilter: function(id: Log::ID, name: string) : bool; + #global get_filter: function(id: ID, name: string) : Filter; } @@ -45,7 +45,7 @@ export { module Input; -global filters: table[ID, string] of Filter; +#global filters: table[ID, string] of Filter; function create_stream(id: Log::ID, description: Input::StreamDescription) : bool { @@ -62,22 +62,22 @@ function force_update(id: Log::ID) : bool return __force_update(id); } -function add_filter(id: Log::ID, filter: Input::Filter) : bool +function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool { - filters[id, filter$name] = filter; - return __add_filter(id, filter); +# filters[id, filter$name] = filter; + return __add_tablefilter(id, filter); } -function remove_filter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Log::ID, name: string) : bool { - delete filters[id, name]; - return __remove_filter(id, name); +# delete filters[id, name]; + return __remove_tablefilter(id, name); } -function get_filter(id: ID, name: string) : Filter - { - if ( [id, name] in filters ) - return filters[id, name]; - - return no_filter; - } +#function get_filter(id: ID, name: string) : Filter +# { +# if ( [id, name] in filters ) +# return filters[id, name]; +# +# return no_filter; +# } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index c4180f4f8d..8fda5d506f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -237,7 +237,7 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } -bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { +bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Stream not found"); @@ -245,7 +245,7 @@ bool InputMgr::AddFilter(EnumVal *id, RecordVal* fval) { } RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, BifType::Record::Input::Filter, 0) ) + if ( ! same_type(rtype, BifType::Record::Input::TableFilter, 0) ) { reporter->Error("filter argument not of right type"); return false; @@ -426,7 +426,7 @@ bool InputMgr::ForceUpdate(const EnumVal* id) return i->reader->Update(); } -bool InputMgr::RemoveFilter(EnumVal* id, const string &name) { +bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { reporter->Error("Reader not found"); diff --git a/src/InputMgr.h b/src/InputMgr.h index 5d531cd6fc..4280ba1d81 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -24,8 +24,8 @@ public: bool ForceUpdate(const EnumVal* id); bool RemoveStream(const EnumVal* id); - bool AddFilter(EnumVal *id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, const string &name); + bool AddTableFilter(EnumVal *id, RecordVal* filter); + bool RemoveTableFilter(EnumVal* id, const string &name); protected: diff --git a/src/input.bif b/src/input.bif index ef069316ab..1300f91bea 100644 --- a/src/input.bif +++ b/src/input.bif @@ -8,7 +8,7 @@ module Input; %%} type StreamDescription: record; -type Filter: record; +type TableFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -28,15 +28,15 @@ function Input::__force_update%(id: Log::ID%) : bool return new Val( res, TYPE_BOOL ); %} -function Input::__add_filter%(id: Log::ID, filter: Input::Filter%) : bool +function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool %{ - bool res = input_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_filter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool %{ - bool res = input_mgr->RemoveFilter(id->AsEnumVal(), name->AsString()->CheckString()); + bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); %} diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7d8283587c..739d0b2ad4 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -3,7 +3,7 @@ TestDirs = doc bifs language core scripts istate coverage TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* *.trace +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 5e0c7be12e..139888fa7c 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -46,7 +46,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 74fd477e28..12dbdd42aa 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 3cc7090462..4eef12d752 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -32,7 +32,7 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_filter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); Input::force_update(A::LOG); print servers; } From 92b3723b0947739fb4aff9a3d7a87662f65011c7 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:36:03 -0800 Subject: [PATCH 34/58] add very basic predicate test. --- .../out | 7 ++ .../base/frameworks/input/predicate.bro | 66 +++++++++++++++++++ 2 files changed, 73 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicate/out create mode 100644 testing/btest/scripts/base/frameworks/input/predicate.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out new file mode 100644 index 0000000000..d805f804d8 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicate/out @@ -0,0 +1,7 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro new file mode 100644 index 0000000000..e82ded6fd0 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -0,0 +1,66 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type idx: record { + i: int; +}; + +type val: record { + b: bool; +}; + +global servers: table[int] of val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + ]); + Input::force_update(A::LOG); + if ( 1 in servers ) { + print "VALID"; + } + if ( 2 in servers ) { + print "VALID"; + } + if ( !(3 in servers) ) { + print "VALID"; + } + if ( !(4 in servers) ) { + print "VALID"; + } + if ( !(5 in servers) ) { + print "VALID"; + } + if ( !(6 in servers) ) { + print "VALID"; + } + if ( 7 in servers ) { + print "VALID"; + } +} From 77a517f2b5e90d94c9de144605c24247c620e4af Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 15:45:27 -0800 Subject: [PATCH 35/58] camel-casing for types --- testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 139888fa7c..3ad45eac69 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -18,11 +18,11 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; e: Log::ID; c: count; @@ -40,13 +40,13 @@ type val: record { ve: vector of int; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 12dbdd42aa..134ceb49e6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 4eef12d752..c07c9c826c 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -18,21 +18,21 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::LOG); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index e82ded6fd0..769536d2a6 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -24,22 +24,22 @@ export { redef enum Log::ID += { LOG }; } -type idx: record { +type Idx: record { i: int; }; -type val: record { +type Val: record { b: bool; }; -global servers: table[int] of val = table(); +global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=idx, $val=val, $destination=servers, $want_record=F, - $pred(typ: Input::Event, left: idx, right: bool) = { return right; } + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); Input::force_update(A::LOG); if ( 1 in servers ) { From 53af0544ccc9a60254228ab702ec39639b5b577d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:01:07 -0800 Subject: [PATCH 36/58] re-enable table events --- scripts/base/frameworks/input/main.bro | 2 +- src/InputMgr.cc | 120 +++++++++++------- src/InputMgr.h | 2 +- .../scripts.base.frameworks.input.event/out | 21 +++ .../scripts/base/frameworks/input/event.bro | 48 +++++++ 5 files changed, 147 insertions(+), 46 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.event/out create mode 100644 testing/btest/scripts/base/frameworks/input/event.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 4560421ecc..69b4d41ebb 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -18,7 +18,7 @@ export { val: any; destination: any; want_record: bool &default=T; - table_ev: any &optional; # event containing idx, val as values. + ev: any &optional; # event containing idx, val as values. ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 8fda5d506f..de9ef158b7 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -214,7 +214,6 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) const BroString* bsource = description->Lookup(rtype->FieldOffset("source"))->AsString(); string source((const char*) bsource->Bytes(), bsource->Len()); - ReaderInfo* info = new ReaderInfo; info->reader = reader_obj; info->type = reader->AsEnumVal(); // ref'd by lookupwithdefault @@ -259,6 +258,50 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + + if ( event ) { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 3 ) + { + reporter->Error("Table event must take 3 arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("table events first attribute must be of type Input::Event"); + return false; + } + + if ( ! same_type((*args)[1], idx) ) + { + reporter->Error("table events index attributes do not match"); + return false; + } + + if ( want_record->InternalInt() == 1 && ! same_type((*args)[2], val) ) + { + reporter->Error("table events value attributes do not match"); + return false; + } else if ( want_record->InternalInt() == 0 && !same_type((*args)[2], val->FieldType(0) ) ) { + reporter->Error("table events value attribute does not match"); + return false; + } + + } + vector fieldsV; // vector, because we don't know the length beforehands bool status = !UnrollRecordType(&fieldsV, idx, ""); @@ -273,24 +316,22 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { return false; } - Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); LogField** fields = new LogField*[fieldsV.size()]; for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { fields[i] = fieldsV[i]; } - // FIXME: remove those funky 0-tests again as the idea was changed. Filter filter; filter.name = name->AsString()->CheckString(); filter.id = id->Ref()->AsEnumVal(); filter.pred = pred ? pred->AsFunc() : 0; filter.num_idx_fields = idxfields; filter.num_val_fields = valfields; - filter.tab = dst ? dst->Ref()->AsTableVal() : 0; - filter.rtype = val ? val->Ref()->AsRecordType() : 0; - filter.itype = idx ? idx->Ref()->AsRecordType() : 0; - // ya - well - we actually don't need them in every case... well, a few bytes of memory wasted + filter.tab = dst->Ref()->AsTableVal(); + filter.rtype = val->Ref()->AsRecordType(); + filter.itype = idx->Ref()->AsRecordType(); + filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter.currDict = new PDict(InputHash); filter.lastDict = new PDict(InputHash); filter.want_record = ( want_record->InternalInt() == 1 ); @@ -601,11 +642,7 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const i->filters[id].currDict->Insert(idxhash, ih); - // send events now that we are kind of finished. - - /* FIXME: fix me. - std::list::iterator filter_iterator = i->events.begin(); - while ( filter_iterator != i->events.end() ) { + if ( i->filters[id].event ) { EnumVal* ev; Ref(idxval); @@ -613,16 +650,13 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(*filter_iterator, ev, idxval, oldval); + SendEvent(i->filters[id].event, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(*filter_iterator, ev, idxval, valval); + SendEvent(i->filters[id].event, ev, idxval, valval); } - - - ++filter_iterator; - } */ + } } @@ -643,12 +677,17 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { - if ( i->filters[id].pred ) { - ListVal *idx = i->filters[id].tab->RecoverIndex(ih->idxkey); - assert(idx != 0); - Val *val = i->filters[id].tab->Lookup(idx); - assert(val != 0); + ListVal * idx; + Val *val; + if ( i->filters[id].pred || i->filters[id].event ) { + idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + assert(idx != 0); + val = i->filters[id].tab->Lookup(idx); + assert(val != 0); + } + + if ( i->filters[id].pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -673,21 +712,13 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { } - // - - { - /* FIXME: events - std::list::iterator it = i->filters[id].events.begin(); - while ( it != i->filters[id].events.end() ) { - Ref(idx); - Ref(val); - EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(*it, ev, idx, val); - ++it; - } - */ - } + } + if ( i->filters[id].event ) { + Ref(idx); + Ref(val); + EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEvent(i->filters[id].event, ev, idx, val); } i->filters[id].tab->Delete(ih->idxkey); @@ -792,20 +823,21 @@ void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c mgr.Dispatch(new Event(handler, vl)); } */ -void InputMgr::SendEvent(const string& name, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { - EventHandler* handler = event_registry->Lookup(name.c_str()); - if ( handler == 0 ) { - reporter->Error("Event %s not found", name.c_str()); - return; - } + //EventHandler* handler = event_registry->Lookup(name.c_str()); + //if ( handler == 0 ) { + // reporter->Error("Event %s not found", name.c_str()); + // return; + //} val_list* vl = new val_list; vl->append(event); vl->append(left); vl->append(right); - mgr.Dispatch(new Event(handler, vl)); + //mgr.Dispatch(new Event(handler, vl)); + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } diff --git a/src/InputMgr.h b/src/InputMgr.h index 4280ba1d81..d04b7c9a2c 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -49,7 +49,7 @@ private: bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(const string& name, EnumVal* event, Val* left, Val* right); + void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.event/out b/testing/btest/Baseline/scripts.base.frameworks.input.event/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.event/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From 3035eb2b219946144fe67097781226d77f34176e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 21 Nov 2011 19:30:16 -0800 Subject: [PATCH 37/58] fix a little bug that prevented several simultaneous filters from working. --- src/InputMgr.cc | 8 +- .../out | 15 +++ .../base/frameworks/input/twofilters.bro | 95 +++++++++++++++++++ 3 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out create mode 100644 testing/btest/scripts/base/frameworks/input/twofilters.bro diff --git a/src/InputMgr.cc b/src/InputMgr.cc index de9ef158b7..2ec3c649be 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -341,8 +341,12 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { assert(filter.want_record); } - i->filters[id->InternalInt()] = filter; - i->reader->AddFilter( id->InternalInt(), fieldsV.size(), fields ); + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), fields ); return true; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out new file mode 100644 index 0000000000..5b1ee5e983 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.twofilters/out @@ -0,0 +1,15 @@ +VALID +VALID +VALID +VALID +VALID +VALID +VALID +MARK +VALID +VALID +VALID +VALID +VALID +VALID +VALID diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro new file mode 100644 index 0000000000..575665e6e5 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -0,0 +1,95 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination1: table[int] of Val = table(); +global destination2: table[int] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } + ]); + Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + + Input::force_update(A::LOG); + if ( 1 in destination1 ) { + print "VALID"; + } + if ( 2 in destination1 ) { + print "VALID"; + } + if ( !(3 in destination1) ) { + print "VALID"; + } + if ( !(4 in destination1) ) { + print "VALID"; + } + if ( !(5 in destination1) ) { + print "VALID"; + } + if ( !(6 in destination1) ) { + print "VALID"; + } + if ( 7 in destination1 ) { + print "VALID"; + } + + print "MARK"; + + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 2 in destination2 ) { + print "VALID"; + } + if ( 3 in destination2 ) { + print "VALID"; + } + if ( 4 in destination2 ) { + print "VALID"; + } + if ( 5 in destination2 ) { + print "VALID"; + } + if ( 6 in destination2 ) { + print "VALID"; + } + if ( 7 in destination2 ) { + print "VALID"; + } + + +} From f82bf3f35fb5f1e298366f9af6d9ec0372ac3d58 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:09:06 -0800 Subject: [PATCH 38/58] re-enable direct event sending from input readers --- src/InputMgr.cc | 18 +++++++++++++----- src/InputMgr.h | 3 ++- src/InputReader.cc | 7 +++---- src/InputReader.h | 2 +- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 2ec3c649be..e6d739d26f 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -810,22 +810,30 @@ void InputMgr::Error(InputReader* reader, const char* msg) reporter->Error("error with input reader for %s: %s", reader->Source().c_str(), msg); } -/* Does not work atm, because LogValToVal needs BroType -void InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { EventHandler* handler = event_registry->Lookup(name.c_str()); if ( handler == 0 ) { reporter->Error("Event %s not found", name.c_str()); - return; + return false; + } + + RecordType *type = handler->FType()->Args(); + int num_event_vals = type->NumFields(); + if ( num_vals != num_event_vals ) { + reporter->Error("Wrong number of values for event %s", name.c_str()); + return false; } val_list* vl = new val_list; for ( int i = 0; i < num_vals; i++) { - vl->append(LogValToVal(vals[i])); + vl->append(LogValToVal(vals[i], type->FieldType(i))); } mgr.Dispatch(new Event(handler, vl)); -} */ + + return true; +} void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) { diff --git a/src/InputMgr.h b/src/InputMgr.h index d04b7c9a2c..ba1ddafc92 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -59,7 +59,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReader.cc b/src/InputReader.cc index 1c65985fd6..3f296dc0aa 100644 --- a/src/InputReader.cc +++ b/src/InputReader.cc @@ -71,11 +71,10 @@ bool InputReader::Update() return DoUpdate(); } -/* -void InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) +bool InputReader::SendEvent(const string& name, const int num_vals, const LogVal* const *vals) { - input_mgr->SendEvent(name, num_vals, vals); -} */ + return input_mgr->SendEvent(name, num_vals, vals); +} // stolen from logwriter const char* InputReader::Fmt(const char* format, ...) diff --git a/src/InputReader.h b/src/InputReader.h index 6e3d689750..34d549308e 100644 --- a/src/InputReader.h +++ b/src/InputReader.h @@ -48,7 +48,7 @@ protected: // A thread-safe version of fmt(). (stolen from logwriter) const char* Fmt(const char* format, ...); - //void SendEvent(const string& name, const int num_vals, const LogVal* const *vals); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); // Content-sendinf-functions (simple mode). Including table-specific stuff that simply is not used if we have no table void Put(int id, const LogVal* const *val); From 3c40f00a539e5d3d2945ffd1df71e19e93c7133d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 22 Nov 2011 11:39:27 -0800 Subject: [PATCH 39/58] make filters pointers (for inheritance) --- scripts/base/frameworks/input/main.bro | 29 +- src/InputMgr.cc | 519 +++++++++++++----- src/InputMgr.h | 21 +- src/input.bif | 13 + .../out | 21 + .../scripts/base/frameworks/input/event.bro | 16 +- .../base/frameworks/input/tableevent.bro | 48 ++ 7 files changed, 507 insertions(+), 160 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out create mode 100644 testing/btest/scripts/base/frameworks/input/tableevent.bro diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 69b4d41ebb..a036eeeebd 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -23,10 +23,19 @@ export { ## decision function, that decides if an insertion, update or removal should really be executed. ## or events should be thought pred: function(typ: Input::Event, left: any, right: any): bool &optional; + }; - ## for "normalized" events - # ev: any &optional; - # ev_description: any &optional; + type EventFilter: record { + ## descriptive name. for later removal + name: string; + + # the event + ev: any; + # record describing the fields + fields: any; + + # does the event want the field unrolled (default) or as a simple record value? + want_record: bool &default=F; }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. @@ -36,6 +45,8 @@ export { global force_update: function(id: Log::ID) : bool; global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; global remove_tablefilter: function(id: Log::ID, name: string) : bool; + global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; + global remove_eventfilter: function(id: Log::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -74,6 +85,18 @@ function remove_tablefilter(id: Log::ID, name: string) : bool return __remove_tablefilter(id, name); } +function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool + { +# filters[id, filter$name] = filter; + return __add_eventfilter(id, filter); + } + +function remove_eventfilter(id: Log::ID, name: string) : bool + { +# delete filters[id, name]; + return __remove_eventfilter(id, name); + } + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index e6d739d26f..0df11ea359 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -28,7 +28,13 @@ public: EnumVal* id; string name; - //int filter_type; // to distinguish between event and table filters + FilterType filter_type; // to distinguish between event and table filters + + virtual ~Filter(); +}; + +class InputMgr::TableFilter: public InputMgr::Filter { +public: unsigned int num_idx_fields; unsigned int num_val_fields; @@ -45,61 +51,42 @@ public: Func* pred; EventHandlerPtr event; - RecordType* event_type; - // ~Filter(); - // Filter(); - // Filter(const InputMgr::Filter& filter); - - void DoCleanup(); + TableFilter(); + ~TableFilter(); }; -/* -InputMgr::Filter::Filter() { +class InputMgr::EventFilter: public InputMgr::Filter { +public: + EventHandlerPtr event; + + RecordType* fields; + unsigned int num_fields; + + bool want_record; + EventFilter(); +}; + +InputMgr::TableFilter::TableFilter() { + filter_type = TABLE_FILTER; + tab = 0; itype = 0; rtype = 0; - event_type = 0; } -InputMgr::Filter::Filter(const InputMgr::Filter& f) { - id = f.id; - id->Ref(); +InputMgr::EventFilter::EventFilter() { + filter_type = EVENT_FILTER; +} - tab = f.tab; - if ( tab ) - tab->Ref(); - - itype = f.itype; - if ( itype ) - itype->Ref(); - - rtype = f.rtype; - if ( rtype ) - Ref(rtype); - - event_type = f.event_type; - if ( event_type ) - Ref(event_type); - - name = f.name; - num_idx_fields = f.num_idx_fields; - num_val_fields = f.num_val_fields; - want_record = f.want_record; - - -} */ - -void InputMgr::Filter::DoCleanup() { +InputMgr::Filter::~Filter() { Unref(id); - if ( tab ) - Unref(tab); - if ( itype ) - Unref(itype); - if ( rtype ) - Unref(rtype); - if ( event_type) - Unref(event_type); +} + +InputMgr::TableFilter::~TableFilter() { + Unref(tab); + Unref(itype); + Unref(rtype); delete currDict; delete lastDict; @@ -111,7 +98,7 @@ struct InputMgr::ReaderInfo { InputReader* reader; //list events; // events we fire when "something" happens - map filters; // filters that can prevent our actions + map filters; // filters that can prevent our actions bool HasFilter(int id); @@ -119,7 +106,11 @@ struct InputMgr::ReaderInfo { }; InputMgr::ReaderInfo::~ReaderInfo() { - // all the contents of filters should delete themselves automatically... + map::iterator it = filters.begin(); + + while ( it != filters.end() ) { + delete (*it).second; + } Unref(type); Unref(id); @@ -128,7 +119,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { } bool InputMgr::ReaderInfo::HasFilter(int id) { - map::iterator it = filters.find(id); + map::iterator it = filters.find(id); if ( it == filters.end() ) { return false; } @@ -236,6 +227,114 @@ InputReader* InputMgr::CreateStream(EnumVal* id, RecordVal* description) } +bool InputMgr::AddEventFilter(EnumVal *id, RecordVal* fval) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Stream not found"); + return false; + } + + RecordType* rtype = fval->Type()->AsRecordType(); + if ( ! same_type(rtype, BifType::Record::Input::EventFilter, 0) ) + { + reporter->Error("filter argument not of right type"); + return false; + } + + Val* name = fval->Lookup(rtype->FieldOffset("name")); + RecordType *fields = fval->Lookup(rtype->FieldOffset("fields"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); + + Val* event_val = fval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val->AsFunc(); + + { + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) { + reporter->Error("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() < 2 ) { + reporter->Error("event takes not enough arguments"); + return false; + } + + if ( ! same_type((*args)[0], BifType::Enum::Input::Event, 0) ) + { + reporter->Error("events first attribute must be of type Input::Event"); + return false; + } + + if ( want_record->InternalInt() == 0 ) { + if ( args->length() != fields->NumFields() + 1 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + for ( int i = 0; i < fields->NumFields(); i++ ) { + if ( !same_type((*args)[i+1], fields->FieldType(i) ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + } + + } else if ( want_record->InternalInt() == 1 ) { + if ( args->length() != 2 ) { + reporter->Error("events has wrong number of arguments"); + return false; + } + + if ( !same_type((*args)[1], fields ) ) { + reporter->Error("Incompatible type for event"); + return false; + } + + } else { + assert(false); + } + + } + + + vector fieldsV; // vector, because UnrollRecordType needs it + + bool status = !UnrollRecordType(&fieldsV, fields, ""); + + if ( status ) { + reporter->Error("Problem unrolling"); + return false; + } + + + LogField** logf = new LogField*[fieldsV.size()]; + for ( unsigned int i = 0; i < fieldsV.size(); i++ ) { + logf[i] = fieldsV[i]; + } + + EventFilter* filter = new EventFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->num_fields = fieldsV.size(); + filter->fields = fields->Ref()->AsRecordType(); + filter->event = event_registry->Lookup(event->GetID()->Name()); + filter->want_record = ( want_record->InternalInt() == 1 ); + Unref(want_record); // ref'd by lookupwithdefault + + int filterid = 0; + if ( i->filters.size() > 0 ) { + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> + } + i->filters[filterid] = filter; + i->reader->AddFilter( filterid, fieldsV.size(), logf ); + + return true; +} + bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { ReaderInfo *i = FindReader(id); if ( i == 0 ) { @@ -299,6 +398,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { reporter->Error("table events value attribute does not match"); return false; } + assert(want_record->InternalInt() == 1 || want_record->InternalInt() == 0); } @@ -322,28 +422,28 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { fields[i] = fieldsV[i]; } - Filter filter; - filter.name = name->AsString()->CheckString(); - filter.id = id->Ref()->AsEnumVal(); - filter.pred = pred ? pred->AsFunc() : 0; - filter.num_idx_fields = idxfields; - filter.num_val_fields = valfields; - filter.tab = dst->Ref()->AsTableVal(); - filter.rtype = val->Ref()->AsRecordType(); - filter.itype = idx->Ref()->AsRecordType(); - filter.event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; - filter.currDict = new PDict(InputHash); - filter.lastDict = new PDict(InputHash); - filter.want_record = ( want_record->InternalInt() == 1 ); + TableFilter* filter = new TableFilter(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->num_idx_fields = idxfields; + filter->num_val_fields = valfields; + filter->tab = dst->Ref()->AsTableVal(); + filter->rtype = val->Ref()->AsRecordType(); + filter->itype = idx->Ref()->AsRecordType(); + filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + filter->currDict = new PDict(InputHash); + filter->lastDict = new PDict(InputHash); + filter->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault if ( valfields > 1 ) { - assert(filter.want_record); + assert(filter->want_record); } int filterid = 0; if ( i->filters.size() > 0 ) { - filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map. new id = old id + 1. + filterid = i->filters.rbegin()->first + 1; // largest element is at beginning of map-> new id = old id + 1-> } i->filters[filterid] = filter; i->reader->AddFilter( filterid, fieldsV.size(), fields ); @@ -478,13 +578,39 @@ bool InputMgr::RemoveTableFilter(EnumVal* id, const string &name) { return false; } - map::iterator it = i->filters.find(id->InternalInt()); + map::iterator it = i->filters.find(id->InternalInt()); if ( it == i->filters.end() ) { return false; } - i->filters[id->InternalInt()].DoCleanup(); + if ( i->filters[id->InternalInt()]->filter_type != TABLE_FILTER ) { + // wrong type; + return false; + } + delete (*it).second; + i->filters.erase(it); + return true; +} + +bool InputMgr::RemoveEventFilter(EnumVal* id, const string &name) { + ReaderInfo *i = FindReader(id); + if ( i == 0 ) { + reporter->Error("Reader not found"); + return false; + } + + map::iterator it = i->filters.find(id->InternalInt()); + if ( it == i->filters.end() ) { + return false; + } + + if ( i->filters[id->InternalInt()]->filter_type != EVENT_FILTER ) { + // wrong type; + return false; + } + + delete (*it).second; i->filters.erase(it); return true; } @@ -524,30 +650,53 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + SendEntryTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + bool updated = false; + assert(i); assert(i->HasFilter(id)); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + //reporter->Error("Hashing %d index fields", i->num_idx_fields); - HashKey* idxhash = HashLogVals(i->filters[id].num_idx_fields, vals); + HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(i->filters[id].num_val_fields, vals+i->filters[id].num_idx_fields); + HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); - InputHash *h = i->filters[id].lastDict->Lookup(idxhash); + InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before if ( h->valhash->Hash() == valhash->Hash() ) { // ok, double. - i->filters[id].lastDict->Remove(idxhash); - i->filters[id].currDict->Insert(idxhash, h); + filter->lastDict->Remove(idxhash); + filter->currDict->Insert(idxhash, h); return; } else { // updated - i->filters[id].lastDict->Remove(idxhash); + filter->lastDict->Remove(idxhash); delete(h); updated = true; @@ -555,30 +704,25 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const } - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[position], i->filters[id].rtype->FieldType(0)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } - /* if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } */ - r->Assign(j,val); } @@ -589,12 +733,12 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const Val* oldval = 0; if ( updated == true ) { // in that case, we need the old value to send the event (if we send an event). - oldval = i->filters[id].tab->Lookup(idxval); + oldval = filter->tab->Lookup(idxval); } // call filter first to determine if we really add / change the entry - if ( i->filters[id].pred ) { + if ( filter->pred ) { EnumVal* ev; Ref(idxval); Ref(valval); @@ -609,18 +753,18 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const vl.append(ev); vl.append(idxval); vl.append(valval); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { if ( !updated ) { // throw away. Hence - we quit. And remove the entry from the current dictionary... - delete(i->filters[id].currDict->RemoveEntry(idxhash)); + delete(filter->currDict->RemoveEntry(idxhash)); return; } else { // keep old one - i->filters[id].currDict->Insert(idxhash, h); + filter->currDict->Insert(idxhash, h); return; } } @@ -629,24 +773,23 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const //i->tab->Assign(idxval, valval); - HashKey* k = i->filters[id].tab->ComputeHash(idxval); + HashKey* k = filter->tab->ComputeHash(idxval); if ( !k ) { reporter->InternalError("could not hash"); return; } - reporter->Error("assigning"); - i->filters[id].tab->Assign(idxval, k, valval); + filter->tab->Assign(idxval, k, valval); InputHash* ih = new InputHash(); - k = i->filters[id].tab->ComputeHash(idxval); + k = filter->tab->ComputeHash(idxval); ih->idxkey = k; ih->valhash = valhash; //i->tab->Delete(k); - i->filters[id].currDict->Insert(idxhash, ih); + filter->currDict->Insert(idxhash, ih); - if ( i->filters[id].event ) { + if ( filter->event ) { EnumVal* ev; Ref(idxval); @@ -654,11 +797,11 @@ void InputMgr::SendEntry(const InputReader* reader, int id, const LogVal* const ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); - SendEvent(i->filters[id].event, ev, idxval, oldval); + SendEvent(filter->event, 3, ev, idxval, oldval); } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(i->filters[id].event, ev, idxval, valval); + SendEvent(filter->event, 3, ev, idxval, valval); } } } @@ -673,25 +816,33 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { assert(i->HasFilter(id)); + if ( i->filters[id]->filter_type == EVENT_FILTER ) { + // nothing to do.. + return; + } + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + // lastdict contains all deleted entries and should be empty apart from that - IterCookie *c = i->filters[id].lastDict->InitForIteration(); - i->filters[id].lastDict->MakeRobustCookie(c); + IterCookie *c = filter->lastDict->InitForIteration(); + filter->lastDict->MakeRobustCookie(c); InputHash* ih; HashKey *lastDictIdxKey; //while ( ( ih = i->lastDict->NextEntry(c) ) ) { - while ( ( ih = i->filters[id].lastDict->NextEntry(lastDictIdxKey, c) ) ) { + while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { ListVal * idx; Val *val; - if ( i->filters[id].pred || i->filters[id].event ) { - idx = i->filters[id].tab->RecoverIndex(ih->idxkey); + if ( filter->pred || filter->event ) { + idx = filter->tab->RecoverIndex(ih->idxkey); assert(idx != 0); - val = i->filters[id].tab->Lookup(idx); + val = filter->tab->Lookup(idx); assert(val != 0); } - if ( i->filters[id].pred ) { + if ( filter->pred ) { bool doBreak = false; // ask predicate, if we want to expire this element... @@ -704,37 +855,37 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { vl.append(ev); vl.append(idx); vl.append(val); - Val* v = i->filters[id].pred->Call(&vl); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); if ( result == false ) { // Keep it. Hence - we quit and simply go to the next entry of lastDict // ah well - and we have to add the entry to currDict... - i->filters[id].currDict->Insert(lastDictIdxKey, i->filters[id].lastDict->RemoveEntry(lastDictIdxKey)); + filter->currDict->Insert(lastDictIdxKey, filter->lastDict->RemoveEntry(lastDictIdxKey)); continue; } } - if ( i->filters[id].event ) { + if ( filter->event ) { Ref(idx); Ref(val); EnumVal *ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - SendEvent(i->filters[id].event, ev, idx, val); + SendEvent(filter->event, 3, ev, idx, val); } - i->filters[id].tab->Delete(ih->idxkey); - i->filters[id].lastDict->Remove(lastDictIdxKey); // deletex in next line + filter->tab->Delete(ih->idxkey); + filter->lastDict->Remove(lastDictIdxKey); // deletex in next line delete(ih); } - i->filters[id].lastDict->Clear(); // should be empty... but... well... who knows... - delete(i->filters[id].lastDict); + filter->lastDict->Clear(); // should be empty->->-> but->->-> well->->-> who knows->->-> + delete(filter->lastDict); - i->filters[id].lastDict = i->filters[id].currDict; - i->filters[id].currDict = new PDict(InputHash); + filter->lastDict = filter->currDict; + filter->currDict = new PDict(InputHash); } void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) { @@ -744,24 +895,86 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) return; } + if ( !i->HasFilter(id) ) { + reporter->InternalError("Unknown filter"); + return; + } + + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + PutTable(reader, id, vals); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + } else { + assert(false); + } + +} + +void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + bool updated = false; + + assert(i); assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); + assert(i->filters[id]->filter_type == EVENT_FILTER); + EventFilter* filter = (EventFilter*) i->filters[id]; + + Val *val; + list out_vals; + // no tracking, send everything with a new event... + //out_vals.push_back(new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event)); + out_vals.push_back(type); + + int position = 0; + if ( filter->want_record ) { + RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); + out_vals.push_back(r); + + } else { + for ( int j = 0; j < filter->fields->NumFields(); j++) { + Val* val = 0; + if ( filter->fields->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->fields->FieldType(j)->AsRecordType(), &position); + } else { + val = LogValToVal(vals[position], filter->fields->FieldType(j)); + position++; + } + out_vals.push_back(val); + } + } + + SendEvent(filter->event, out_vals); + +} + +void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const *vals) { + ReaderInfo *i = FindReader(reader); + + assert(i); + assert(i->HasFilter(id)); + + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); Val* valval; - int position = i->filters[id].num_idx_fields; - if ( i->filters[id].num_val_fields == 1 && !i->filters[id].want_record ) { - valval = LogValToVal(vals[i->filters[id].num_idx_fields], i->filters[id].rtype->FieldType(i->filters[id].num_idx_fields)); + int position = filter->num_idx_fields; + if ( filter->num_val_fields == 1 && !filter->want_record ) { + valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(i->filters[id].rtype); + RecordVal * r = new RecordVal(filter->rtype); - for ( int j = 0; j < i->filters[id].rtype->NumFields(); j++) { + for ( int j = 0; j < filter->rtype->NumFields(); j++) { Val* val = 0; - if ( i->filters[id].rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, i->filters[id].rtype->FieldType(j)->AsRecordType(), &position); + if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { + val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); } else { - val = LogValToVal(vals[position], i->filters[id].rtype->FieldType(j)); + val = LogValToVal(vals[position], filter->rtype->FieldType(j)); position++; } @@ -776,7 +989,7 @@ void InputMgr::Put(const InputReader* reader, int id, const LogVal* const *vals) valval = r; } - i->filters[id].tab->Assign(idxval, valval); + filter->tab->Assign(idxval, valval); } void InputMgr::Clear(const InputReader* reader, int id) { @@ -788,7 +1001,10 @@ void InputMgr::Clear(const InputReader* reader, int id) { assert(i->HasFilter(id)); - i->filters[id].tab->RemoveAll(); + assert(i->filters[id]->filter_type == TABLE_FILTER); + TableFilter* filter = (TableFilter*) i->filters[id]; + + filter->tab->RemoveAll(); } bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *vals) { @@ -800,9 +1016,18 @@ bool InputMgr::Delete(const InputReader* reader, int id, const LogVal* const *va assert(i->HasFilter(id)); - Val* idxval = LogValToIndexVal(i->filters[id].num_idx_fields, i->filters[id].itype, vals); - - return ( i->filters[id].tab->Delete(idxval) != 0 ); + if ( i->filters[id]->filter_type == TABLE_FILTER ) { + TableFilter* filter = (TableFilter*) i->filters[id]; + Val* idxval = LogValToIndexVal(filter->num_idx_fields, filter->itype, vals); + return( filter->tab->Delete(idxval) != 0 ); + } else if ( i->filters[id]->filter_type == EVENT_FILTER ) { + EnumVal *type = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); + SendEventFilterEvent(reader, type, id, vals); + return true; + } else { + assert(false); + return false; + } } void InputMgr::Error(InputReader* reader, const char* msg) @@ -835,25 +1060,35 @@ bool InputMgr::SendEvent(const string& name, const int num_vals, const LogVal* c return true; } -void InputMgr::SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right) +void InputMgr::SendEvent(EventHandlerPtr ev, const int numvals, ...) { - //EventHandler* handler = event_registry->Lookup(name.c_str()); - //if ( handler == 0 ) { - // reporter->Error("Event %s not found", name.c_str()); - // return; - //} - val_list* vl = new val_list; - vl->append(event); - vl->append(left); - vl->append(right); + + va_list lP; + va_start(lP, numvals); + for ( int i = 0; i < numvals; i++ ) + { + vl->append( va_arg(lP, Val*) ); + } + va_end(lP); + + mgr.QueueEvent(ev, vl, SOURCE_LOCAL); +} + +void InputMgr::SendEvent(EventHandlerPtr ev, list events) +{ + val_list* vl = new val_list; + + for ( list::iterator i = events.begin(); i != events.end(); i++ ) { + vl->append( *i ); + } - //mgr.Dispatch(new Event(handler, vl)); mgr.QueueEvent(ev, vl, SOURCE_LOCAL); } -Val* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { + +RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { reporter->InternalError("Need position"); return 0; diff --git a/src/InputMgr.h b/src/InputMgr.h index ba1ddafc92..cebed231e4 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -27,6 +27,8 @@ public: bool AddTableFilter(EnumVal *id, RecordVal* filter); bool RemoveTableFilter(EnumVal* id, const string &name); + bool AddEventFilter(EnumVal *id, RecordVal* filter); + bool RemoveEventFilter(EnumVal* id, const string &name); protected: friend class InputReader; @@ -46,10 +48,17 @@ protected: private: struct ReaderInfo; + void SendEntryTable(const InputReader* reader, int id, const LogVal* const *vals); + void PutTable(const InputReader* reader, int id, const LogVal* const *vals); + void SendEventFilterEvent(const InputReader* reader, EnumVal* type, int id, const LogVal* const *vals); + bool IsCompatibleType(BroType* t, bool atomic_only=false); bool UnrollRecordType(vector *fields, const RecordType *rec, const string& nameprepend); - void SendEvent(EventHandlerPtr ev, EnumVal* event, Val* left, Val* right); + + void SendEvent(EventHandlerPtr ev, const int numvals, ...); + void SendEvent(EventHandlerPtr ev, list events); + bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); HashKey* HashLogVals(const int num_elements, const LogVal* const *vals); int GetLogValLength(const LogVal* val); @@ -57,9 +66,8 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); - Val* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); - bool SendEvent(const string& name, const int num_vals, const LogVal* const *vals); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); @@ -68,7 +76,12 @@ private: string Hash(const string &input); - struct Filter; + class Filter; + class TableFilter; + class EventFilter; + + enum FilterType { TABLE_FILTER, EVENT_FILTER }; + }; diff --git a/src/input.bif b/src/input.bif index 1300f91bea..b1d57b1df6 100644 --- a/src/input.bif +++ b/src/input.bif @@ -9,6 +9,7 @@ module Input; type StreamDescription: record; type TableFilter: record; +type EventFilter: record; function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool %{ @@ -40,6 +41,18 @@ function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool return new Val( res, TYPE_BOOL); %} +function Input::__add_eventfilter%(id: Log::ID, filter: Input::EventFilter%) : bool + %{ + bool res = input_mgr->AddEventFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val( res, TYPE_BOOL ); + %} + +function Input::__remove_eventfilter%(id: Log::ID, name: string%) : bool + %{ + bool res = input_mgr->RemoveEventFilter(id->AsEnumVal(), name->AsString()->CheckString()); + return new Val( res, TYPE_BOOL); + %} + # Options for Ascii Reader module InputAscii; diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out new file mode 100644 index 0000000000..e32a2aea00 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -0,0 +1,21 @@ +Input::EVENT_NEW +1 +T +Input::EVENT_NEW +2 +T +Input::EVENT_NEW +3 +F +Input::EVENT_NEW +4 +F +Input::EVENT_NEW +5 +F +Input::EVENT_NEW +6 +F +Input::EVENT_NEW +7 +T diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 36e8171689..d9be733a1b 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -16,7 +16,6 @@ 7 T @TEST-END-FILE -redef InputAscii::empty_field = "EMPTY"; module A; @@ -24,25 +23,20 @@ export { redef enum Log::ID += { LOG }; } -type Idx: record { - i: int; -}; - type Val: record { + i: int; b: bool; }; -global destination: table[int] of Val = table(); - -event line(tpe: Input::Event, left: Idx, right: bool) { +event line(tpe: Input::Event, i: int, b: bool) { print tpe; - print left; - print right; + print i; + print b; } event bro_init() { Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); Input::force_update(A::LOG); } diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro new file mode 100644 index 0000000000..36e8171689 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -0,0 +1,48 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields i b +#types int bool +1 T +2 T +3 F +4 F +5 F +6 F +7 T +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Log::ID += { LOG }; +} + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; +}; + +global destination: table[int] of Val = table(); + +event line(tpe: Input::Event, left: Idx, right: bool) { + print tpe; + print left; + print right; +} + +event bro_init() +{ + Input::create_stream(A::LOG, [$source="input.log"]); + Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]); + Input::force_update(A::LOG); +} From be1b3ce5e1f239861013517b8d8c5a5fafbdf0bd Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 28 Nov 2011 13:29:02 -0600 Subject: [PATCH 40/58] Add note about independent component releases to Broxygen index. --- doc/index.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/index.rst b/doc/index.rst index ba3df81e7d..0a6b225431 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -46,6 +46,12 @@ Script Reference Other Bro Components -------------------- +The following are snapshots of documentation for components that come +with this version of Bro (|version|). Since they can also be used +independently, see the `download page +`_ for documentation of any +current, independent component releases. + .. toctree:: :maxdepth: 1 From 4975584e01af095cf6a6d10a4c09eb88a3f198e0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 13:28:44 -0800 Subject: [PATCH 41/58] change Log enum to Input enum. --- scripts/base/frameworks/input/main.bro | 10 +++++----- src/input.bif | 10 +++++----- src/types.bif | 1 - testing/btest/scripts/base/frameworks/input/basic.bro | 8 ++++---- testing/btest/scripts/base/frameworks/input/event.bro | 8 ++++---- .../base/frameworks/input/onecolumn-norecord.bro | 8 ++++---- .../scripts/base/frameworks/input/onecolumn-record.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/predicate.bro | 8 ++++---- .../btest/scripts/base/frameworks/input/twofilters.bro | 10 +++++----- 9 files changed, 35 insertions(+), 36 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index a036eeeebd..1a9e8b885c 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -58,28 +58,28 @@ module Input; #global filters: table[ID, string] of Filter; -function create_stream(id: Log::ID, description: Input::StreamDescription) : bool +function create_stream(id: Input::ID, description: Input::StreamDescription) : bool { return __create_stream(id, description); } -function remove_stream(id: Log::ID) : bool +function remove_stream(id: Input::ID) : bool { return __remove_stream(id); } -function force_update(id: Log::ID) : bool +function force_update(id: Input::ID) : bool { return __force_update(id); } -function add_tablefilter(id: Log::ID, filter: Input::TableFilter) : bool +function add_tablefilter(id: Input::ID, filter: Input::TableFilter) : bool { # filters[id, filter$name] = filter; return __add_tablefilter(id, filter); } -function remove_tablefilter(id: Log::ID, name: string) : bool +function remove_tablefilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_tablefilter(id, name); diff --git a/src/input.bif b/src/input.bif index b1d57b1df6..a7d561c060 100644 --- a/src/input.bif +++ b/src/input.bif @@ -11,31 +11,31 @@ type StreamDescription: record; type TableFilter: record; type EventFilter: record; -function Input::__create_stream%(id: Log::ID, description: Input::StreamDescription%) : bool +function Input::__create_stream%(id: Input::ID, description: Input::StreamDescription%) : bool %{ InputReader *the_reader = input_mgr->CreateStream(id->AsEnumVal(), description->AsRecordVal()); return new Val( the_reader != 0, TYPE_BOOL ); %} -function Input::__remove_stream%(id: Log::ID%) : bool +function Input::__remove_stream%(id: Input::ID%) : bool %{ bool res = input_mgr->RemoveStream(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__force_update%(id: Log::ID%) : bool +function Input::__force_update%(id: Input::ID%) : bool %{ bool res = input_mgr->ForceUpdate(id->AsEnumVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__add_tablefilter%(id: Log::ID, filter: Input::TableFilter%) : bool +function Input::__add_tablefilter%(id: Input::ID, filter: Input::TableFilter%) : bool %{ bool res = input_mgr->AddTableFilter(id->AsEnumVal(), filter->AsRecordVal()); return new Val( res, TYPE_BOOL ); %} -function Input::__remove_tablefilter%(id: Log::ID, name: string%) : bool +function Input::__remove_tablefilter%(id: Input::ID, name: string%) : bool %{ bool res = input_mgr->RemoveTableFilter(id->AsEnumVal(), name->AsString()->CheckString()); return new Val( res, TYPE_BOOL); diff --git a/src/types.bif b/src/types.bif index f90a954224..15fab1a7b1 100644 --- a/src/types.bif +++ b/src/types.bif @@ -180,7 +180,6 @@ enum Event %{ EVENT_REMOVED, %} - enum ID %{ Unknown, %} diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 3ad45eac69..10cc7376a8 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -45,8 +45,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index d9be733a1b..a07f0934a0 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -20,7 +20,7 @@ module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Val: record { @@ -36,7 +36,7 @@ event line(tpe: Input::Event, i: int, b: bool) { event bro_init() { - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_eventfilter(A::LOG, [$name="input", $fields=Val, $ev=line]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_eventfilter(A::INPUT, [$name="input", $fields=Val, $ev=line]); + Input::force_update(A::INPUT); } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 134ceb49e6..88838cc8d6 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index c07c9c826c..fc4d862cd3 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -15,7 +15,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -31,8 +31,8 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers]); - Input::force_update(A::LOG); + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); print servers; } diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 769536d2a6..5e6bae7b62 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -37,11 +37,11 @@ global servers: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in servers ) { print "VALID"; } diff --git a/testing/btest/scripts/base/frameworks/input/twofilters.bro b/testing/btest/scripts/base/frameworks/input/twofilters.bro index 575665e6e5..5af664e0e9 100644 --- a/testing/btest/scripts/base/frameworks/input/twofilters.bro +++ b/testing/btest/scripts/base/frameworks/input/twofilters.bro @@ -21,7 +21,7 @@ redef InputAscii::empty_field = "EMPTY"; module A; export { - redef enum Log::ID += { LOG }; + redef enum Input::ID += { INPUT }; } type Idx: record { @@ -38,13 +38,13 @@ global destination2: table[int] of Val = table(); event bro_init() { # first read in the old stuff into the table... - Input::create_stream(A::LOG, [$source="input.log"]); - Input::add_tablefilter(A::LOG, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=destination1, $want_record=F, $pred(typ: Input::Event, left: Idx, right: bool) = { return right; } ]); - Input::add_tablefilter(A::LOG, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); + Input::add_tablefilter(A::INPUT, [$name="input2",$idx=Idx, $val=Val, $destination=destination2]); - Input::force_update(A::LOG); + Input::force_update(A::INPUT); if ( 1 in destination1 ) { print "VALID"; } From 1abb1424b8f1e050f24ffb9a9a7829fd8e10c3f0 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 28 Nov 2011 14:15:04 -0800 Subject: [PATCH 42/58] begin documenting... --- doc/index.rst | 1 + doc/input.rst | 190 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 191 insertions(+) create mode 100644 doc/input.rst diff --git a/doc/index.rst b/doc/index.rst index 0a6b225431..975ee7def4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -20,6 +20,7 @@ Frameworks notice logging + input cluster signatures diff --git a/doc/input.rst b/doc/input.rst new file mode 100644 index 0000000000..78e96fe06e --- /dev/null +++ b/doc/input.rst @@ -0,0 +1,190 @@ +===================== +Loading Data into Bro +===================== + +.. rst-class:: opening + + Bro comes with a flexible input interface that allows to read + previously stored data. Data is either read into bro tables or + sent to scripts using events. + This document describes how the input framework can be used. + +.. contents:: + +Terminology +=========== + +Bro's input framework is built around three main abstracts, that are +very similar to the abstracts used in the logging framework: + + Input Streams + An input stream corresponds to a single input source + (usually a textfile). It defined the information necessary + to find the source (e.g. the filename) + + Filters + Each input stream has a set of filters attached to it, that + determine exaclty what kind of information is read. + There are two different kind of streams, event streams and table + streams. + By default, event streams generate an event for each line read + from the input source. + Table streams on the other hand read the input source in a bro + table for easy later access. + + Readers + A reader defines the input format for the specific input stream. + At the moment, Bro comes with only one type of reader, which can + read the tab seperated ASCII logfiles that were generated by the + logging framework. + + +Basics +====== + +For examples, please look at the unit tests in +``testing/btest/scripts/base/frameworks/input/``. + +A very basic example to open an input stream is: + +.. code:: bro + + module Foo; + + export { + # Create an ID for our new stream + redef enum Input::ID += { INPUT }; + } + + event bro_init() { + Input::create_stream(FOO::INPUT, [$source="input.log"]); + } + +The fields that can be set when creating a stream are: + + ``source`` + A mandatory string identifying the source of the data. + For the ASCII reader this is the filename. + + ``reader`` + The reader used for this stream. Default is ``READER_ASCII``. + + +Filters +======= + +Each filter defines the data fields that it wants to receive from the respective +input file. Depending on the type of filter, events or a table are created from +the data in the source file. + +Event Filters +------------- + +Event filters are filters that generate an event for each line in of the input source. + +For example, a simple filter retrieving the fields ``i`` and ``b`` from an inputSource +could be defined as follows: + +.. code:: bro + + type Val: record { + i: int; + b: bool; + }; + + event line(tpe: Input::Event, i: int, b: bool) { + # work with event data + } + + event bro_init { + # Input stream definition, etc + ... + + Input::add_eventfilter(Foo::INPUT, [$name="input", $fields=Val, $ev=line]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The fields that can be set for an event filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``fields`` + Name of a record type containing the fields, which should be retrieved from + the input stream. + + ``ev`` + The event which is fired, after a line has been read from the input source. + The first argument that is passed to the event is an Input::Event structure, + followed by the data, either inside of a record (if ``want_record is set``) or as + individual fields. + The Input::Event structure can contain information, if the received line is ``NEW``, has + been ``CHANGED`` or ``DELETED``. Singe the ascii reader cannot track this information + for event filters, the value is always ``NEW`` at the moment. + + ``want_record`` + Boolean value, that defines if the event wants to receive the fields inside of + a single record value, or individually (default). + +Table Filters +------------- + +Table filters are the second, more complex type of filter. + +Table filters store the information they read from an input source in a bro table. For example, +when reading a file that contains ip addresses and connection attemt information one could use +an approach similar to this: + +.. code:: bro + + type Idx: record { + a: addr; + }; + + type Val: record { + tries: count; + }; + + global conn_attempts: table[addr] of count = table(); + + event bro_init { + # Input stream definitions, etc. + ... + + Input::add_tablefilter(Foo::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=conn_attempts]); + + # read the file after all filters have been set + Input::force_update(Foo::INPUT); + } + +The table conn_attempts will then contain the information about connection attemps. + +The possible fields that can be set for an table filter are: + + ``name`` + A mandatory name for the filter that can later be used + to manipulate it further. + + ``idx`` + Record type that defines the index of the table + + ``val`` + Record type that defines the values of the table + + ``want_record`` + Defines if the values of the table should be stored as a record (default), + or as a simple value. Has to be set if Val contains more than one element. + + ``destination`` + The destination table + + ``ev`` + Optional event that is raised, when values are added to, changed in or deleted from the table. + Events are passed an Input::Event description as the first argument, the index record as the second argument + and the values as the third argument. + + ``pred`` + Optional predicate, that can prevent entries from being added to the table and events from being sent. From 2a6387129ceb68941beccb866822958c661ab2c4 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 11:25:11 -0800 Subject: [PATCH 43/58] documentation --- doc/scripts/DocSourcesList.cmake | 3 + scripts/base/frameworks/input/main.bro | 96 ++++++++++++++++++++------ 2 files changed, 78 insertions(+), 21 deletions(-) diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 9d99effc02..bd41d301c0 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal) rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro) +rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro) rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro) @@ -31,6 +32,8 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro) rest_target(${psd} base/frameworks/communication/main.bro) rest_target(${psd} base/frameworks/control/main.bro) rest_target(${psd} base/frameworks/dpd/main.bro) +rest_target(${psd} base/frameworks/input/main.bro) +rest_target(${psd} base/frameworks/input/readers/ascii.bro) rest_target(${psd} base/frameworks/intel/main.bro) rest_target(${psd} base/frameworks/logging/main.bro) rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 1a9e8b885c..66b13743b8 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -1,52 +1,106 @@ +##! The input framework provides a way to read previously stored data either +##! as an event stream or into a bro table. module Input; export { + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; + ## Stream decription type used for the `create_stream` method type StreamDescription: record { + ## String that allows the reader to find the source. + ## For `READER_ASCII`, this is the filename. source: string; + + ## Reader to use for this steam reader: Reader &default=default_reader; }; + ## TableFilter description type used for the `add_tablefilter` method. type TableFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - ## for tables - idx: any; - val: any; + ## Table which will contain the data read by the input framework destination: any; + ## Record that defines the values used as the index of the table + idx: any; + ## Record that defines the values used as the values of the table + val: any; + ## Defines if the value of the table is a record (default), or a single value. + ## Val can only contain one element when this is set to false. want_record: bool &default=T; + + ## The event that is raised each time a value is added to, changed in or removed from the table. + ## The event will receive an Input::Event enum as the first argument, the idx record as the second argument + ## and the value (record) as the third argument. ev: any &optional; # event containing idx, val as values. - ## decision function, that decides if an insertion, update or removal should really be executed. - ## or events should be thought + ## Predicate function, that can decide if an insertion, update or removal should really be executed. + ## Parameters are the same as for the event. If true is returned, the update is performed. If false + ## is returned, it is skipped pred: function(typ: Input::Event, left: any, right: any): bool &optional; }; + ## EventFilter description type used for the `add_eventfilter` method. type EventFilter: record { - ## descriptive name. for later removal + ## Descriptive name. Used to remove a filter at a later time name: string; - # the event - ev: any; - # record describing the fields + ## Record describing the fields to be retrieved from the source input. fields: any; - - # does the event want the field unrolled (default) or as a simple record value? + ## If want_record if false (default), the event receives each value in fields as a seperate argument. + ## If it is set to true, the event receives all fields in a signle record value. want_record: bool &default=F; + + ## The event that is rised each time a new line is received from the reader. + ## The event will receive an Input::Event enum as the first element, and the fields as the following arguments. + ev: any; + }; #const no_filter: Filter = [$name="", $idx="", $val="", $destination=""]; # Sentinel. - global create_stream: function(id: Log::ID, description: Input::StreamDescription) : bool; - global remove_stream: function(id: Log::ID) : bool; - global force_update: function(id: Log::ID) : bool; - global add_tablefilter: function(id: Log::ID, filter: Input::TableFilter) : bool; - global remove_tablefilter: function(id: Log::ID, name: string) : bool; - global add_eventfilter: function(id: Log::ID, filter: Input::EventFilter) : bool; - global remove_eventfilter: function(id: Log::ID, name: string) : bool; + ## Create a new input stream from a given source. Returns true on success. + ## + ## id: `Input::ID` enum value identifying this stream + ## description: `StreamDescription` record describing the source. + global create_stream: function(id: Input::ID, description: Input::StreamDescription) : bool; + + ## Remove a current input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream to be removed + global remove_stream: function(id: Input::ID) : bool; + + ## Forces the current input to be checked for changes. + ## + ## id: `Input::ID` enum value identifying the stream + global force_update: function(id: Input::ID) : bool; + + ## Adds a table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `TableFilter` record describing the filter. + global add_tablefilter: function(id: Input::ID, filter: Input::TableFilter) : bool; + + ## Removes a named table filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_tablefilter: function(id: Input::ID, name: string) : bool; + + ## Adds an event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## filter: the `EventFilter` record describing the filter. + global add_eventfilter: function(id: Input::ID, filter: Input::EventFilter) : bool; + + ## Removes a named event filter to a specific input stream. Returns true on success. + ## + ## id: `Input::ID` enum value identifying the stream + ## name: the name of the filter to be removed. + global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; } @@ -85,13 +139,13 @@ function remove_tablefilter(id: Input::ID, name: string) : bool return __remove_tablefilter(id, name); } -function add_eventfilter(id: Log::ID, filter: Input::EventFilter) : bool +function add_eventfilter(id: Input::ID, filter: Input::EventFilter) : bool { # filters[id, filter$name] = filter; return __add_eventfilter(id, filter); } -function remove_eventfilter(id: Log::ID, name: string) : bool +function remove_eventfilter(id: Input::ID, name: string) : bool { # delete filters[id, name]; return __remove_eventfilter(id, name); From a68e6b9fa49211a7c51c45f83ec0c610f03a956d Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 29 Nov 2011 14:32:53 -0800 Subject: [PATCH 44/58] allow sets to be read from files, convenience function for reading a file once, bug in destructor that could lead to a segfault. --- scripts/base/frameworks/input/main.bro | 34 +++++++++++- src/InputMgr.cc | 54 ++++++++++++++----- .../scripts/base/frameworks/input/basic.bro | 2 + 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro index 66b13743b8..c76eba80b9 100644 --- a/scripts/base/frameworks/input/main.bro +++ b/scripts/base/frameworks/input/main.bro @@ -4,6 +4,8 @@ module Input; export { + redef enum Input::ID += { TABLE_READ }; + ## The default input reader used. Defaults to `READER_ASCII`. const default_reader = READER_ASCII &redef; @@ -27,7 +29,8 @@ export { ## Record that defines the values used as the index of the table idx: any; ## Record that defines the values used as the values of the table - val: any; + ## If val is undefined, destination has to be a set. + val: any &optional; ## Defines if the value of the table is a record (default), or a single value. ## Val can only contain one element when this is set to false. want_record: bool &default=T; @@ -102,6 +105,14 @@ export { ## name: the name of the filter to be removed. global remove_eventfilter: function(id: Input::ID, name: string) : bool; #global get_filter: function(id: ID, name: string) : Filter; + + ## Convenience function for reading a specific input source exactly once using + ## exactly one tablefilter + ## + ## id: `Input::ID` enum value identifying the stream + ## description: `StreamDescription` record describing the source. + ## filter: the `TableFilter` record describing the filter. + global read_table: function(description: Input::StreamDescription, filter: Input::TableFilter) : bool; } @@ -151,6 +162,27 @@ function remove_eventfilter(id: Input::ID, name: string) : bool return __remove_eventfilter(id, name); } +function read_table(description: Input::StreamDescription, filter: Input::TableFilter) : bool { + local ok: bool = T; + # since we create and delete it ourselves this should be ok... at least for singlethreaded operation + local id: Input::ID = Input::TABLE_READ; + + ok = create_stream(id, description); + if ( ok ) { + ok = add_tablefilter(id, filter); + } + if ( ok ) { + ok = force_update(id); + } + if ( ok ) { + ok = remove_stream(id); + } else { + remove_stream(id); + } + + return ok; +} + #function get_filter(id: ID, name: string) : Filter # { # if ( [id, name] in filters ) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0df11ea359..7ec5974199 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -86,7 +86,8 @@ InputMgr::Filter::~Filter() { InputMgr::TableFilter::~TableFilter() { Unref(tab); Unref(itype); - Unref(rtype); + if ( rtype ) // can be 0 for sets + Unref(rtype); delete currDict; delete lastDict; @@ -110,6 +111,7 @@ InputMgr::ReaderInfo::~ReaderInfo() { while ( it != filters.end() ) { delete (*it).second; + ++it; } Unref(type); @@ -354,7 +356,10 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { Val* pred = fval->Lookup(rtype->FieldOffset("pred")); RecordType *idx = fval->Lookup(rtype->FieldOffset("idx"))->AsType()->AsTypeType()->Type()->AsRecordType(); - RecordType *val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + RecordType *val = 0; + if ( fval->Lookup(rtype->FieldOffset("val")) != 0 ) { + val = fval->Lookup(rtype->FieldOffset("val"))->AsType()->AsTypeType()->Type()->AsRecordType(); + } TableVal *dst = fval->Lookup(rtype->FieldOffset("destination"))->AsTableVal(); Val *want_record = fval->LookupWithDefault(rtype->FieldOffset("want_record")); @@ -408,9 +413,14 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { int idxfields = fieldsV.size(); - status = status || !UnrollRecordType(&fieldsV, val, ""); + if ( val ) // if we are not a set + status = status || !UnrollRecordType(&fieldsV, val, ""); + int valfields = fieldsV.size() - idxfields; + if ( !val ) + assert(valfields == 0); + if ( status ) { reporter->Error("Problem unrolling"); return false; @@ -429,7 +439,7 @@ bool InputMgr::AddTableFilter(EnumVal *id, RecordVal* fval) { filter->num_idx_fields = idxfields; filter->num_val_fields = valfields; filter->tab = dst->Ref()->AsTableVal(); - filter->rtype = val->Ref()->AsRecordType(); + filter->rtype = val ? val->Ref()->AsRecordType() : 0; filter->itype = idx->Ref()->AsRecordType(); filter->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; filter->currDict = new PDict(InputHash); @@ -681,7 +691,10 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c HashKey* idxhash = HashLogVals(filter->num_idx_fields, vals); //reporter->Error("Result: %d", (uint64_t) idxhash->Hash()); //reporter->Error("Hashing %d val fields", i->num_val_fields); - HashKey* valhash = HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + HashKey* valhash = 0; + if ( filter->num_val_fields > 0 ) + HashLogVals(filter->num_val_fields, vals+filter->num_idx_fields); + //reporter->Error("Result: %d", (uint64_t) valhash->Hash()); //reporter->Error("received entry with idxhash %d and valhash %d", (uint64_t) idxhash->Hash(), (uint64_t) valhash->Hash()); @@ -689,12 +702,13 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c InputHash *h = filter->lastDict->Lookup(idxhash); if ( h != 0 ) { // seen before - if ( h->valhash->Hash() == valhash->Hash() ) { - // ok, double. + if ( filter->num_val_fields == 0 || h->valhash->Hash() == valhash->Hash() ) { + // ok, exact duplicate filter->lastDict->Remove(idxhash); filter->currDict->Insert(idxhash, h); return; } else { + assert( filter->num_val_fields > 0 ); // updated filter->lastDict->Remove(idxhash); delete(h); @@ -708,7 +722,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { RecordVal * r = new RecordVal(filter->rtype); @@ -732,8 +748,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Val* oldval = 0; if ( updated == true ) { - // in that case, we need the old value to send the event (if we send an event). - oldval = filter->tab->Lookup(idxval); + assert(filter->num_val_fields > 0); + // in that case, we need the old value to send the event (if we send an event). + oldval = filter->tab->Lookup(idxval); } @@ -749,10 +766,12 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); } - val_list vl(3); + val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); vl.append(idxval); - vl.append(valval); + if ( filter->num_val_fields > 0 ) + vl.append(valval); + Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); Unref(v); @@ -794,6 +813,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c Ref(idxval); if ( updated ) { // in case of update send back the old value. + assert ( filter->num_val_fields > 0 ); ev = new EnumVal(BifEnum::Input::EVENT_CHANGED, BifType::Enum::Input::Event); assert ( oldval != 0 ); Ref(oldval); @@ -801,7 +821,11 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else { ev = new EnumVal(BifEnum::Input::EVENT_NEW, BifType::Enum::Input::Event); Ref(valval); - SendEvent(filter->event, 3, ev, idxval, valval); + if ( filter->num_val_fields == 0 ) { + SendEvent(filter->event, 3, ev, idxval); + } else { + SendEvent(filter->event, 3, ev, idxval, valval); + } } } } @@ -963,7 +987,9 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * Val* valval; int position = filter->num_idx_fields; - if ( filter->num_val_fields == 1 && !filter->want_record ) { + if ( filter->num_val_fields == 0 ) { + valval = 0; + } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { RecordVal * r = new RecordVal(filter->rtype); diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index 10cc7376a8..d1b6659eb6 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -49,4 +49,6 @@ event bro_init() Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); } From 78b24da7e4f84de7f499cc9365a30e7c966a7d14 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 15:02:03 -0800 Subject: [PATCH 45/58] start support for annotation for log field types. commit before rolling part of it back... --- src/Attr.h | 1 + src/InputMgr.cc | 10 ++++++++++ src/InputReaderAscii.h | 3 +++ src/LogMgr.cc | 8 +++++--- src/LogMgr.h | 6 ++++-- src/parse.y | 5 ++++- src/scan.l | 1 + 7 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/Attr.h b/src/Attr.h index 6c835dc61c..471acfe4ba 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_GROUP, ATTR_LOG, ATTR_ERROR_HANDLER, + ATTR_TYPE_COLUMN, // for input framework ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 7ec5974199..0cfb59ee90 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -561,6 +561,16 @@ bool InputMgr::UnrollRecordType(vector *fields, const RecordType *rec field->subtype = rec->FieldType(i)->AsSetType()->Indices()->PureType()->Tag(); } else if ( field->type == TYPE_VECTOR ) { field->subtype = rec->FieldType(i)->AsVectorType()->YieldType()->Tag(); + } else if ( field->type == TYPE_PORT && + rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN) ) { + // we have an annotation for the second column + + Val* c = rec->FieldDecl(i)->FindAttr(ATTR_TYPE_COLUMN)->AttrExpr()->Eval(0); + + assert(c); + assert(c->Type()->Tag() == TYPE_STRING); + + field->secondary_name = c->AsStringVal()->AsString()->CheckString(); } fields->push_back(field); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index 01169a3cfc..c174248454 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -12,8 +12,11 @@ struct FieldMapping { string name; TypeTag type; + // internal type for sets and vectors TypeTag subtype; int position; + // for ports: pos of the second field + int secondary_position; FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 7fc8d4ef86..dd4f9c8e5a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -83,7 +83,8 @@ bool LogField::Read(SerializationFormat* fmt) int t; int it; - bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); + bool success = (fmt->Read(&name, "name") && fmt->Read(&secondary_name, "secondary_name") && + fmt->Read(&t, "type") && fmt->Read(&it, "subtype") ); type = (TypeTag) t; subtype = (TypeTag) it; @@ -92,7 +93,8 @@ bool LogField::Read(SerializationFormat* fmt) bool LogField::Write(SerializationFormat* fmt) const { - return (fmt->Write(name, "name") && fmt->Write((int)type, "type") && fmt->Write((int)subtype, "subtype")); + return (fmt->Write(name, "name") && fmt->Write(secondary_name, "secondary_name") && fmt->Write((int)type, "type") && + fmt->Write((int)subtype, "subtype")); } LogVal::~LogVal() @@ -151,7 +153,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) if ( ! t->IsSet() ) return false; - return IsCompatibleType(t->AsSetType()->Indices()->PureType(), true); + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); } case TYPE_VECTOR: diff --git a/src/LogMgr.h b/src/LogMgr.h index b8530d29ab..4ccdeb793c 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -14,13 +14,15 @@ class SerializationFormat; // Description of a log field. struct LogField { string name; + // needed by input framework. port fields have two names (one for the port, one for the type) - this specifies the secondary name. + string secondary_name; TypeTag type; - // needed by input framework. otherwise it cannot determine the inner type of a set or vector. + // needed by input framework. otherwise it cannot determine the inner type of a set. TypeTag subtype; LogField() { } LogField(const LogField& other) - : name(other.name), type(other.type), subtype(other.subtype) { } + : name(other.name), secondary_name(other.secondary_name), type(other.type), subtype(other.subtype) { } // (Un-)serialize. bool Read(SerializationFormat* fmt); diff --git a/src/parse.y b/src/parse.y index 495931aae0..988a19714b 100644 --- a/src/parse.y +++ b/src/parse.y @@ -2,7 +2,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 88 +%expect 91 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,6 +24,7 @@ %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE %token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG TOK_ATTR_ERROR_HANDLER +%token TOK_ATTR_TYPE_COLUMN %token TOK_DEBUG @@ -1313,6 +1314,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_TYPE_COLUMN '=' expr + { $$ = new Attr(ATTR_TYPE_COLUMN, $3); } | TOK_ATTR_LOG { $$ = new Attr(ATTR_LOG); } | TOK_ATTR_ERROR_HANDLER diff --git a/src/scan.l b/src/scan.l index 7ebd7894e1..6a85e89780 100644 --- a/src/scan.l +++ b/src/scan.l @@ -308,6 +308,7 @@ when return TOK_WHEN; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; &priority return TOK_ATTR_PRIORITY; +&type_column return TOK_ATTR_TYPE_COLUMN; &read_expire return TOK_ATTR_EXPIRE_READ; &redef return TOK_ATTR_REDEF; &rotate_interval return TOK_ATTR_ROTATE_INTERVAL; From aecbbdd966c896b475591a3415578adc1629273e Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 46/58] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index dd4f9c8e5a..ed32e4e40b 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -118,6 +118,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -190,9 +194,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -305,9 +312,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1066,6 +1075,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1097,7 +1122,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 4ccdeb793c..d9d61236e0 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -38,10 +38,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -136,6 +138,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From 4a690484ecaf5ea086090d2b6ef855cc6c913cad Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:42:37 -0800 Subject: [PATCH 47/58] make port annotation work and ascii input reader way more rebust with better error messages. --- src/Attr.cc | 22 ++- src/InputMgr.cc | 44 ++++- src/InputMgr.h | 2 + src/InputReaderAscii.cc | 150 +++++++++--------- src/InputReaderAscii.h | 4 +- src/LogMgr.cc | 1 - .../scripts.base.frameworks.input.port/out | 6 + .../scripts/base/frameworks/input/port.bro | 39 +++++ 8 files changed, 187 insertions(+), 81 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.port/out create mode 100644 testing/btest/scripts/base/frameworks/input/port.bro diff --git a/src/Attr.cc b/src/Attr.cc index a5a350f452..1d610f7cb4 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -17,7 +17,7 @@ const char* attr_name(attr_tag t) "&persistent", "&synchronized", "&postprocessor", "&encrypt", "&match", "&disable_print_hook", "&raw_output", "&mergeable", "&priority", - "&group", "&log", "&error_handler", "(&tracked)", + "&group", "&log", "&error_handler", "&type_column", "(&tracked)", }; return attr_names[int(t)]; @@ -417,6 +417,26 @@ void Attributes::CheckAttr(Attr* a) Error("&log applied to a type that cannot be logged"); break; + case ATTR_TYPE_COLUMN: + { + if ( type->Tag() != TYPE_PORT ) + { + Error("type_column tag only applicable to ports"); + break; + } + + BroType* atype = a->AttrExpr()->Type(); + + if ( atype->Tag() != TYPE_STRING ) { + Error("type column needs to have a string argument"); + break; + } + + + break; + } + + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0cfb59ee90..612461bae8 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -1167,9 +1167,14 @@ int InputMgr::GetLogValLength(const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: length += sizeof(val->val.uint_val); break; + + case TYPE_PORT: + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) + length += val->val.port_val.proto->size(); + break; case TYPE_DOUBLE: case TYPE_TIME: @@ -1228,12 +1233,24 @@ int InputMgr::CopyLogVal(char *data, const int startpos, const LogVal* val) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: //*(data+startpos) = val->val.uint_val; memcpy(data+startpos, (const void*) &(val->val.uint_val), sizeof(val->val.uint_val)); return sizeof(val->val.uint_val); break; + case TYPE_PORT: { + int length = 0; + memcpy(data+startpos, (const void*) &(val->val.port_val.port), sizeof(val->val.port_val.port)); + length += sizeof(val->val.port_val.port); + if ( val->val.port_val.proto != 0 ) { + memcpy(data+startpos, val->val.port_val.proto->c_str(), val->val.port_val.proto->length()); + length += val->val.port_val.proto->size(); + } + return length; + break; + } + + case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: @@ -1320,6 +1337,24 @@ HashKey* InputMgr::HashLogVals(const int num_elements, const LogVal* const *vals } +TransportProto InputMgr::StringToProto(const string &proto) { + if ( proto == "unknown" ) { + return TRANSPORT_UNKNOWN; + } else if ( proto == "tcp" ) { + return TRANSPORT_TCP; + } else if ( proto == "udp" ) { + return TRANSPORT_UDP; + } else if ( proto == "icmp" ) { + return TRANSPORT_ICMP; + } + + //assert(false); + + reporter->Error("Tried to parse invalid/unknown protocol: %s", proto.c_str()); + + return TRANSPORT_UNKNOWN; +} + Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { if ( request_type->Tag() != TYPE_ANY && request_type->Tag() != val->type ) { @@ -1357,7 +1392,10 @@ Val* InputMgr::LogValToVal(const LogVal* val, BroType* request_type) { } case TYPE_PORT: - return new PortVal(val->val.uint_val); + if ( val->val.port_val.proto == 0 ) + return new PortVal(val->val.port_val.port); + else + return new PortVal(val->val.port_val.port, StringToProto(*val->val.port_val.proto)); break; case TYPE_ADDR: diff --git a/src/InputMgr.h b/src/InputMgr.h index cebed231e4..ba6e208fd1 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -68,6 +68,8 @@ private: Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + TransportProto StringToProto(const string &proto); + ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 84feb74e61..501022d58e 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -10,25 +10,27 @@ FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, int : name(arg_name), type(arg_type) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position) : name(arg_name), type(arg_type), subtype(arg_subtype) { position = arg_position; + secondary_position = -1; } FieldMapping::FieldMapping(const FieldMapping& arg) : name(arg.name), type(arg.type), subtype(arg.subtype) { position = arg.position; + secondary_position = arg.secondary_position; } FieldMapping FieldMapping::subType() { return FieldMapping(name, subtype, position); } - InputReaderAscii::InputReaderAscii() { file = 0; @@ -122,45 +124,45 @@ bool InputReaderAscii::ReadHeader() { return false; } + map fields; + + // construcr list of field names. + istringstream splitstream(line); + int pos=0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0])) + break; + + fields[s] = pos; + pos++; + } + + for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs... - istringstream splitstream(line); - unsigned int currTab = 0; - int wantFields = 0; - while ( splitstream ) { - string s; - if ( !getline(splitstream, s, separator[0])) - break; - // current found heading in s... compare if we want it - for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { - const LogField* field = (*it).second.fields[i]; - if ( field->name == s ) { - // cool, found field. note position - FieldMapping f(field->name, field->type, field->subtype, i); - (*it).second.columnMap.push_back(f); - wantFields++; - break; // done with searching + for ( unsigned int i = 0; i < (*it).second.num_fields; i++ ) { + const LogField* field = (*it).second.fields[i]; + + map::iterator fit = fields.find(field->name); + if ( fit == fields.end() ) { + Error(Fmt("Did not find requested field %s in input data file.", field->name.c_str())); + return false; + } + + + FieldMapping f(field->name, field->type, field->subtype, fields[field->name]); + if ( field->secondary_name != "" ) { + map::iterator fit2 = fields.find(field->secondary_name); + if ( fit2 == fields.end() ) { + Error(Fmt("Could not find requested port type field %s in input data file.", field->secondary_name.c_str())); + return false; } + f.secondary_position = fields[field->secondary_name]; } - - // look if we did push something... - if ( (*it).second.columnMap.size() == currTab ) { - // no, we didn't. note that... - FieldMapping empty; - (*it).second.columnMap.push_back(empty); - } - - // done - currTab++; - } - - if ( wantFields != (int) (*it).second.num_fields ) { - // we did not find all fields? - // :( - Error(Fmt("One of the requested fields could not be found in the input data file. Found %d fields, wanted %d. Filternum: %d", wantFields, (*it).second.num_fields, (*it).first)); - return false; + (*it).second.columnMap.push_back(f); } + } // well, that seems to have worked... @@ -220,10 +222,14 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: val->val.uint_val = atoi(s.c_str()); break; + case TYPE_PORT: + val->val.port_val.port = atoi(s.c_str()); + val->val.port_val.proto = 0; + break; + case TYPE_SUBNET: { int pos = s.find("/"); string width = s.substr(pos+1); @@ -346,59 +352,55 @@ bool InputReaderAscii::DoUpdate() { string line; while ( GetLine(line ) ) { + // split on tabs + istringstream splitstream(line); + + map stringfields; + int pos = 0; + while ( splitstream ) { + string s; + if ( !getline(splitstream, s, separator[0]) ) + break; + + stringfields[pos] = s; + pos++; + } + + pos--; // for easy comparisons of max element. for ( map::iterator it = filters.begin(); it != filters.end(); it++ ) { - // split on tabs - - istringstream splitstream(line); - LogVal** fields = new LogVal*[(*it).second.num_fields]; - //string string_fields[num_fields]; - unsigned int currTab = 0; - unsigned int currField = 0; - while ( splitstream ) { + int fpos = 0; + for ( vector::iterator fit = (*it).second.columnMap.begin(); + fit != (*it).second.columnMap.end(); + fit++ ){ - string s; - if ( !getline(splitstream, s, separator[0]) ) - break; - - - if ( currTab >= (*it).second.columnMap.size() ) { - Error("Tabs in heading do not match tabs in data?"); - //disabled = true; + if ( (*fit).position > pos || (*fit).secondary_position > pos ) { + Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", line.c_str(), pos, (*fit).position, (*fit).secondary_position)); return false; } - FieldMapping currMapping = (*it).second.columnMap[currTab]; - currTab++; - - if ( currMapping.IsEmpty() ) { - // well, that was easy - continue; - } - - if ( currField >= (*it).second.num_fields ) { - Error("internal error - fieldnum greater as possible"); - return false; - } - - LogVal* val = EntryToVal(s, currMapping); + LogVal* val = EntryToVal(stringfields[(*fit).position], *fit); if ( val == 0 ) { return false; } - fields[currMapping.position] = val; - //string_fields[currMapping.position] = s; + + if ( (*fit).secondary_position != -1 ) { + // we have a port definition :) + assert(val->type == TYPE_PORT ); + // Error(Fmt("Got type %d != PORT with secondary position!", val->type)); - currField++; - } - - if ( currField != (*it).second.num_fields ) { - Error("curr_field != num_fields in DoUpdate. Columns in file do not match column definition."); - return false; + val->val.port_val.proto = new string(stringfields[(*fit).secondary_position]); + } + + fields[fpos] = val; + + fpos++; } + assert ( (unsigned int) fpos == (*it).second.num_fields ); SendEntry((*it).first, fields); diff --git a/src/InputReaderAscii.h b/src/InputReaderAscii.h index c174248454..2670d785d5 100644 --- a/src/InputReaderAscii.h +++ b/src/InputReaderAscii.h @@ -21,10 +21,10 @@ struct FieldMapping { FieldMapping(const string& arg_name, const TypeTag& arg_type, int arg_position); FieldMapping(const string& arg_name, const TypeTag& arg_type, const TypeTag& arg_subtype, int arg_position); FieldMapping(const FieldMapping& arg); - FieldMapping() { position = -1; } + FieldMapping() { position = -1; secondary_position = -1; } FieldMapping subType(); - bool IsEmpty() { return position == -1; } + //bool IsEmpty() { return position == -1; } }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ed32e4e40b..307a2e24e8 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -1088,7 +1088,6 @@ string LogMgr::TransportProtoToString(TransportProto p) { } assert(false); - return ""; } LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out new file mode 100644 index 0000000000..6f2bd3271b --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -0,0 +1,6 @@ +Trying to find field: t +{ +[1.2.3.4] = [p=80/tcp], +[1.2.4.6] = [p=30/unknown], +[1.2.3.5] = [p=52/udp] +} diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro new file mode 100644 index 0000000000..6f98c363c7 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -0,0 +1,39 @@ +# +# @TEST-EXEC: bro %INPUT >out +# @TEST-EXEC: btest-diff out + +@TEST-START-FILE input.log +#fields i p t +1.2.3.4 80 tcp +1.2.3.5 52 udp +1.2.4.6 30 unknown +@TEST-END-FILE + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +export { + redef enum Input::ID += { INPUT }; +} + +type Idx: record { + i: addr; +}; + +type Val: record { + p: port &type_column="t"; +}; + +global servers: table[addr] of Val = table(); + +event bro_init() +{ + # first read in the old stuff into the table... + Input::create_stream(A::INPUT, [$source="input.log"]); + Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::force_update(A::INPUT); + print servers; + Input::remove_tablefilter(A::INPUT, "ssh"); + Input::remove_stream(A::INPUT); +} From 9f32f68a13f290f01747f02b05dbe4b8b63d2790 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:50:36 -0800 Subject: [PATCH 48/58] make test more robust. --- .../Baseline/scripts.base.frameworks.input.port/out | 9 +++------ testing/btest/scripts/base/frameworks/input/port.bro | 10 ++++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.port/out b/testing/btest/Baseline/scripts.base.frameworks.input.port/out index 6f2bd3271b..858551aa2f 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.port/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.port/out @@ -1,6 +1,3 @@ -Trying to find field: t -{ -[1.2.3.4] = [p=80/tcp], -[1.2.4.6] = [p=30/unknown], -[1.2.3.5] = [p=52/udp] -} +[p=80/tcp] +[p=52/udp] +[p=30/unknown] diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 6f98c363c7..c14892ae36 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -6,7 +6,7 @@ #fields i p t 1.2.3.4 80 tcp 1.2.3.5 52 udp -1.2.4.6 30 unknown +1.2.3.6 30 unknown @TEST-END-FILE redef InputAscii::empty_field = "EMPTY"; @@ -31,9 +31,11 @@ event bro_init() { # first read in the old stuff into the table... Input::create_stream(A::INPUT, [$source="input.log"]); - Input::add_tablefilter(A::INPUT, [$name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_tablefilter(A::INPUT, [$name="input", $idx=Idx, $val=Val, $destination=servers]); Input::force_update(A::INPUT); - print servers; - Input::remove_tablefilter(A::INPUT, "ssh"); + print servers[1.2.3.4]; + print servers[1.2.3.5]; + print servers[1.2.3.6]; + Input::remove_tablefilter(A::INPUT, "input"); Input::remove_stream(A::INPUT); } From eb64eeedcd27c543becf330d7da48db9aa541ea8 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 49/58] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 307a2e24e8..dd5bc4b66a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -119,8 +119,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From ca17a1cf46ce50a84038eecfa5da401351777d4b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 5 Dec 2011 16:18:54 -0800 Subject: [PATCH 50/58] make logging framework send the protocol to the writer. for use in future writers, that have a special type for port, which includes the protocol. --- src/LogMgr.cc | 32 +++++++++++++++++++++++++++++--- src/LogMgr.h | 4 ++++ src/LogWriterAscii.cc | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0b706f6417..8873b22b46 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -114,6 +114,10 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } + +// if ( type == TYPE_PORT && present ) +// delete val.port_val.proto; + } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -186,9 +190,12 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Read(&val.uint_val, "uint"); + case TYPE_PORT: + val.port_val.proto = new string; + return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -301,9 +308,11 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: return fmt->Write(val.uint_val, "uint"); + case TYPE_PORT: + return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + case TYPE_SUBNET: { uint32 net[4]; @@ -1062,6 +1071,22 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +string LogMgr::TransportProtoToString(TransportProto p) { + switch ( p ) { + case TRANSPORT_UNKNOWN: + return "unknown"; + case TRANSPORT_TCP: + return "tcp"; + case TRANSPORT_UDP: + return "udp"; + case TRANSPORT_ICMP: + return "icmp"; + } + + assert(false); + return ""; +} + LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1093,7 +1118,8 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; case TYPE_PORT: - lval->val.uint_val = val->AsPortVal()->Port(); + lval->val.port_val.port = val->AsPortVal()->Port(); + lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 10530960cb..52acd04be5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,10 +34,12 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; + struct port_t { bro_uint_t port; string* proto; }; union _val { bro_int_t int_val; bro_uint_t uint_val; + port_t port_val; uint32 addr_val[NUM_ADDR_WORDS]; subnet_type subnet_val; double double_val; @@ -132,6 +134,8 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); + string TransportProtoToString(TransportProto p); + vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 9b1fda3b62..c449c1a788 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -169,10 +169,13 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: desc->Add(val->val.uint_val); break; + case TYPE_PORT: + desc->Add(val->val.port_val.port); + break; + case TYPE_SUBNET: desc->Add(dotted_addr(val->val.subnet_val.net)); desc->Add("/"); From a0da991030836d53bc669b64ad3de9b4dba34070 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Tue, 6 Dec 2011 10:56:26 -0800 Subject: [PATCH 51/58] memleak fix. --- src/LogMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 8873b22b46..729979b4ef 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,8 +115,8 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } -// if ( type == TYPE_PORT && present ) -// delete val.port_val.proto; + if ( type == TYPE_PORT && present ) + delete val.port_val.proto; } From 4b3cc95f7206d61614a0193508acc0f60828e3df Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 7 Dec 2011 12:43:15 -0800 Subject: [PATCH 52/58] send enum instead of string --- src/LogMgr.cc | 50 ++++++++++++++++++++++++++------------------------ src/LogMgr.h | 4 +--- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 729979b4ef..58581e2943 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -115,9 +115,6 @@ LogVal::~LogVal() delete [] val.vector_val.vals; } - if ( type == TYPE_PORT && present ) - delete val.port_val.proto; - } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -192,9 +189,30 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_COUNTER: return fmt->Read(&val.uint_val, "uint"); - case TYPE_PORT: - val.port_val.proto = new string; - return fmt->Read(&val.port_val.port, "port") && fmt->Read(val.port_val.proto, "proto"); + case TYPE_PORT: { + int proto; + if ( ! (fmt->Read(&val.port_val.port, "port") && fmt->Read(&proto, "proto") ) ) { + return false; + } + + switch (proto) { + case 0: + val.port_val.proto = TRANSPORT_UNKNOWN; + break; + case 1: + val.port_val.proto = TRANSPORT_TCP; + break; + case 2: + val.port_val.proto = TRANSPORT_UDP; + break; + case 3: + val.port_val.proto = TRANSPORT_ICMP; + break; + default: + return false; + } + return true; + } case TYPE_SUBNET: { @@ -311,7 +329,7 @@ bool LogVal::Write(SerializationFormat* fmt) const return fmt->Write(val.uint_val, "uint"); case TYPE_PORT: - return fmt->Write(val.port_val.port, "port") && fmt->Write(*val.port_val.proto, "proto"); + return fmt->Write(val.port_val.port, "port") && fmt->Write(val.port_val.proto, "proto"); case TYPE_SUBNET: { @@ -1071,22 +1089,6 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -string LogMgr::TransportProtoToString(TransportProto p) { - switch ( p ) { - case TRANSPORT_UNKNOWN: - return "unknown"; - case TRANSPORT_TCP: - return "tcp"; - case TRANSPORT_UDP: - return "udp"; - case TRANSPORT_ICMP: - return "icmp"; - } - - assert(false); - return ""; -} - LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { if ( ! ty ) @@ -1119,7 +1121,7 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) case TYPE_PORT: lval->val.port_val.port = val->AsPortVal()->Port(); - lval->val.port_val.proto = new string(TransportProtoToString(val->AsPortVal()->PortType())); + lval->val.port_val.proto = val->AsPortVal()->PortType(); break; case TYPE_SUBNET: diff --git a/src/LogMgr.h b/src/LogMgr.h index 52acd04be5..8c2c8250f8 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -34,7 +34,7 @@ struct LogVal { // types we can log directly. struct set_t { bro_int_t size; LogVal** vals; }; typedef set_t vec_t; - struct port_t { bro_uint_t port; string* proto; }; + struct port_t { bro_uint_t port; TransportProto proto; }; union _val { bro_int_t int_val; @@ -134,8 +134,6 @@ private: Filter* FindFilter(EnumVal* id, StringVal* filter); WriterInfo* FindWriter(LogWriter* writer); - string TransportProtoToString(TransportProto p); - vector streams; // Indexed by stream enum. }; From e0b7dc04512a228b332e2b9aaddd39300646d5cc Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 8 Dec 2011 14:12:59 -0800 Subject: [PATCH 53/58] fix compile warnings --- src/InputMgr.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 00a1a26311..1fe3a82abe 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -866,8 +866,8 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { //while ( ( ih = i->lastDict->NextEntry(c) ) ) { while ( ( ih = filter->lastDict->NextEntry(lastDictIdxKey, c) ) ) { - ListVal * idx; - Val *val; + ListVal * idx = 0; + Val *val = 0; if ( filter->pred || filter->event ) { idx = filter->tab->RecoverIndex(ih->idxkey); From a14ec02d3b75788966db7dccb1e8a533e627300a Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:25 -0800 Subject: [PATCH 54/58] change empty field defenition like in logging framework --- scripts/base/frameworks/input/readers/ascii.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 9f630975a2..14c04757f7 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -12,7 +12,7 @@ export { const set_separator = "," &redef; ## String to use for empty fields. - const empty_field = "-" &redef; + const empty_field = "(empty)" &redef; ## String to use for an unset &optional field. const unset_field = "-" &redef; From 70a2cf67324c607014b067af13fc9c6887e571e3 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 19 Dec 2011 12:43:51 -0800 Subject: [PATCH 55/58] update baseline to include input framework --- .../coverage.bare-load-baseline/canonified_loaded_scripts.log | 4 ++++ .../canonified_loaded_scripts.log | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 8fab67304e..5a0de92a6e 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -18,4 +18,8 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/policy/misc/loaded-scripts.bro diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 3f77797df8..e727928579 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -18,6 +18,10 @@ scripts/base/init-bare.bro scripts/base/frameworks/logging/./postprocessors/__load__.bro scripts/base/frameworks/logging/./postprocessors/./scp.bro scripts/base/frameworks/logging/./writers/ascii.bro + scripts/base/frameworks/input/__load__.bro + scripts/base/frameworks/input/./main.bro + build/src/base/input.bif.bro + scripts/base/frameworks/input/./readers/ascii.bro scripts/base/init-default.bro scripts/base/utils/site.bro scripts/base/utils/./patterns.bro From eb53a3d1c8bb9b7a590fc4f5ba70a5ea5017b6ee Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 12 Jan 2012 11:51:12 -0800 Subject: [PATCH 56/58] make input framework compile with brov6 --- src/InputReaderAscii.cc | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/InputReaderAscii.cc b/src/InputReaderAscii.cc index 6da161a2bf..257cb4cf71 100644 --- a/src/InputReaderAscii.cc +++ b/src/InputReaderAscii.cc @@ -253,19 +253,37 @@ LogVal* InputReaderAscii::EntryToVal(string s, FieldMapping field) { val->val.subnet_val.width = atoi(width.c_str()); string addr = s.substr(0, pos); s = addr; - // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. + // NOTE: dotted_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... +#ifdef BROv6 + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.subnet_val.net, addr); + delete addr; + } else { + val->val.subnet_val.net[0] = val->val.subnet_val.net[1] = val->val.subnet_val.net[2] = 0; + val->val.subnet_val.net[3] = dotted_to_addr(s.c_str()); + } +#else val->val.subnet_val.net = dotted_to_addr(s.c_str()); +#endif break; } case TYPE_ADDR: { // NOTE: dottet_to_addr BREAKS THREAD SAFETY! it uses reporter. // Solve this some other time.... - addr_type t = dotted_to_addr(s.c_str()); #ifdef BROv6 - copy_addr(t, val->val.addr_val); + if ( s.find(':') != s.npos ) { + uint32* addr = dotted_to_addr6(s.c_str()); + copy_addr(val->val.addr_val, addr); + delete addr; + } else { + val->val.addr_val[0] = val->val.addr_val[1] = val->val.addr_val[2] = 0; + val->val.addr_val[3] = dotted_to_addr(s.c_str()); + } #else + uint32 t = dotted_to_addr(s.c_str()); copy_addr(&t, val->val.addr_val); #endif break; From ac1708f8436126d1be5b522461dc88c8e1fa6158 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 20 Jan 2012 12:33:48 -0800 Subject: [PATCH 57/58] fix handling of predicates - now the second argument that is sent to the predicate really is a recordVal and not a ListVal. --- src/InputMgr.cc | 52 ++++++++++++++++++++++++++++++------------------- src/InputMgr.h | 1 + 2 files changed, 33 insertions(+), 20 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 1fe3a82abe..0e01bd6333 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -737,22 +737,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[position], filter->rtype->FieldType(0)); } else { - RecordVal * r = new RecordVal(filter->rtype); - - for ( int j = 0; j < filter->rtype->NumFields(); j++) { - - Val* val = 0; - if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); - } else { - val = LogValToVal(vals[position], filter->rtype->FieldType(j)); - position++; - } - - r->Assign(j,val); - - } - valval = r; + valval = LogValToRecordVal(vals, filter->rtype, &position); } @@ -767,7 +752,9 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c // call filter first to determine if we really add / change the entry if ( filter->pred ) { EnumVal* ev; - Ref(idxval); + //Ref(idxval); + int startpos = 0; + Val* predidx = LogValToRecordVal(vals, filter->itype, &startpos); Ref(valval); if ( updated ) { @@ -778,7 +765,7 @@ void InputMgr::SendEntryTable(const InputReader* reader, int id, const LogVal* c val_list vl( 2 + (filter->num_val_fields > 0) ); // 2 if we don't have values, 3 otherwise. vl.append(ev); - vl.append(idxval); + vl.append(predidx); if ( filter->num_val_fields > 0 ) vl.append(valval); @@ -882,12 +869,14 @@ void InputMgr::EndCurrentSend(const InputReader* reader, int id) { // ask predicate, if we want to expire this element... EnumVal* ev = new EnumVal(BifEnum::Input::EVENT_REMOVED, BifType::Enum::Input::Event); - Ref(idx); + //Ref(idx); + int startpos = 0; + Val* predidx = ListValToRecordVal(idx, filter->itype, &startpos); Ref(val); val_list vl(3); vl.append(ev); - vl.append(idx); + vl.append(predidx); vl.append(val); Val* v = filter->pred->Call(&vl); bool result = v->AsBool(); @@ -1123,6 +1112,29 @@ void InputMgr::SendEvent(EventHandlerPtr ev, list events) } +RecordVal* InputMgr::ListValToRecordVal(ListVal* list, RecordType *request_type, int* position) { + RecordVal* rec = new RecordVal(request_type->AsRecordType()); + + int maxpos = list->Length(); + + for ( int i = 0; i < request_type->NumFields(); i++ ) { + assert ( (*position) <= maxpos ); + + Val* fieldVal = 0; + if ( request_type->FieldType(i)->Tag() == TYPE_RECORD ) { + fieldVal = ListValToRecordVal(list, request_type->FieldType(i)->AsRecordType(), position); + } else { + fieldVal = list->Index(*position); + (*position)++; + } + + rec->Assign(i, fieldVal); + } + + return rec; +} + + RecordVal* InputMgr::LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position) { if ( position == 0 ) { diff --git a/src/InputMgr.h b/src/InputMgr.h index 21af018a47..5623c15338 100644 --- a/src/InputMgr.h +++ b/src/InputMgr.h @@ -67,6 +67,7 @@ private: Val* LogValToVal(const LogVal* val, BroType* request_type); Val* LogValToIndexVal(int num_fields, const RecordType* type, const LogVal* const *vals); RecordVal* LogValToRecordVal(const LogVal* const *vals, RecordType *request_type, int* position); + RecordVal* ListValToRecordVal(ListVal* list, RecordType *request_type, int* position); ReaderInfo* FindReader(const InputReader* reader); ReaderInfo* FindReader(const EnumVal* id); From f24c50b49a88b22db612d098489eeac042122a76 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 20 Jan 2012 12:42:23 -0800 Subject: [PATCH 58/58] remove unnecessary stuff from function. --- src/InputMgr.cc | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/InputMgr.cc b/src/InputMgr.cc index 0e01bd6333..89530febe6 100644 --- a/src/InputMgr.cc +++ b/src/InputMgr.cc @@ -955,7 +955,6 @@ void InputMgr::SendEventFilterEvent(const InputReader* reader, EnumVal* type, in if ( filter->want_record ) { RecordVal * r = LogValToRecordVal(vals, filter->fields, &position); out_vals.push_back(r); - } else { for ( int j = 0; j < filter->fields->NumFields(); j++) { Val* val = 0; @@ -991,27 +990,7 @@ void InputMgr::PutTable(const InputReader* reader, int id, const LogVal* const * } else if ( filter->num_val_fields == 1 && !filter->want_record ) { valval = LogValToVal(vals[filter->num_idx_fields], filter->rtype->FieldType(filter->num_idx_fields)); } else { - RecordVal * r = new RecordVal(filter->rtype); - - for ( int j = 0; j < filter->rtype->NumFields(); j++) { - - Val* val = 0; - if ( filter->rtype->FieldType(j)->Tag() == TYPE_RECORD ) { - val = LogValToRecordVal(vals, filter->rtype->FieldType(j)->AsRecordType(), &position); - } else { - val = LogValToVal(vals[position], filter->rtype->FieldType(j)); - position++; - } - - if ( val == 0 ) { - reporter->InternalError("conversion error"); - return; - } - - r->Assign(j,val); - - } - valval = r; + valval = LogValToRecordVal(vals, filter->rtype, &position); } filter->tab->Assign(idxval, valval);